diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig index 7fb0b93bc1ca..12e4a41bf1f0 100644 --- a/drivers/gpu/drm/amd/amdgpu/Kconfig +++ b/drivers/gpu/drm/amd/amdgpu/Kconfig @@ -3,7 +3,6 @@ config DRM_AMDGPU tristate "AMD GPU" depends on DRM && PCI - depends on !UML select FW_LOADER select DRM_CLIENT select DRM_CLIENT_SELECTION diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 80b18bbd7f3a..5d7bfa59424a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1074,6 +1074,7 @@ struct amdgpu_device { long psp_timeout; uint64_t unique_id; + uint8_t unitid; uint64_t df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS]; /* enable runtime pm on the device */ @@ -1153,7 +1154,6 @@ struct amdgpu_device { bool apu_prefer_gtt; bool userq_halt_for_enforce_isolation; - struct work_struct userq_reset_work; struct amdgpu_uid *uid_info; struct amdgpu_uma_carveout_info uma_info; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c index 62807b65f2af..4c732e0f776e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c @@ -228,7 +228,6 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) u64 acp_base; u32 val = 0; u32 count = 0; - struct i2s_platform_data *i2s_pdata = NULL; struct amdgpu_device *adev = ip_block->adev; @@ -272,18 +271,18 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) goto failure; } - i2s_pdata = kzalloc_objs(struct i2s_platform_data, 1); - if (!i2s_pdata) { + adev->acp.i2s_pdata = kzalloc_objs(struct i2s_platform_data, 1); + if (!adev->acp.i2s_pdata) { r = -ENOMEM; goto failure; } - i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | - DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; - i2s_pdata[0].cap = DWC_I2S_PLAY | DWC_I2S_RECORD; - i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000; - i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET; - i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET; + adev->acp.i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | + DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; + adev->acp.i2s_pdata[0].cap = DWC_I2S_PLAY | DWC_I2S_RECORD; + adev->acp.i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000; + adev->acp.i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET; + adev->acp.i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET; adev->acp.acp_res[0].name = "acp2x_dma"; adev->acp.acp_res[0].flags = IORESOURCE_MEM; @@ -311,7 +310,7 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) adev->acp.acp_cell[1].id = 1; adev->acp.acp_cell[1].num_resources = 1; adev->acp.acp_cell[1].resources = &adev->acp.acp_res[1]; - adev->acp.acp_cell[1].platform_data = &i2s_pdata[0]; + adev->acp.acp_cell[1].platform_data = &adev->acp.i2s_pdata[0]; adev->acp.acp_cell[1].pdata_size = sizeof(struct i2s_platform_data); r = mfd_add_devices(adev->acp.parent, 0, adev->acp.acp_cell, 2, NULL, 0, NULL); if (r) @@ -336,53 +335,53 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) goto failure; } - i2s_pdata = kzalloc_objs(struct i2s_platform_data, 3); - if (!i2s_pdata) { + adev->acp.i2s_pdata = kzalloc_objs(struct i2s_platform_data, 3); + if (!adev->acp.i2s_pdata) { r = -ENOMEM; goto failure; } switch (adev->asic_type) { case CHIP_STONEY: - i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | - DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; + adev->acp.i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | + DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; break; default: - i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET; + adev->acp.i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET; } - i2s_pdata[0].cap = DWC_I2S_PLAY; - i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000; - i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_PLAY_REG_OFFSET; - i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_PLAY_REG_OFFSET; + adev->acp.i2s_pdata[0].cap = DWC_I2S_PLAY; + adev->acp.i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000; + adev->acp.i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_PLAY_REG_OFFSET; + adev->acp.i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_PLAY_REG_OFFSET; switch (adev->asic_type) { case CHIP_STONEY: - i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | - DW_I2S_QUIRK_COMP_PARAM1 | - DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; + adev->acp.i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | + DW_I2S_QUIRK_COMP_PARAM1 | + DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; break; default: - i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | - DW_I2S_QUIRK_COMP_PARAM1; + adev->acp.i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET | + DW_I2S_QUIRK_COMP_PARAM1; } - i2s_pdata[1].cap = DWC_I2S_RECORD; - i2s_pdata[1].snd_rates = SNDRV_PCM_RATE_8000_96000; - i2s_pdata[1].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET; - i2s_pdata[1].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET; + adev->acp.i2s_pdata[1].cap = DWC_I2S_RECORD; + adev->acp.i2s_pdata[1].snd_rates = SNDRV_PCM_RATE_8000_96000; + adev->acp.i2s_pdata[1].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET; + adev->acp.i2s_pdata[1].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET; - i2s_pdata[2].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET; + adev->acp.i2s_pdata[2].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET; switch (adev->asic_type) { case CHIP_STONEY: - i2s_pdata[2].quirks |= DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; + adev->acp.i2s_pdata[2].quirks |= DW_I2S_QUIRK_16BIT_IDX_OVERRIDE; break; default: break; } - i2s_pdata[2].cap = DWC_I2S_PLAY | DWC_I2S_RECORD; - i2s_pdata[2].snd_rates = SNDRV_PCM_RATE_8000_96000; - i2s_pdata[2].i2s_reg_comp1 = ACP_BT_COMP1_REG_OFFSET; - i2s_pdata[2].i2s_reg_comp2 = ACP_BT_COMP2_REG_OFFSET; + adev->acp.i2s_pdata[2].cap = DWC_I2S_PLAY | DWC_I2S_RECORD; + adev->acp.i2s_pdata[2].snd_rates = SNDRV_PCM_RATE_8000_96000; + adev->acp.i2s_pdata[2].i2s_reg_comp1 = ACP_BT_COMP1_REG_OFFSET; + adev->acp.i2s_pdata[2].i2s_reg_comp2 = ACP_BT_COMP2_REG_OFFSET; adev->acp.acp_res[0].name = "acp2x_dma"; adev->acp.acp_res[0].flags = IORESOURCE_MEM; @@ -420,21 +419,21 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) adev->acp.acp_cell[1].id = 1; adev->acp.acp_cell[1].num_resources = 1; adev->acp.acp_cell[1].resources = &adev->acp.acp_res[1]; - adev->acp.acp_cell[1].platform_data = &i2s_pdata[0]; + adev->acp.acp_cell[1].platform_data = &adev->acp.i2s_pdata[0]; adev->acp.acp_cell[1].pdata_size = sizeof(struct i2s_platform_data); adev->acp.acp_cell[2].name = "designware-i2s"; adev->acp.acp_cell[2].id = 2; adev->acp.acp_cell[2].num_resources = 1; adev->acp.acp_cell[2].resources = &adev->acp.acp_res[2]; - adev->acp.acp_cell[2].platform_data = &i2s_pdata[1]; + adev->acp.acp_cell[2].platform_data = &adev->acp.i2s_pdata[1]; adev->acp.acp_cell[2].pdata_size = sizeof(struct i2s_platform_data); adev->acp.acp_cell[3].name = "designware-i2s"; adev->acp.acp_cell[3].id = 3; adev->acp.acp_cell[3].num_resources = 1; adev->acp.acp_cell[3].resources = &adev->acp.acp_res[3]; - adev->acp.acp_cell[3].platform_data = &i2s_pdata[2]; + adev->acp.acp_cell[3].platform_data = &adev->acp.i2s_pdata[2]; adev->acp.acp_cell[3].pdata_size = sizeof(struct i2s_platform_data); r = mfd_add_devices(adev->acp.parent, 0, adev->acp.acp_cell, ACP_DEVS, NULL, 0, NULL); @@ -491,7 +490,7 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) return 0; failure: - kfree(i2s_pdata); + kfree(adev->acp.i2s_pdata); kfree(adev->acp.acp_res); kfree(adev->acp.acp_cell); kfree(adev->acp.acp_genpd); @@ -556,6 +555,7 @@ static int acp_hw_fini(struct amdgpu_ip_block *ip_block) acp_genpd_remove_device); mfd_remove_devices(adev->acp.parent); + kfree(adev->acp.i2s_pdata); kfree(adev->acp.acp_res); kfree(adev->acp.acp_genpd); kfree(adev->acp.acp_cell); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h index a288ce25c176..13b48c582314 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h @@ -35,6 +35,7 @@ struct amdgpu_acp { struct mfd_cell *acp_cell; struct resource *acp_res; struct acp_pm_domain *acp_genpd; + struct i2s_platform_data *i2s_pdata; }; extern const struct amdgpu_ip_block_version acp_ip_block; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index d9e283f3b57d..9783a3cefb04 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -36,6 +36,9 @@ #include "amdgpu_ras.h" #include "amdgpu_umc.h" #include "amdgpu_reset.h" +#if IS_ENABLED(CONFIG_HSA_AMD) +#include "kfd_priv.h" +#endif /* Total memory size in system memory and all GPU VRAM. Used to * estimate worst case amount of memory to reserve for page tables @@ -320,6 +323,28 @@ void amdgpu_amdkfd_gpu_reset(struct amdgpu_device *adev) (void)amdgpu_reset_domain_schedule(adev->reset_domain, &adev->kfd.reset_work); } +void amdgpu_amdkfd_clear_kfd_mapping(struct amdgpu_device *adev) +{ +#if IS_ENABLED(CONFIG_HSA_AMD) + struct kfd_dev *kfd = adev->kfd.dev; + unsigned int i; + + if (!kfd) + return; + + for (i = 0; i < kfd->num_nodes; i++) { + struct kfd_node *node = kfd->nodes[i]; + + kfd_dev_unmap_mapping_range(KFD_MMAP_TYPE_DOORBELL | + KFD_MMAP_GPU_ID(node->id), + kfd_doorbell_process_slice(kfd)); + kfd_dev_unmap_mapping_range(KFD_MMAP_TYPE_MMIO | + KFD_MMAP_GPU_ID(node->id), + PAGE_SIZE); + } +#endif +} + int amdgpu_amdkfd_alloc_kernel_mem(struct amdgpu_device *adev, size_t size, u32 domain, void **mem_obj, uint64_t *gpu_addr, void **cpu_ptr, bool cp_mqd_gfx9) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 2bf6a31c194d..5333e052d56d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -360,6 +360,7 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag, int8_t xcp_id); void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag, int8_t xcp_id); +void amdgpu_amdkfd_clear_kfd_mapping(struct amdgpu_device *adev); u64 amdgpu_amdkfd_xcp_memory_size(struct amdgpu_device *adev, int xcp_id); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c index 3698dd0330ff..acd22bff1882 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c @@ -36,6 +36,21 @@ #include "atombios_encoders.h" #include "bif/bif_4_1_d.h" +/* VBIOS-reported table size is unchecked against the image; cap iterations and + * adev->i2c_bus[] indexing to AMDGPU_MAX_I2C_BUS. + */ +static int amdgpu_atombios_gpio_i2c_num_entries(uint16_t size) +{ + u32 bytes; + + if (size < sizeof(ATOM_COMMON_TABLE_HEADER)) + return 0; + + bytes = size - sizeof(ATOM_COMMON_TABLE_HEADER); + return (int)min_t(u32, bytes / sizeof(ATOM_GPIO_I2C_ASSIGMENT), + AMDGPU_MAX_I2C_BUS); +} + static struct amdgpu_i2c_bus_rec amdgpu_atombios_get_bus_rec_for_i2c_gpio(ATOM_GPIO_I2C_ASSIGMENT *gpio) { struct amdgpu_i2c_bus_rec i2c; @@ -96,8 +111,7 @@ struct amdgpu_i2c_bus_rec amdgpu_atombios_lookup_i2c_gpio(struct amdgpu_device * if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset); - num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / - sizeof(ATOM_GPIO_I2C_ASSIGMENT); + num_indices = amdgpu_atombios_gpio_i2c_num_entries(size); gpio = &i2c_info->asGPIO_Info[0]; for (i = 0; i < num_indices; i++) { @@ -127,8 +141,7 @@ void amdgpu_atombios_i2c_init(struct amdgpu_device *adev) if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset); - num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / - sizeof(ATOM_GPIO_I2C_ASSIGMENT); + num_indices = amdgpu_atombios_gpio_i2c_num_entries(size); gpio = &i2c_info->asGPIO_Info[0]; for (i = 0; i < num_indices; i++) { @@ -158,8 +171,7 @@ void amdgpu_atombios_oem_i2c_init(struct amdgpu_device *adev, u8 i2c_id) if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset); - num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / - sizeof(ATOM_GPIO_I2C_ASSIGMENT); + num_indices = amdgpu_atombios_gpio_i2c_num_entries(size); gpio = &i2c_info->asGPIO_Info[0]; for (i = 0; i < num_indices; i++) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c index 004edc28d0cc..d5e59c24d907 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c @@ -484,7 +484,7 @@ static u32 amdgpu_cper_ring_get_ent_sz(struct amdgpu_ring *ring, u64 pos) void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count) { - u64 pos, wptr_old, rptr; + u64 pos, wptr_old, rptr, next_rptr; int rec_cnt_dw = count >> 2; u32 chunk, ent_sz; u8 *s = (u8 *)src; @@ -525,9 +525,19 @@ void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count) do { ent_sz = amdgpu_cper_ring_get_ent_sz(ring, pos); + next_rptr = rptr; + if (ent_sz >= sizeof(u32)) + next_rptr = (rptr + (ent_sz >> 2)) & ring->ptr_mask; - rptr += (ent_sz >> 2); - rptr &= ring->ptr_mask; + if (next_rptr == rptr) { + /* Corrupt entry size, reset the ring to avoid an infinite loop. */ + rptr = ring->wptr; + *ring->rptr_cpu_addr = rptr; + ring->count_dw = (ring->ring_size - 4) >> 2; + goto out_unlock; + } + + rptr = next_rptr; *ring->rptr_cpu_addr = rptr; pos = rptr; @@ -536,6 +546,8 @@ void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count) if (ring->count_dw >= rec_cnt_dw) ring->count_dw -= rec_cnt_dw; + +out_unlock: mutex_unlock(&ring->adev->cper.ring_lock); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index b951b42d66bd..389bad724273 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -2126,6 +2126,9 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev) debugfs_create_x32("amdgpu_smu_debug", 0600, root, &adev->pm.smu_debug_mask); + debugfs_create_x64("unique_id", 0444, root, &adev->unique_id); + debugfs_create_x8("unitid", 0444, root, &adev->unitid); + ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev, &fops_ib_preempt); if (IS_ERR(ent)) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index be42e8f01def..21a3fb574d53 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3789,7 +3789,6 @@ int amdgpu_device_init(struct amdgpu_device *adev, } INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); - INIT_WORK(&adev->userq_reset_work, amdgpu_userq_reset_work); amdgpu_coredump_init(adev); @@ -5485,7 +5484,7 @@ static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev) if (!amdgpu_sriov_vf(adev)) cancel_work(&adev->reset_work); #endif - cancel_work(&adev->userq_reset_work); + amdgpu_userq_mgr_cancel_reset_work(adev); if (adev->kfd.dev) cancel_work(&adev->kfd.reset_work); @@ -5843,6 +5842,12 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, /* We need to lock reset domain only once both for XGMI and single device */ amdgpu_device_recovery_get_reset_lock(adev, &device_list); + /* unmap all the mappings of doorbell and framebuffer to prevent user space from + * accessing them + */ + unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); + amdgpu_amdkfd_clear_kfd_mapping(adev); + amdgpu_device_halt_activities(adev, job, reset_context, &device_list, hive, need_emergency_restart); if (need_emergency_restart) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index 623f7a399d65..21f6761624c2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -305,7 +305,7 @@ static int amdgpu_discovery_get_tmr_info(struct amdgpu_device *adev, adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].offset; adev->discovery.size = adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].size_kb << 10; - if (!adev->discovery.offset || !adev->discovery.size) + if (!adev->discovery.size) return -EINVAL; } else { goto out; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 99688391e70b..1781c0c3d010 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -2810,12 +2810,11 @@ static int amdgpu_runtime_idle_check_userq(struct device *dev) return xa_empty(&adev->userq_doorbell_xa) ? 0 : -EBUSY; } -static int amdgpu_pmops_runtime_suspend(struct device *dev) +static int amdgpu_pmops_runtime_checks(struct device *dev) { - struct pci_dev *pdev = to_pci_dev(dev); - struct drm_device *drm_dev = pci_get_drvdata(pdev); + struct drm_device *drm_dev = dev_get_drvdata(dev); struct amdgpu_device *adev = drm_to_adev(drm_dev); - int ret, i; + int ret; if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) { pm_runtime_forbid(dev); @@ -2825,7 +2824,27 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev) ret = amdgpu_runtime_idle_check_display(dev); if (ret) return ret; - ret = amdgpu_runtime_idle_check_userq(dev); + + return amdgpu_runtime_idle_check_userq(dev); +} + +static int amdgpu_pmops_runtime_idle(struct device *dev) +{ + int ret; + + ret = amdgpu_pmops_runtime_checks(dev); + pm_runtime_autosuspend(dev); + return ret; +} + +static int amdgpu_pmops_runtime_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct drm_device *drm_dev = pci_get_drvdata(pdev); + struct amdgpu_device *adev = drm_to_adev(drm_dev); + int ret, i; + + ret = amdgpu_pmops_runtime_checks(dev); if (ret) return ret; @@ -2937,27 +2956,6 @@ static int amdgpu_pmops_runtime_resume(struct device *dev) return 0; } -static int amdgpu_pmops_runtime_idle(struct device *dev) -{ - struct drm_device *drm_dev = dev_get_drvdata(dev); - struct amdgpu_device *adev = drm_to_adev(drm_dev); - int ret; - - if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) { - pm_runtime_forbid(dev); - return -EBUSY; - } - - ret = amdgpu_runtime_idle_check_display(dev); - if (ret) - goto done; - - ret = amdgpu_runtime_idle_check_userq(dev); -done: - pm_runtime_autosuspend(dev); - return ret; -} - static int amdgpu_drm_release(struct inode *inode, struct file *filp) { struct drm_file *file_priv = filp->private_data; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 9ef80bca4102..27be5083f2af 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -509,6 +510,9 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, if (offset_in_page(args->addr | args->size)) return -EINVAL; + if (!access_ok((void __user *)(uintptr_t)args->addr, args->size)) + return -EFAULT; + /* reject unknown flag values */ if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY | AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE | @@ -822,7 +826,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, struct drm_syncobj *timeline_syncobj = NULL; struct dma_fence_chain *timeline_chain = NULL; struct drm_exec exec; - uint64_t vm_size; + uint64_t vm_size, tmp; int r = 0; /* Validate virtual address range against reserved regions. */ @@ -846,7 +850,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE; vm_size -= AMDGPU_VA_RESERVED_TOP; - if (args->va_address + args->map_size > vm_size) { + if (check_add_overflow(args->va_address, args->map_size, &tmp) || tmp > vm_size) { dev_dbg(dev->dev, "va_address 0x%llx is in top reserved area 0x%llx\n", args->va_address + args->map_size, vm_size); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c index 620fddde4c4d..a5d26b943f6d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c @@ -199,11 +199,18 @@ int amdgpu_gtt_mgr_alloc_entries(struct amdgpu_gtt_mgr *mgr, enum drm_mm_insert_mode mode) { struct amdgpu_device *adev = container_of(mgr, typeof(*adev), mman.gtt_mgr); + u32 alignment = 0; int r; + /* Align to TLB L2 cache entry size to work around "V bit HW bug" */ + if (adev->asic_type == CHIP_TAHITI) { + alignment = 32 * 1024 / AMDGPU_GPU_PAGE_SIZE; + num_pages = ALIGN(num_pages, alignment); + } + spin_lock(&mgr->lock); r = drm_mm_insert_node_in_range(&mgr->mm, mm_node, num_pages, - 0, GART_ENTRY_WITHOUT_BO_COLOR, 0, + alignment, GART_ENTRY_WITHOUT_BO_COLOR, 0, adev->gmc.gart_size >> PAGE_SHIFT, mode); spin_unlock(&mgr->lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c index 610dc2ca67a2..c9467b26e42c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c @@ -312,6 +312,9 @@ int amdgpu_mes_suspend(struct amdgpu_device *adev, u32 xcc_id) memset(&input, 0x0, sizeof(struct mes_suspend_gang_input)); input.suspend_all_gangs = 1; input.xcc_id = xcc_id; + if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) && + ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x71)) + input.suspend_all_sdma_gangs = 1; /* * Avoid taking any other locks under MES lock to avoid circular diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h index 31cad8a5bb9f..93990d4990f2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h @@ -298,6 +298,7 @@ struct mes_unmap_legacy_queue_input { struct mes_suspend_gang_input { uint32_t xcc_id; bool suspend_all_gangs; + bool suspend_all_sdma_gangs; uint64_t gang_context_addr; uint64_t suspend_fence_addr; uint32_t suspend_fence_value; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 912c9afaf9e1..4d68732d6223 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -96,7 +96,8 @@ struct amdgpu_bo_va { * if non-zero, cannot unmap from GPU because user queues may still access it */ unsigned int queue_refcount; - atomic_t userq_va_mapped; + /* Indicates if this buffer is mapped for any user queue. Once set, never reset. */ + bool userq_va_mapped; }; struct amdgpu_bo { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index d525443e31f6..665a2632379d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -857,7 +857,11 @@ static int psp_load_toc(struct psp_context *psp, struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); /* Copy toc to psp firmware private buffer */ - psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes); + ret = psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes); + if (ret) { + release_psp_cmd_buf(psp); + return ret; + } psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc.size_bytes); @@ -1173,8 +1177,11 @@ static int psp_rl_load(struct amdgpu_device *adev) cmd = acquire_psp_cmd_buf(psp); - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes); + ret = psp_copy_fw(psp, psp->rl.start_addr, psp->rl.size_bytes); + if (ret) { + release_psp_cmd_buf(psp); + return ret; + } cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW; cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr); @@ -1760,8 +1767,12 @@ int psp_ta_load(struct psp_context *psp, struct ta_context *context) cmd = acquire_psp_cmd_buf(psp); - psp_copy_fw(psp, context->bin_desc.start_addr, - context->bin_desc.size_bytes); + ret = psp_copy_fw(psp, context->bin_desc.start_addr, + context->bin_desc.size_bytes); + if (ret) { + release_psp_cmd_buf(psp); + return ret; + } if (amdgpu_virt_xgmi_migrate_enabled(psp->adev) && context->mem_context.shared_bo) @@ -4562,17 +4573,24 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev, return count; } -void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size) +int psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size) { int idx; if (!drm_dev_enter(adev_to_drm(psp->adev), &idx)) - return; + return -ENODEV; + + if (!bin_size || bin_size > PSP_1_MEG) { + dev_err(psp->adev->dev, "PSP firmware is invalid\n"); + drm_dev_exit(idx); + return -EINVAL; + } memset(psp->fw_pri_buf, 0, PSP_1_MEG); memcpy(psp->fw_pri_buf, start_addr, bin_size); drm_dev_exit(idx); + return 0; } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index 4197179a7701..2d838b1b2b11 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -638,7 +638,7 @@ int psp_get_fw_attestation_records_addr(struct psp_context *psp, int psp_update_fw_reservation(struct psp_context *psp); int psp_load_fw_list(struct psp_context *psp, struct amdgpu_firmware_info **ucode_list, int ucode_count); -void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size); +int psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size); int psp_spatial_partition(struct psp_context *psp, int mode); int psp_memory_partition(struct psp_context *psp, int mode); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 6c644cfe6695..3e1f8b15f75d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -3093,6 +3093,25 @@ static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, return -EINVAL; } +static bool __check_record_in_range(struct amdgpu_device *adev, + struct eeprom_table_record *bps, int count) +{ + int i; + + for (i = 0; i < count; i++) { + if (bps[i].retired_page >= + (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT)) { + dev_warn(adev->dev, + "Recorded address out of range: 0x%llx, 0x%llx, 0x%x, 0x%x\n", + bps[i].address, bps[i].retired_page, + bps[i].mem_channel, bps[i].mcumc_id); + return false; + } + } + + return true; +} + static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, struct eeprom_table_record *bps, int count) { @@ -3100,6 +3119,9 @@ static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, struct amdgpu_ras *con = amdgpu_ras_get_context(adev); struct ras_err_handler_data *data = con->eh_data; + if (!__check_record_in_range(adev, bps, count)) + return 0; + for (j = 0; j < count; j++) { if (!data->space_left && amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { @@ -5641,6 +5663,11 @@ int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; int ret = 0; + if (pfn >= (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT)) { + dev_warn(adev->dev, "Ignoring out-of-range bad page 0x%llx", start); + return 0; + } + if (amdgpu_ras_check_critical_address(adev, start)) return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index 0c57fe259894..c61389a07982 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -1051,6 +1051,7 @@ int amdgpu_ras_eeprom_read_idx(struct amdgpu_ras_eeprom_control *control, uint64_t ts, end_idx; int i, ret; u64 mca, ipid; + u32 cu, mem_channel, mcumc_id; if (!amdgpu_ras_smu_eeprom_supported(adev)) return 0; @@ -1079,9 +1080,10 @@ int amdgpu_ras_eeprom_read_idx(struct amdgpu_ras_eeprom_control *control, record[i - rec_idx].err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE; adev->umc.ras->mca_ipid_parse(adev, ipid, - (uint32_t *)&(record[i - rec_idx].cu), - (uint32_t *)&(record[i - rec_idx].mem_channel), - (uint32_t *)&(record[i - rec_idx].mcumc_id), NULL); + &cu, &mem_channel, &mcumc_id, NULL); + record[i - rec_idx].cu = (u8)cu; + record[i - rec_idx].mem_channel = (u8)mem_channel; + record[i - rec_idx].mcumc_id = (u8)mcumc_id; } return 0; @@ -1632,6 +1634,14 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control) } control->ras_fri = RAS_OFFSET_TO_INDEX(control, hdr->first_rec_offset); + if (hdr->first_rec_offset < control->ras_record_offset || + control->ras_fri >= control->ras_max_record_count) { + dev_err(adev->dev, + "RAS header invalid, ras_fri: %u, first_rec_offset:0x%x", + control->ras_fri, hdr->first_rec_offset); + return -EINVAL; + } + control->ras_num_mca_recs = 0; control->ras_num_pa_recs = 0; return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c index a0b479d5fff1..f4be19223588 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c @@ -175,11 +175,14 @@ int amdgpu_seq64_alloc(struct amdgpu_device *adev, u64 *va, { unsigned long bit_pos; - bit_pos = find_first_zero_bit(adev->seq64.used, adev->seq64.num_sem); - if (bit_pos >= adev->seq64.num_sem) - return -ENOSPC; + for (;;) { + bit_pos = find_first_zero_bit(adev->seq64.used, adev->seq64.num_sem); + if (bit_pos >= adev->seq64.num_sem) + return -ENOSPC; - __set_bit(bit_pos, adev->seq64.used); + if (!test_and_set_bit(bit_pos, adev->seq64.used)) + break; + } *va = bit_pos * sizeof(u64) + amdgpu_seq64_get_va_base(adev); @@ -205,7 +208,7 @@ void amdgpu_seq64_free(struct amdgpu_device *adev, u64 va) bit_pos = (va - amdgpu_seq64_get_va_base(adev)) / sizeof(u64); if (bit_pos < adev->seq64.num_sem) - __clear_bit(bit_pos, adev->seq64.used); + clear_bit(bit_pos, adev->seq64.used); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c index 0238c2798de4..b8ed931f8a40 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c @@ -130,6 +130,7 @@ void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev, if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && adev->umc.ras->ras_block.hw_ops->query_ras_error_address && adev->umc.max_ras_err_cnt_per_query) { + kfree(err_data->err_addr); err_data->err_addr = kzalloc_objs(struct eeprom_table_record, adev->umc.max_ras_err_cnt_per_query); @@ -160,6 +161,7 @@ void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev, if (adev->umc.ras && adev->umc.ras->ecc_info_query_ras_error_address && adev->umc.max_ras_err_cnt_per_query) { + kfree(err_data->err_addr); err_data->err_addr = kzalloc_objs(struct eeprom_table_record, adev->umc.max_ras_err_cnt_per_query); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index 70d74f04d2dd..b634defb0be8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -82,19 +82,11 @@ static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev, return false; } -static void amdgpu_userq_gpu_reset(struct amdgpu_device *adev) -{ - if (amdgpu_device_should_recover_gpu(adev)) { - amdgpu_reset_domain_schedule(adev->reset_domain, - &adev->userq_reset_work); - /* Wait for the reset job to complete */ - flush_work(&adev->userq_reset_work); - } -} - -static int -amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr) +static void amdgpu_userq_mgr_reset_work(struct work_struct *work) { + struct amdgpu_userq_mgr *uq_mgr = + container_of(work, struct amdgpu_userq_mgr, + reset_work); struct amdgpu_device *adev = uq_mgr->adev; const int queue_types[] = { AMDGPU_RING_TYPE_COMPUTE, @@ -103,12 +95,11 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr) }; const int num_queue_types = ARRAY_SIZE(queue_types); bool gpu_reset = false; - int r = 0; - int i; + int i, r; if (unlikely(adev->debug_disable_gpu_ring_reset)) { dev_err(adev->dev, "userq reset disabled by debug mask\n"); - return 0; + return; } /* @@ -116,7 +107,7 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr) * skip all reset detection logic */ if (!amdgpu_gpu_recovery) - return 0; + return; /* * Iterate through all queue types to detect and reset problematic queues @@ -141,10 +132,19 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr) } } - if (gpu_reset) - amdgpu_userq_gpu_reset(adev); + if (gpu_reset) { + struct amdgpu_reset_context reset_context; - return r; + memset(&reset_context, 0, sizeof(reset_context)); + + reset_context.method = AMD_RESET_METHOD_NONE; + reset_context.reset_req_dev = adev; + reset_context.src = AMDGPU_RESET_SRC_USERQ; + set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); + /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/ + + amdgpu_device_gpu_recover(adev, NULL, &reset_context); + } } static void amdgpu_userq_hang_detect_work(struct work_struct *work) @@ -153,7 +153,11 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work) container_of(work, struct amdgpu_usermode_queue, hang_detect_work.work); - amdgpu_userq_detect_and_reset_queues(queue->userq_mgr); + /* + * Don't schedule the work here! Scheduling or queue work from one reset + * handler to another is illegal if you don't take extra precautions! + */ + amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work); } /* @@ -182,8 +186,8 @@ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue) break; } - schedule_delayed_work(&queue->hang_detect_work, - msecs_to_jiffies(timeout_ms)); + queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work, + msecs_to_jiffies(timeout_ms)); } void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell) @@ -223,7 +227,7 @@ static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue, INIT_LIST_HEAD(&va_cursor->list); va_cursor->gpu_addr = addr; - atomic_set(&va_map->bo_va->userq_va_mapped, 1); + va_map->bo_va->userq_va_mapped = true; list_add(&va_cursor->list, &queue->userq_va_list); return 0; @@ -270,7 +274,7 @@ static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr) dma_resv_assert_held(vm->root.bo->tbo.base.resv); mapping = amdgpu_vm_bo_lookup_mapping(vm, addr); - if (!IS_ERR_OR_NULL(mapping) && atomic_read(&mapping->bo_va->userq_va_mapped)) + if (!IS_ERR_OR_NULL(mapping) && mapping->bo_va->userq_va_mapped) r = true; else r = false; @@ -296,16 +300,8 @@ static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue) return false; } -static void amdgpu_userq_buffer_va_list_del(struct amdgpu_bo_va_mapping *mapping, - struct amdgpu_userq_va_cursor *va_cursor) -{ - atomic_set(&mapping->bo_va->userq_va_mapped, 0); - list_del(&va_cursor->list); - kfree(va_cursor); -} - -static int amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev, - struct amdgpu_usermode_queue *queue) +static void amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev, + struct amdgpu_usermode_queue *queue) { struct amdgpu_userq_va_cursor *va_cursor, *tmp; struct amdgpu_bo_va_mapping *mapping; @@ -315,15 +311,12 @@ static int amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev, list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) { mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, va_cursor->gpu_addr); - if (!mapping) { - return -EINVAL; - } - dev_dbg(adev->dev, "delete the userq:%p va:%llx\n", - queue, va_cursor->gpu_addr); - amdgpu_userq_buffer_va_list_del(mapping, va_cursor); + if (mapping) + dev_dbg(adev->dev, "delete the userq:%p va:%llx\n", + queue, va_cursor->gpu_addr); + list_del(&va_cursor->list); + kfree(va_cursor); } - - return 0; } static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue) @@ -504,16 +497,20 @@ int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr, goto free_obj; } + r = amdgpu_bo_pin(userq_obj->obj, AMDGPU_GEM_DOMAIN_GTT); + if (r) + goto unresv; + r = amdgpu_ttm_alloc_gart(&(userq_obj->obj)->tbo); if (r) { drm_file_err(uq_mgr->file, "Failed to alloc GART for userqueue object (%d)", r); - goto unresv; + goto unpin_bo; } r = amdgpu_bo_kmap(userq_obj->obj, &userq_obj->cpu_ptr); if (r) { drm_file_err(uq_mgr->file, "Failed to map BO for userqueue (%d)", r); - goto unresv; + goto unpin_bo; } userq_obj->gpu_addr = amdgpu_bo_gpu_offset(userq_obj->obj); @@ -521,11 +518,13 @@ int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr, memset(userq_obj->cpu_ptr, 0, size); return 0; +unpin_bo: + amdgpu_bo_unpin(userq_obj->obj); unresv: amdgpu_bo_unreserve(userq_obj->obj); - free_obj: amdgpu_bo_unref(&userq_obj->obj); + return r; } @@ -533,6 +532,7 @@ void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_userq_obj *userq_obj) { amdgpu_bo_kunmap(userq_obj->obj); + amdgpu_bo_unpin(userq_obj->obj); amdgpu_bo_unref(&userq_obj->obj); } @@ -708,14 +708,14 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) const struct amdgpu_userq_funcs *uq_funcs; struct amdgpu_usermode_queue *queue; struct amdgpu_db_info db_info; - bool skip_map_queue; - u32 qid; uint64_t index; - int r = 0; - int priority = - (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) >> - AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT; + int priority; + u32 qid; + int r; + priority = + (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) + >> AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT; r = amdgpu_userq_priority_permit(filp, priority); if (r) return r; @@ -728,40 +728,43 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) uq_funcs = adev->userq_funcs[args->in.ip_type]; if (!uq_funcs) { - drm_file_err(uq_mgr->file, "Usermode queue is not supported for this IP (%u)\n", - args->in.ip_type); r = -EINVAL; goto err_pm_runtime; } queue = kzalloc_obj(struct amdgpu_usermode_queue); if (!queue) { - drm_file_err(uq_mgr->file, "Failed to allocate memory for queue\n"); r = -ENOMEM; goto err_pm_runtime; } + kref_init(&queue->refcount); INIT_LIST_HEAD(&queue->userq_va_list); queue->doorbell_handle = args->in.doorbell_handle; queue->queue_type = args->in.ip_type; queue->vm = &fpriv->vm; queue->priority = priority; - - db_info.queue_type = queue->queue_type; - db_info.doorbell_handle = queue->doorbell_handle; - db_info.db_obj = &queue->db_obj; - db_info.doorbell_offset = args->in.doorbell_offset; - queue->userq_mgr = uq_mgr; + INIT_DELAYED_WORK(&queue->hang_detect_work, + amdgpu_userq_hang_detect_work); - /* Validate the userq virtual address.*/ - r = amdgpu_bo_reserve(fpriv->vm.root.bo, false); + mutex_init(&queue->fence_drv_lock); + xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC); + r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv); if (r) goto free_queue; - if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va, args->in.queue_size) || - amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va, AMDGPU_GPU_PAGE_SIZE) || - amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va, AMDGPU_GPU_PAGE_SIZE)) { + /* Make sure the queue can actually run with those virtual addresses. */ + r = amdgpu_bo_reserve(fpriv->vm.root.bo, false); + if (r) + goto free_fence_drv; + + if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va, + args->in.queue_size) || + amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va, + AMDGPU_GPU_PAGE_SIZE) || + amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va, + AMDGPU_GPU_PAGE_SIZE)) { r = -EINVAL; amdgpu_bo_unreserve(fpriv->vm.root.bo); goto clean_mapping; @@ -769,6 +772,10 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) amdgpu_bo_unreserve(fpriv->vm.root.bo); /* Convert relative doorbell offset into absolute doorbell index */ + db_info.queue_type = queue->queue_type; + db_info.doorbell_handle = queue->doorbell_handle; + db_info.db_obj = &queue->db_obj; + db_info.doorbell_offset = args->in.doorbell_offset; index = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp); if (index == (uint64_t)-EINVAL) { drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n"); @@ -777,82 +784,64 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) } queue->doorbell_index = index; - mutex_init(&queue->fence_drv_lock); - xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC); - r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv); - if (r) { - drm_file_err(uq_mgr->file, "Failed to alloc fence driver\n"); - goto clean_mapping; - } - r = uq_funcs->mqd_create(queue, &args->in); if (r) { drm_file_err(uq_mgr->file, "Failed to create Queue\n"); - goto clean_fence_driver; + goto clean_mapping; } + /* Update VM owner at userq submit-time for page-fault attribution. */ + amdgpu_vm_set_task_info(&fpriv->vm); + + r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue, + GFP_KERNEL)); + if (r) + goto clean_mqd; + amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); /* don't map the queue if scheduling is halted */ - if (adev->userq_halt_for_enforce_isolation && - ((queue->queue_type == AMDGPU_HW_IP_GFX) || - (queue->queue_type == AMDGPU_HW_IP_COMPUTE))) - skip_map_queue = true; - else - skip_map_queue = false; - if (!skip_map_queue) { + if (!adev->userq_halt_for_enforce_isolation || + ((queue->queue_type != AMDGPU_HW_IP_GFX) && + (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) { r = amdgpu_userq_map_helper(queue); if (r) { drm_file_err(uq_mgr->file, "Failed to map Queue\n"); - goto clean_mqd; + mutex_unlock(&uq_mgr->userq_mutex); + goto clean_doorbell; } } - /* drop this refcount during queue destroy */ - kref_init(&queue->refcount); - - /* Wait for mode-1 reset to complete */ - down_read(&adev->reset_domain->sem); - - r = xa_alloc(&uq_mgr->userq_xa, &qid, queue, - XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), GFP_KERNEL); - if (r) { - if (!skip_map_queue) - amdgpu_userq_unmap_helper(queue); - r = -ENOMEM; - goto clean_reset_domain; - } - - r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue, GFP_KERNEL)); - if (r) { - xa_erase(&uq_mgr->userq_xa, qid); - if (!skip_map_queue) - amdgpu_userq_unmap_helper(queue); - goto clean_reset_domain; - } - up_read(&adev->reset_domain->sem); - - amdgpu_debugfs_userq_init(filp, queue, qid); - INIT_DELAYED_WORK(&queue->hang_detect_work, - amdgpu_userq_hang_detect_work); - - args->out.queue_id = qid; atomic_inc(&uq_mgr->userq_count[queue->queue_type]); mutex_unlock(&uq_mgr->userq_mutex); + + r = xa_alloc(&uq_mgr->userq_xa, &qid, queue, + XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), + GFP_KERNEL); + if (r) { + /* + * This drops the last reference which should take care of + * all cleanup. + */ + amdgpu_userq_put(queue); + return r; + } + + amdgpu_debugfs_userq_init(filp, queue, qid); + args->out.queue_id = qid; return 0; -clean_reset_domain: - up_read(&adev->reset_domain->sem); +clean_doorbell: + xa_erase_irq(&adev->userq_doorbell_xa, index); clean_mqd: - mutex_unlock(&uq_mgr->userq_mutex); uq_funcs->mqd_destroy(queue); -clean_fence_driver: - amdgpu_userq_fence_driver_free(queue); clean_mapping: amdgpu_bo_reserve(fpriv->vm.root.bo, true); amdgpu_userq_buffer_vas_list_cleanup(adev, queue); amdgpu_bo_unreserve(fpriv->vm.root.bo); mutex_destroy(&queue->fence_drv_lock); +free_fence_drv: + amdgpu_userq_fence_driver_free(queue); free_queue: kfree(queue); err_pm_runtime: @@ -1039,12 +1028,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec, struct amdgpu_bo *bo; int ret; - spin_lock(&vm->status_lock); - while (!list_empty(&vm->invalidated)) { - bo_va = list_first_entry(&vm->invalidated, + spin_lock(&vm->individual_lock); + while (!list_empty(&vm->always_valid.evicted)) { + bo_va = list_first_entry(&vm->always_valid.evicted, struct amdgpu_bo_va, base.vm_status); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); bo = bo_va->base.bo; ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2); @@ -1056,14 +1045,14 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec, if (ret) return ret; - /* This moves the bo_va to the done list */ + /* This moves the bo_va to the idle list */ ret = amdgpu_vm_bo_update(adev, bo_va, false); if (ret) return ret; - spin_lock(&vm->status_lock); + spin_lock(&vm->individual_lock); } - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); return 0; } @@ -1095,7 +1084,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr) if (unlikely(ret)) goto unlock_all; - ret = amdgpu_vm_lock_done_list(vm, &exec, 1); + ret = amdgpu_vm_lock_individual(vm, &exec, 1); drm_exec_retry_on_contention(&exec); if (unlikely(ret)) goto unlock_all; @@ -1138,7 +1127,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr) key = 0; /* Validate User Ptr BOs */ - list_for_each_entry(bo_va, &vm->done, base.vm_status) { + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) { bo = bo_va->base.bo; if (!bo) continue; @@ -1188,10 +1177,10 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr) /* * We need to wait for all VM updates to finish before restarting the - * queues. Using the done list like that is now ok since everything is + * queues. Using the idle list like that is now ok since everything is * locked in place. */ - list_for_each_entry(bo_va, &vm->done, base.vm_status) + list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) dma_fence_wait(bo_va->last_pt_update, false); dma_fence_wait(vm->last_update, false); @@ -1252,28 +1241,13 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr) if (ret) { drm_file_err(uq_mgr->file, "Couldn't unmap all the queues, eviction failed ret=%d\n", ret); - amdgpu_userq_detect_and_reset_queues(uq_mgr); + amdgpu_reset_domain_schedule(uq_mgr->adev->reset_domain, + &uq_mgr->reset_work); + flush_work(&uq_mgr->reset_work); } return ret; } -void amdgpu_userq_reset_work(struct work_struct *work) -{ - struct amdgpu_device *adev = container_of(work, struct amdgpu_device, - userq_reset_work); - struct amdgpu_reset_context reset_context; - - memset(&reset_context, 0, sizeof(reset_context)); - - reset_context.method = AMD_RESET_METHOD_NONE; - reset_context.reset_req_dev = adev; - reset_context.src = AMDGPU_RESET_SRC_USERQ; - set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); - /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/ - - amdgpu_device_gpu_recover(adev, NULL, &reset_context); -} - static void amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr) { @@ -1307,9 +1281,24 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f userq_mgr->file = file_priv; INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker); + INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work); return 0; } +void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev) +{ + struct xarray *xa = &adev->userq_doorbell_xa; + struct amdgpu_usermode_queue *queue; + unsigned long flags, queue_id; + + xa_lock_irqsave(xa, flags); + xa_for_each(xa, queue_id, queue) { + cancel_delayed_work(&queue->hang_detect_work); + cancel_work(&queue->userq_mgr->reset_work); + } + xa_unlock_irqrestore(xa, flags); +} + void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr) { cancel_delayed_work_sync(&userq_mgr->resume_work); @@ -1335,6 +1324,14 @@ void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr) } xa_destroy(&userq_mgr->userq_xa); + + /* + * Drain any in-flight reset_work. By this point all queues are freed + * and userq_count is 0, so if reset_work starts now it exits early. + * We still need to wait in case it was already executing gpu_recover. + */ + cancel_work_sync(&userq_mgr->reset_work); + mutex_destroy(&userq_mgr->userq_mutex); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h index 85f460e7c31b..49b33e2d6932 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h @@ -84,7 +84,13 @@ struct amdgpu_usermode_queue { u32 xcp_id; int priority; struct dentry *debugfs_queue; - struct delayed_work hang_detect_work; + + /** + * @hang_detect_work: + * + * Delayed work which runs when userq_fences time out. + */ + struct delayed_work hang_detect_work; struct kref refcount; struct list_head userq_va_list; @@ -116,6 +122,13 @@ struct amdgpu_userq_mgr { struct amdgpu_device *adev; struct delayed_work resume_work; struct drm_file *file; + + /** + * @reset_work: + * + * Reset work which is used when eviction fails. + */ + struct work_struct reset_work; atomic_t userq_count[AMDGPU_RING_TYPE_MAX]; }; @@ -134,6 +147,7 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv, struct amdgpu_device *adev); +void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev); void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr); void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c index 53a8944bab05..a41fb72dba94 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c @@ -370,51 +370,48 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev, { struct amdgpu_bo_va_mapping *mapping; struct amdgpu_bo *bo; + struct drm_exec exec; u64 addr, *ptr; - int r; - - r = amdgpu_bo_reserve(queue->vm->root.bo, false); - if (r) - return r; + int ret; addr = queue->userq_prop->wptr_gpu_addr; addr &= AMDGPU_GMC_HOLE_MASK; - mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr >> PAGE_SHIFT); - if (!mapping) { - amdgpu_bo_unreserve(queue->vm->root.bo); - DRM_ERROR("Failed to lookup amdgpu_bo_va_mapping\n"); - return -EINVAL; + drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2); + drm_exec_until_all_locked(&exec) { + ret = amdgpu_vm_lock_pd(queue->vm, &exec, 1); + drm_exec_retry_on_contention(&exec); + if (unlikely(ret)) + goto lock_error; + + mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr >> PAGE_SHIFT); + if (!mapping) { + ret = -EINVAL; + goto lock_error; + } + + ret = drm_exec_lock_obj(&exec, &mapping->bo_va->base.bo->tbo.base); + drm_exec_retry_on_contention(&exec); + if (unlikely(ret)) + goto lock_error; } - bo = amdgpu_bo_ref(mapping->bo_va->base.bo); - amdgpu_bo_unreserve(queue->vm->root.bo); - r = amdgpu_bo_reserve(bo, true); - if (r) { - amdgpu_bo_unref(&bo); - DRM_ERROR("Failed to reserve userqueue wptr bo"); - return r; - } - - r = amdgpu_bo_kmap(bo, (void **)&ptr); - if (r) { + bo = mapping->bo_va->base.bo; + ret = amdgpu_bo_kmap(bo, (void **)&ptr); + if (ret) { DRM_ERROR("Failed mapping the userqueue wptr bo"); - goto map_error; + goto lock_error; } *wptr = le64_to_cpu(*ptr); amdgpu_bo_kunmap(bo); - amdgpu_bo_unreserve(bo); - amdgpu_bo_unref(&bo); - + drm_exec_fini(&exec); return 0; -map_error: - amdgpu_bo_unreserve(bo); - amdgpu_bo_unref(&bo); - - return r; +lock_error: + drm_exec_fini(&exec); + return ret; } static void diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index e8d180a412d1..3fcbd722ee3b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -463,6 +463,9 @@ static void amdgpu_virt_add_bad_page(struct amdgpu_device *adev, static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) { struct amd_sriov_msg_pf2vf_info_header *pf2vf_info = adev->virt.fw_reserve.p_pf2vf; + struct amdgim_pf2vf_info_v1 *pf2vf_v1; + struct amd_sriov_msg_pf2vf_info *pf2vf; + uint32_t checksum; uint32_t checkval; @@ -479,7 +482,8 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) switch (pf2vf_info->version) { case 1: - checksum = ((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->checksum; + pf2vf_v1 = (struct amdgim_pf2vf_info_v1 *)pf2vf_info; + checksum = pf2vf_v1->checksum; checkval = amd_sriov_msg_checksum( adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size, adev->virt.fw_reserve.checksum_key, checksum); @@ -490,12 +494,12 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) return -EINVAL; } - adev->virt.gim_feature = - ((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->feature_flags; + adev->virt.gim_feature = pf2vf_v1->feature_flags; break; case 2: /* TODO: missing key, need to add it later */ - checksum = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->checksum; + pf2vf = (struct amd_sriov_msg_pf2vf_info *)pf2vf_info; + checksum = pf2vf->checksum; checkval = amd_sriov_msg_checksum( adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size, 0, checksum); @@ -507,11 +511,9 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) } adev->virt.vf2pf_update_interval_ms = - ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->vf2pf_update_interval_ms; - adev->virt.gim_feature = - ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->feature_flags.all; - adev->virt.reg_access = - ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->reg_access_flags.all; + pf2vf->vf2pf_update_interval_ms; + adev->virt.gim_feature = pf2vf->feature_flags.all; + adev->virt.reg_access = pf2vf->reg_access_flags.all; adev->virt.decode_max_dimension_pixels = 0; adev->virt.decode_max_frame_pixels = 0; @@ -519,26 +521,30 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev) adev->virt.encode_max_frame_pixels = 0; adev->virt.is_mm_bw_enabled = false; for (i = 0; i < AMD_SRIOV_MSG_RESERVE_VCN_INST; i++) { - tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_dimension_pixels; + tmp = pf2vf->mm_bw_management[i].decode_max_dimension_pixels; adev->virt.decode_max_dimension_pixels = max(tmp, adev->virt.decode_max_dimension_pixels); - tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_frame_pixels; + tmp = pf2vf->mm_bw_management[i].decode_max_frame_pixels; adev->virt.decode_max_frame_pixels = max(tmp, adev->virt.decode_max_frame_pixels); - tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_dimension_pixels; + tmp = pf2vf->mm_bw_management[i].encode_max_dimension_pixels; adev->virt.encode_max_dimension_pixels = max(tmp, adev->virt.encode_max_dimension_pixels); - tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_frame_pixels; + tmp = pf2vf->mm_bw_management[i].encode_max_frame_pixels; adev->virt.encode_max_frame_pixels = max(tmp, adev->virt.encode_max_frame_pixels); } if ((adev->virt.decode_max_dimension_pixels > 0) || (adev->virt.encode_max_dimension_pixels > 0)) adev->virt.is_mm_bw_enabled = true; - adev->unique_id = - ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->uuid; - adev->virt.ras_en_caps.all = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->ras_en_caps.all; + adev->unique_id = pf2vf->uuid; + + adev->unitid = 0; + if (amdgpu_sriov_is_unitid_support(adev)) + adev->unitid = pf2vf->unitid; + + adev->virt.ras_en_caps.all = pf2vf->ras_en_caps.all; adev->virt.ras_telemetry_en_caps.all = - ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->ras_telemetry_en_caps.all; + pf2vf->ras_telemetry_en_caps.all; break; default: dev_err(adev->dev, "invalid pf2vf version: 0x%x\n", pf2vf_info->version); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 2f2ac3c4c2f7..a99d01f9dc75 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -138,6 +138,47 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm) dma_resv_assert_held(vm->root.bo->tbo.base.resv); } +/* Initialize the amdgpu_vm_bo_status object */ +static void amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists) +{ + INIT_LIST_HEAD(&lists->evicted); + INIT_LIST_HEAD(&lists->moved); + INIT_LIST_HEAD(&lists->idle); +} + +/* + * Make sure we have the lock to modify the vm_bo status and return the object + * with the status lists. + */ +static struct amdgpu_vm_bo_status * +amdgpu_vm_bo_lock_lists(struct amdgpu_vm_bo_base *vm_bo) +{ + struct amdgpu_vm *vm = vm_bo->vm; + struct amdgpu_bo *bo = vm_bo->bo; + + if (amdgpu_vm_is_bo_always_valid(vm, bo)) { + /* No extra locking needed, protected by the root PD resv lock */ + amdgpu_vm_assert_locked(vm); + + if (bo->tbo.type == ttm_bo_type_kernel) + return &vm->kernel; + + return &vm->always_valid; + } + + spin_lock(&vm_bo->vm->individual_lock); + return &vm->individual; +} + +/* Eventually unlock the status list lock again */ +static void amdgpu_vm_bo_unlock_lists(struct amdgpu_vm_bo_base *vm_bo) +{ + if (amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo)) + amdgpu_vm_assert_locked(vm_bo->vm); + else + spin_unlock(&vm_bo->vm->individual_lock); +} + /** * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid * @@ -157,37 +198,44 @@ bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo) * * @vm_bo: vm_bo which is evicted * - * State for PDs/PTs and per VM BOs which are not at the location they should - * be. + * State for vm_bo objects meaning the underlying BO was evicted and need to + * move in place again. */ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) { - struct amdgpu_vm *vm = vm_bo->vm; - struct amdgpu_bo *bo = vm_bo->bo; + struct amdgpu_vm_bo_status *lists; + lists = amdgpu_vm_bo_lock_lists(vm_bo); vm_bo->moved = true; - amdgpu_vm_assert_locked(vm); - spin_lock(&vm_bo->vm->status_lock); - if (bo->tbo.type == ttm_bo_type_kernel) - list_move(&vm_bo->vm_status, &vm->evicted); - else - list_move_tail(&vm_bo->vm_status, &vm->evicted); - spin_unlock(&vm_bo->vm->status_lock); + list_move(&vm_bo->vm_status, &lists->evicted); + amdgpu_vm_bo_unlock_lists(vm_bo); } /** * amdgpu_vm_bo_moved - vm_bo is moved * * @vm_bo: vm_bo which is moved * - * State for per VM BOs which are moved, but that change is not yet reflected - * in the page tables. + * State for vm_bo objects meaning the underlying BO was moved but the new + * location not yet reflected in the page tables. */ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) { - amdgpu_vm_assert_locked(vm_bo->vm); - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->moved); - spin_unlock(&vm_bo->vm->status_lock); + struct amdgpu_vm_bo_status *lists; + struct amdgpu_bo *bo = vm_bo->bo; + + /* + * The root PD doesn't have a parent PDE and goes directly into the + * idle state. + */ + lists = amdgpu_vm_bo_lock_lists(vm_bo); + if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) { + vm_bo->moved = false; + list_move(&vm_bo->vm_status, &lists->idle); + } else { + vm_bo->moved = true; + list_move(&vm_bo->vm_status, &lists->moved); + } + amdgpu_vm_bo_unlock_lists(vm_bo); } /** @@ -195,120 +243,43 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) * * @vm_bo: vm_bo which is now idle * - * State for PDs/PTs and per VM BOs which have gone through the state machine - * and are now idle. + * State for vm_bo objects meaning we are done with the state machine and no + * further action is necessary. */ static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) { - amdgpu_vm_assert_locked(vm_bo->vm); - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->idle); - spin_unlock(&vm_bo->vm->status_lock); - vm_bo->moved = false; -} + struct amdgpu_vm_bo_status *lists; -/** - * amdgpu_vm_bo_invalidated - vm_bo is invalidated - * - * @vm_bo: vm_bo which is now invalidated - * - * State for normal BOs which are invalidated and that change not yet reflected - * in the PTs. - */ -static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) -{ - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); - spin_unlock(&vm_bo->vm->status_lock); -} - -/** - * amdgpu_vm_bo_evicted_user - vm_bo is evicted - * - * @vm_bo: vm_bo which is evicted - * - * State for BOs used by user mode queues which are not at the location they - * should be. - */ -static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo) -{ - vm_bo->moved = true; - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user); - spin_unlock(&vm_bo->vm->status_lock); -} - -/** - * amdgpu_vm_bo_relocated - vm_bo is reloacted - * - * @vm_bo: vm_bo which is relocated - * - * State for PDs/PTs which needs to update their parent PD. - * For the root PD, just move to idle state. - */ -static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) -{ - amdgpu_vm_assert_locked(vm_bo->vm); - if (vm_bo->bo->parent) { - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); - spin_unlock(&vm_bo->vm->status_lock); - } else { - amdgpu_vm_bo_idle(vm_bo); - } -} - -/** - * amdgpu_vm_bo_done - vm_bo is done - * - * @vm_bo: vm_bo which is now done - * - * State for normal BOs which are invalidated and that change has been updated - * in the PTs. - */ -static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) -{ - amdgpu_vm_assert_locked(vm_bo->vm); - spin_lock(&vm_bo->vm->status_lock); - list_move(&vm_bo->vm_status, &vm_bo->vm->done); - spin_unlock(&vm_bo->vm->status_lock); + lists = amdgpu_vm_bo_lock_lists(vm_bo); + if (!amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo)) + vm_bo->moved = false; + list_move(&vm_bo->vm_status, &lists->idle); + amdgpu_vm_bo_unlock_lists(vm_bo); } /** * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine * @vm: the VM which state machine to reset * - * Move all vm_bo object in the VM into a state where they will be updated - * again during validation. + * Move all vm_bo object in the VM into a state where their location will be + * updated in the page tables again. */ static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm) { - struct amdgpu_vm_bo_base *vm_bo, *tmp; - amdgpu_vm_assert_locked(vm); + list_splice_init(&vm->kernel.idle, &vm->kernel.moved); + list_splice_init(&vm->always_valid.idle, &vm->always_valid.moved); - spin_lock(&vm->status_lock); - list_splice_init(&vm->done, &vm->invalidated); - list_for_each_entry(vm_bo, &vm->invalidated, vm_status) - vm_bo->moved = true; - - list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) { - struct amdgpu_bo *bo = vm_bo->bo; - - vm_bo->moved = true; - if (!bo || bo->tbo.type != ttm_bo_type_kernel) - list_move(&vm_bo->vm_status, &vm_bo->vm->moved); - else if (bo->parent) - list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); - } - spin_unlock(&vm->status_lock); + spin_lock(&vm->individual_lock); + list_splice_init(&vm->individual.idle, &vm->individual.moved); + spin_unlock(&vm->individual_lock); } /** * amdgpu_vm_update_shared - helper to update shared memory stat * @base: base structure for tracking BO usage in a VM * - * Takes the vm status_lock and updates the shared memory stat. If the basic + * Takes the vm stats_lock and updates the shared memory stat. If the basic * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called * as well. */ @@ -321,7 +292,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base) bool shared; dma_resv_assert_held(bo->tbo.base.resv); - spin_lock(&vm->status_lock); + spin_lock(&vm->stats_lock); shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base); if (base->shared != shared) { base->shared = shared; @@ -333,7 +304,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base) vm->stats[bo_memtype].drm.private += size; } } - spin_unlock(&vm->status_lock); + spin_unlock(&vm->stats_lock); } /** @@ -358,11 +329,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo) * be bo->tbo.resource * @sign: if we should add (+1) or subtract (-1) from the stat * - * Caller need to have the vm status_lock held. Useful for when multiple update + * Caller need to have the vm stats_lock held. Useful for when multiple update * need to happen at the same time. */ static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base, - struct ttm_resource *res, int sign) + struct ttm_resource *res, int sign) { struct amdgpu_vm *vm = base->vm; struct amdgpu_bo *bo = base->bo; @@ -386,7 +357,8 @@ static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base, */ if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) vm->stats[res_memtype].drm.purgeable += size; - if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype))) + if (!(bo->preferred_domains & + amdgpu_mem_type_to_domain(res_memtype))) vm->stats[bo_memtype].evicted += size; } } @@ -405,9 +377,9 @@ void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base, { struct amdgpu_vm *vm = base->vm; - spin_lock(&vm->status_lock); + spin_lock(&vm->stats_lock); amdgpu_vm_update_stats_locked(base, res, sign); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->stats_lock); } /** @@ -428,37 +400,34 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, base->next = NULL; INIT_LIST_HEAD(&base->vm_status); + dma_resv_assert_held(vm->root.bo->tbo.base.resv); if (!bo) return; + base->next = bo->vm_bo; bo->vm_bo = base; - spin_lock(&vm->status_lock); + spin_lock(&vm->stats_lock); base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base); amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->stats_lock); - if (!amdgpu_vm_is_bo_always_valid(vm, bo)) + if (!amdgpu_vm_is_bo_always_valid(vm, bo)) { + amdgpu_vm_bo_idle(base); return; - - dma_resv_assert_held(vm->root.bo->tbo.base.resv); + } ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move); - if (bo->tbo.type == ttm_bo_type_kernel && bo->parent) - amdgpu_vm_bo_relocated(base); - else - amdgpu_vm_bo_idle(base); - - if (bo->preferred_domains & - amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type)) - return; /* - * we checked all the prerequisites, but it looks like this per vm bo - * is currently evicted. add the bo to the evicted list to make sure it - * is validated on next vm use to avoid fault. - * */ - amdgpu_vm_bo_evicted(base); + * When a per VM isn't in the desired domain put it into the evicted + * state to make sure that it gets validated on the next best occasion. + */ + if (bo->preferred_domains & + amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type)) + amdgpu_vm_bo_moved(base); + else + amdgpu_vm_bo_evicted(base); } /** @@ -479,41 +448,41 @@ int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec, } /** - * amdgpu_vm_lock_done_list - lock all BOs on the done list + * amdgpu_vm_lock_individual - lock all BOs on the individual idle list * @vm: vm providing the BOs * @exec: drm execution context * @num_fences: number of extra fences to reserve * - * Lock the BOs on the done list in the DRM execution context. + * Lock the BOs on the individual idle list in the DRM execution context. */ -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec, - unsigned int num_fences) +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec, + unsigned int num_fences) { - struct list_head *prev = &vm->done; + struct list_head *prev = &vm->individual.idle; struct amdgpu_bo_va *bo_va; struct amdgpu_bo *bo; int ret; /* We can only trust prev->next while holding the lock */ - spin_lock(&vm->status_lock); - while (!list_is_head(prev->next, &vm->done)) { + spin_lock(&vm->individual_lock); + while (!list_is_head(prev->next, &vm->individual.idle)) { bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status); bo = bo_va->base.bo; if (bo) { amdgpu_bo_ref(bo); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1); amdgpu_bo_unref(&bo); if (unlikely(ret)) return ret; - spin_lock(&vm->status_lock); + spin_lock(&vm->individual_lock); } prev = prev->next; } - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); return 0; } @@ -609,10 +578,10 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, void *param) { uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm); - struct amdgpu_vm_bo_base *bo_base; - struct amdgpu_bo *bo; + struct amdgpu_vm_bo_base *bo_base, *tmp; int r; + dma_resv_assert_held(vm->root.bo->tbo.base.resv); if (vm->generation != new_vm_generation) { vm->generation = new_vm_generation; amdgpu_vm_bo_reset_state_machine(vm); @@ -622,50 +591,60 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, return r; } - spin_lock(&vm->status_lock); - while (!list_empty(&vm->evicted)) { - bo_base = list_first_entry(&vm->evicted, - struct amdgpu_vm_bo_base, - vm_status); - spin_unlock(&vm->status_lock); - - bo = bo_base->bo; - - r = validate(param, bo); + list_for_each_entry_safe(bo_base, tmp, &vm->kernel.evicted, vm_status) { + r = validate(param, bo_base->bo); if (r) return r; - if (bo->tbo.type != ttm_bo_type_kernel) { - amdgpu_vm_bo_moved(bo_base); - } else { - vm->update_funcs->map_table(to_amdgpu_bo_vm(bo)); - amdgpu_vm_bo_relocated(bo_base); - } - spin_lock(&vm->status_lock); + vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base->bo)); + amdgpu_vm_bo_moved(bo_base); } - while (ticket && !list_empty(&vm->evicted_user)) { - bo_base = list_first_entry(&vm->evicted_user, - struct amdgpu_vm_bo_base, - vm_status); - spin_unlock(&vm->status_lock); - - bo = bo_base->bo; - dma_resv_assert_held(bo->tbo.base.resv); - - r = validate(param, bo); - if (r) - return r; - - amdgpu_vm_bo_invalidated(bo_base); - - spin_lock(&vm->status_lock); - } - spin_unlock(&vm->status_lock); + /* + * As soon as all page tables are in place we can start updating them + * again. + */ amdgpu_vm_eviction_lock(vm); vm->evicting = false; amdgpu_vm_eviction_unlock(vm); + list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted, + vm_status) { + r = validate(param, bo_base->bo); + if (r) + return r; + + amdgpu_vm_bo_moved(bo_base); + } + + if (!ticket) + return 0; + + spin_lock(&vm->individual_lock); +restart: + list_for_each_entry(bo_base, &vm->individual.evicted, vm_status) { + struct amdgpu_bo *bo = bo_base->bo; + + if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket) + continue; + + spin_unlock(&vm->individual_lock); + + r = validate(param, bo); + if (r) + return r; + + amdgpu_vm_bo_moved(bo_base); + + /* It's a bit inefficient to always jump back to the start, but + * we would need to re-structure the KFD for properly fixing + * that. + */ + spin_lock(&vm->individual_lock); + goto restart; + } + spin_unlock(&vm->individual_lock); + return 0; } @@ -689,9 +668,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm) ret = !vm->evicting; amdgpu_vm_eviction_unlock(vm); - spin_lock(&vm->status_lock); - ret &= list_empty(&vm->evicted); - spin_unlock(&vm->status_lock); + ret &= list_empty(&vm->kernel.evicted); spin_lock(&vm->immediate.lock); ret &= !vm->immediate.stopped; @@ -985,18 +962,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, struct amdgpu_vm *vm, bool immediate) { struct amdgpu_vm_update_params params; - struct amdgpu_vm_bo_base *entry; + struct amdgpu_vm_bo_base *entry, *tmp; bool flush_tlb_needed = false; - LIST_HEAD(relocated); int r, idx; amdgpu_vm_assert_locked(vm); - spin_lock(&vm->status_lock); - list_splice_init(&vm->relocated, &relocated); - spin_unlock(&vm->status_lock); - - if (list_empty(&relocated)) + if (list_empty(&vm->kernel.moved)) return 0; if (!drm_dev_enter(adev_to_drm(adev), &idx)) @@ -1012,7 +984,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, if (r) goto error; - list_for_each_entry(entry, &relocated, vm_status) { + list_for_each_entry(entry, &vm->kernel.moved, vm_status) { /* vm_flush_needed after updating moved PDEs */ flush_tlb_needed |= entry->moved; @@ -1028,11 +1000,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, if (flush_tlb_needed) atomic64_inc(&vm->tlb_seq); - while (!list_empty(&relocated)) { - entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base, - vm_status); + list_for_each_entry_safe(entry, tmp, &vm->kernel.moved, vm_status) amdgpu_vm_bo_idle(entry); - } error: drm_dev_exit(idx); @@ -1260,9 +1229,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm, void amdgpu_vm_get_memory(struct amdgpu_vm *vm, struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]) { - spin_lock(&vm->status_lock); + spin_lock(&vm->stats_lock); memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->stats_lock); } /** @@ -1406,7 +1375,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, else amdgpu_vm_bo_idle(&bo_va->base); } else { - amdgpu_vm_bo_done(&bo_va->base); + amdgpu_vm_bo_idle(&bo_va->base); } list_splice_init(&bo_va->invalids, &bo_va->valids); @@ -1629,29 +1598,25 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket) { - struct amdgpu_bo_va *bo_va; + struct amdgpu_bo_va *bo_va, *tmp; struct dma_resv *resv; bool clear, unlock; int r; - spin_lock(&vm->status_lock); - while (!list_empty(&vm->moved)) { - bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va, - base.vm_status); - spin_unlock(&vm->status_lock); - + list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.moved, + base.vm_status) { /* Per VM BOs never need to bo cleared in the page tables */ r = amdgpu_vm_bo_update(adev, bo_va, false); if (r) return r; - spin_lock(&vm->status_lock); } - while (!list_empty(&vm->invalidated)) { - bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, - base.vm_status); + spin_lock(&vm->individual_lock); + while (!list_empty(&vm->individual.moved)) { + bo_va = list_first_entry(&vm->individual.moved, + typeof(*bo_va), base.vm_status); resv = bo_va->base.bo->tbo.base.resv; - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); /* Try to reserve the BO to avoid clearing its ptes */ if (!adev->debug_vm && dma_resv_trylock(resv)) { @@ -1681,11 +1646,11 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev, drm_gem_is_imported(&bo_va->base.bo->tbo.base) && (!bo_va->base.bo->tbo.resource || bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM)) - amdgpu_vm_bo_evicted_user(&bo_va->base); + amdgpu_vm_bo_evicted(&bo_va->base); - spin_lock(&vm->status_lock); + spin_lock(&vm->individual_lock); } - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); return 0; } @@ -2002,7 +1967,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, * during user requests GEM unmap IOCTL except for forcing the unmap * from user space. */ - if (unlikely(atomic_read(&bo_va->userq_va_mapped) > 0)) + if (unlikely(bo_va->userq_va_mapped)) amdgpu_userq_gem_va_unmap_validate(adev, mapping, saddr); list_del(&mapping->list); @@ -2223,9 +2188,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev, } } - spin_lock(&vm->status_lock); + spin_lock(&vm->individual_lock); list_del(&bo_va->base.vm_status); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->individual_lock); list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { list_del(&mapping->list); @@ -2305,14 +2270,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted) if (bo_base->moved) continue; - bo_base->moved = true; - - if (bo->tbo.type == ttm_bo_type_kernel) - amdgpu_vm_bo_relocated(bo_base); - else if (amdgpu_vm_is_bo_always_valid(vm, bo)) - amdgpu_vm_bo_moved(bo_base); - else - amdgpu_vm_bo_invalidated(bo_base); + amdgpu_vm_bo_moved(bo_base); } } @@ -2333,10 +2291,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem, for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { struct amdgpu_vm *vm = bo_base->vm; - spin_lock(&vm->status_lock); + spin_lock(&vm->stats_lock); amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1); amdgpu_vm_update_stats_locked(bo_base, new_mem, +1); - spin_unlock(&vm->status_lock); + spin_unlock(&vm->stats_lock); } amdgpu_vm_bo_invalidate(bo, evicted); @@ -2600,16 +2558,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, vm->va = RB_ROOT_CACHED; for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) vm->reserved_vmid[i] = NULL; - INIT_LIST_HEAD(&vm->evicted); - INIT_LIST_HEAD(&vm->evicted_user); - INIT_LIST_HEAD(&vm->relocated); - INIT_LIST_HEAD(&vm->moved); - INIT_LIST_HEAD(&vm->idle); - INIT_LIST_HEAD(&vm->invalidated); - spin_lock_init(&vm->status_lock); + + amdgpu_vm_bo_status_init(&vm->kernel); + amdgpu_vm_bo_status_init(&vm->always_valid); + spin_lock_init(&vm->individual_lock); + amdgpu_vm_bo_status_init(&vm->individual); INIT_LIST_HEAD(&vm->freed); - INIT_LIST_HEAD(&vm->done); INIT_KFIFO(vm->faults); + spin_lock_init(&vm->stats_lock); r = amdgpu_vm_init_entities(adev, vm); if (r) @@ -3086,6 +3042,42 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, } #if defined(CONFIG_DEBUG_FS) + +/* print the debug info for a specific set of status lists */ +static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m, + struct amdgpu_vm_bo_status *lists) +{ + struct amdgpu_vm_bo_base *base; + unsigned int id; + + id = 0; + seq_puts(m, "\tEvicted BOs:\n"); + list_for_each_entry(base, &lists->evicted, vm_status) { + if (!base->bo) + continue; + + amdgpu_bo_print_info(id++, base->bo, m); + } + + id = 0; + seq_puts(m, "\tMoved BOs:\n"); + list_for_each_entry(base, &lists->moved, vm_status) { + if (!base->bo) + continue; + + amdgpu_bo_print_info(id++, base->bo, m); + } + + id = 0; + seq_puts(m, "\tIdle BOs:\n"); + list_for_each_entry(base, &lists->moved, vm_status) { + if (!base->bo) + continue; + + amdgpu_bo_print_info(id++, base->bo, m); + } +} + /** * amdgpu_debugfs_vm_bo_info - print BO info for the VM * @@ -3096,90 +3088,18 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, */ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m) { - struct amdgpu_bo_va *bo_va, *tmp; - u64 total_idle = 0; - u64 total_evicted = 0; - u64 total_relocated = 0; - u64 total_moved = 0; - u64 total_invalidated = 0; - u64 total_done = 0; - unsigned int total_idle_objs = 0; - unsigned int total_evicted_objs = 0; - unsigned int total_relocated_objs = 0; - unsigned int total_moved_objs = 0; - unsigned int total_invalidated_objs = 0; - unsigned int total_done_objs = 0; - unsigned int id = 0; - amdgpu_vm_assert_locked(vm); - spin_lock(&vm->status_lock); - seq_puts(m, "\tIdle BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) { - if (!bo_va->base.bo) - continue; - total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m); - } - total_idle_objs = id; - id = 0; + seq_puts(m, "\tKernel PT/PDs:\n"); + amdgpu_debugfs_vm_bo_status_info(m, &vm->kernel); - seq_puts(m, "\tEvicted BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) { - if (!bo_va->base.bo) - continue; - total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m); - } - total_evicted_objs = id; - id = 0; + seq_puts(m, "\tPer VM BOs:\n"); + amdgpu_debugfs_vm_bo_status_info(m, &vm->always_valid); - seq_puts(m, "\tRelocated BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) { - if (!bo_va->base.bo) - continue; - total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m); - } - total_relocated_objs = id; - id = 0; - - seq_puts(m, "\tMoved BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { - if (!bo_va->base.bo) - continue; - total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m); - } - total_moved_objs = id; - id = 0; - - seq_puts(m, "\tInvalidated BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) { - if (!bo_va->base.bo) - continue; - total_invalidated += amdgpu_bo_print_info(id++, bo_va->base.bo, m); - } - total_invalidated_objs = id; - id = 0; - - seq_puts(m, "\tDone BOs:\n"); - list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) { - if (!bo_va->base.bo) - continue; - total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m); - } - spin_unlock(&vm->status_lock); - total_done_objs = id; - - seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle, - total_idle_objs); - seq_printf(m, "\tTotal evicted size: %12lld\tobjs:\t%d\n", total_evicted, - total_evicted_objs); - seq_printf(m, "\tTotal relocated size: %12lld\tobjs:\t%d\n", total_relocated, - total_relocated_objs); - seq_printf(m, "\tTotal moved size: %12lld\tobjs:\t%d\n", total_moved, - total_moved_objs); - seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated, - total_invalidated_objs); - seq_printf(m, "\tTotal done size: %12lld\tobjs:\t%d\n", total_done, - total_done_objs); + seq_puts(m, "\tIndividual BOs:\n"); + spin_lock(&vm->individual_lock); + amdgpu_debugfs_vm_bo_status_info(m, &vm->individual); + spin_unlock(&vm->individual_lock); } #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index f4b0e9da3fa2..3695299f1a03 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -205,17 +205,34 @@ struct amdgpu_vm_bo_base { /* protected by bo being reserved */ struct amdgpu_vm_bo_base *next; - /* protected by vm status_lock */ + /* protected by vm reservation and invalidated_lock */ struct list_head vm_status; /* if the bo is counted as shared in mem stats - * protected by vm status_lock */ + * protected by vm BO being reserved */ bool shared; /* protected by the BO being reserved */ bool moved; }; +/* + * The following status lists contain amdgpu_vm_bo_base objects for + * either PD/PTs, per VM BOs or BOs with individual resv object. + * + * The state transits are: evicted -> moved -> idle + */ +struct amdgpu_vm_bo_status { + /* BOs evicted which need to move into place again */ + struct list_head evicted; + + /* BOs which moved but new location hasn't been updated in the PDs/PTs */ + struct list_head moved; + + /* BOs done with the state machine and need no further action */ + struct list_head idle; +}; + /* provided by hw blocks that can write ptes, e.g., sdma */ struct amdgpu_vm_pte_funcs { /* number of dw to reserve per operation */ @@ -345,47 +362,29 @@ struct amdgpu_vm { bool evicting; unsigned int saved_flags; - /* Lock to protect vm_bo add/del/move on all lists of vm */ - spinlock_t status_lock; - - /* Memory statistics for this vm, protected by status_lock */ + /* Memory statistics for this vm, protected by stats_lock */ + spinlock_t stats_lock; struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]; + /* BO's belonging to PD/PT which are internal to the kernel. */ + struct amdgpu_vm_bo_status kernel; + /* - * The following lists contain amdgpu_vm_bo_base objects for either - * PDs, PTs or per VM BOs. The state transits are: - * - * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle + * BOs allocated by userspace where the dma_resv is shared with the + * root PD */ - - /* Per-VM and PT BOs who needs a validation */ - struct list_head evicted; - - /* PT BOs which relocated and their parent need an update */ - struct list_head relocated; - - /* per VM BOs moved, but not yet updated in the PT */ - struct list_head moved; - - /* All BOs of this VM not currently in the state machine */ - struct list_head idle; + struct amdgpu_vm_bo_status always_valid; /* * The following lists contain amdgpu_vm_bo_base objects for BOs which - * have their own dma_resv object and not depend on the root PD. Their - * state transits are: + * have their own dma_resv object and not depend on the root PD. * - * evicted_user or invalidated -> done + * Lists are protected by the individual_lock. */ + spinlock_t individual_lock; - /* BOs for user mode queues that need a validation */ - struct list_head evicted_user; - - /* regular invalidated BOs, but not yet updated in the PT */ - struct list_head invalidated; - - /* BOs which are invalidated, has been updated in the PTs */ - struct list_head done; + /* Userspace BOs with individual resv object */ + struct amdgpu_vm_bo_status individual; /* * This list contains amdgpu_bo_va_mapping objects which have been freed @@ -507,8 +506,8 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm); void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm); int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec, unsigned int num_fences); -int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec, - unsigned int num_fences); +int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec, + unsigned int num_fences); bool amdgpu_vm_ready(struct amdgpu_vm *vm); uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm); int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index b2494381b98a..e43a60d09808 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c @@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry) entry->bo->vm_bo = NULL; ttm_bo_set_bulk_move(&entry->bo->tbo, NULL); - spin_lock(&entry->vm->status_lock); list_del(&entry->vm_status); - spin_unlock(&entry->vm->status_lock); amdgpu_bo_unref(&entry->bo); } @@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params, struct amdgpu_vm_pt_cursor seek; struct amdgpu_vm_bo_base *entry; - spin_lock(¶ms->vm->status_lock); for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) { if (entry && entry->bo) list_move(&entry->vm_status, ¶ms->tlb_flush_waitlist); @@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params, /* enter start node now */ list_move(&cursor->entry->vm_status, ¶ms->tlb_flush_waitlist); - spin_unlock(¶ms->vm->status_lock); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c index 0a34a27d1106..601420491477 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c @@ -566,6 +566,11 @@ static void vpe_ring_emit_fence(struct amdgpu_ring *ring, uint64_t addr, amdgpu_ring_write(ring, 0); } + /* WA: Force sync after TRAP to avoid VPE1 fail to power off */ + if (ring->adev->vpe.collaborate_mode) { + amdgpu_ring_write(ring, VPE_CMD_HEADER(VPE_CMD_OPCODE_COLLAB_SYNC, 0)); + amdgpu_ring_write(ring, 0xabcd); + } } static void vpe_ring_emit_pipeline_sync(struct amdgpu_ring *ring) @@ -972,7 +977,7 @@ static const struct amdgpu_ring_funcs vpe_ring_funcs = { .emit_frame_size = 5 + /* vpe_ring_init_cond_exec */ 6 + /* vpe_ring_emit_pipeline_sync */ - 10 + 10 + 10 + /* vpe_ring_emit_fence */ + 12 + 12 + 12 + /* vpe_ring_emit_fence */ /* vpe_ring_emit_vm_flush */ SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h index 9dcf0b07d513..d80f01c0e754 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h @@ -295,7 +295,7 @@ struct amd_sriov_msg_pf2vf_info { uint32_t vf2pf_update_interval_ms; /* identification in ROCm SMI */ uint64_t uuid; - uint32_t fcn_idx; + uint32_t pad; /* flags to indicate which register access method VF should use */ union amd_sriov_reg_access_flags reg_access_flags; /* MM BW management */ diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c index 6e37961f6be5..ca5d091549e1 100644 --- a/drivers/gpu/drm/amd/amdgpu/atom.c +++ b/drivers/gpu/drm/amd/amdgpu/atom.c @@ -59,6 +59,9 @@ #define ATOM_CMD_TIMEOUT_SEC 20 +/* Limit ATOM command table recursion (calltable) to avoid kernel stack overflow. */ +#define ATOM_EXECUTE_MAX_DEPTH 32 + typedef struct { struct atom_context *ctx; uint32_t *ps, *ws; @@ -1229,6 +1232,13 @@ static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index, if (!base) return -EINVAL; + if (ctx->execute_depth >= ATOM_EXECUTE_MAX_DEPTH) { + DRM_ERROR("atombios command table nesting exceeded limit (%u)\n", + ATOM_EXECUTE_MAX_DEPTH); + return -ELOOP; + } + ctx->execute_depth++; + len = CU16(base + ATOM_CT_SIZE_PTR); ws = CU8(base + ATOM_CT_WS_PTR); ps = CU8(base + ATOM_CT_PS_PTR) & ATOM_CT_PS_MASK; @@ -1285,6 +1295,7 @@ static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index, free: if (ws) kfree(ectx.ws); + ctx->execute_depth--; return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/atom.h b/drivers/gpu/drm/amd/amdgpu/atom.h index 825ff28731f5..bb3d9eb7eb6b 100644 --- a/drivers/gpu/drm/amd/amdgpu/atom.h +++ b/drivers/gpu/drm/amd/amdgpu/atom.h @@ -153,6 +153,9 @@ struct atom_context { uint8_t vbios_ver_str[STRLEN_NORMAL]; uint8_t date[STRLEN_NORMAL]; uint8_t build_num[STRLEN_NORMAL]; + + /* Nesting depth for ATOM_OP_CALLTABLE */ + unsigned int execute_depth; }; extern int amdgpu_atom_debug; diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c index 4217b3fea0f3..b169e577e583 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c @@ -480,6 +480,7 @@ static int mes_v12_1_suspend_gang(struct amdgpu_mes *mes, mes_suspend_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS; mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs; + mes_suspend_gang_pkt.suspend_all_sdma_gangs = input->suspend_all_sdma_gangs; mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr; mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr; mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index fb7aaf5ae05c..479690c44f0d 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -217,7 +217,9 @@ static int psp_v11_0_bootloader_load_component(struct psp_context *psp, return ret; /* Copy PSP System Driver binary to memory */ - psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -263,7 +265,9 @@ static int psp_v11_0_bootloader_load_sos(struct psp_context *psp) return ret; /* Copy Secure OS binary to PSP memory */ - psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c index c3cae29eeca1..f823f042788d 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c @@ -87,7 +87,9 @@ static int psp_v12_0_bootloader_load_sysdrv(struct psp_context *psp) return ret; /* Copy PSP System Driver binary to memory */ - psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + ret = psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + if (ret) + return ret; /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -123,7 +125,9 @@ static int psp_v12_0_bootloader_load_sos(struct psp_context *psp) return ret; /* Copy Secure OS binary to PSP memory */ - psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c index a0c84f81c0c9..00b4a34e6601 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c @@ -271,10 +271,9 @@ static int psp_v13_0_bootloader_load_component(struct psp_context *psp, if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy PSP KDB binary to memory */ - memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the PSP KDB to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, @@ -353,10 +352,9 @@ static int psp_v13_0_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c index 5f39a2edcc95..3d5e26b3fa00 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c @@ -105,10 +105,9 @@ static int psp_v13_0_4_bootloader_load_component(struct psp_context *psp, if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy PSP KDB binary to memory */ - memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the PSP KDB to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, @@ -168,10 +167,9 @@ static int psp_v13_0_4_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c index 38dfc5c19f2a..040a61aefa86 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c @@ -140,10 +140,9 @@ static int psp_v14_0_bootloader_load_component(struct psp_context *psp, if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy PSP KDB binary to memory */ - memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the PSP KDB to bootloader */ WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_36, @@ -214,10 +213,9 @@ static int psp_v14_0_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c index f5030efc6c80..24856c91c135 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c @@ -96,7 +96,9 @@ static int psp_v3_1_bootloader_load_sysdrv(struct psp_context *psp) return ret; /* Copy PSP System Driver binary to memory */ - psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + ret = psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + if (ret) + return ret; /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -135,7 +137,9 @@ static int psp_v3_1_bootloader_load_sos(struct psp_context *psp) return ret; /* Copy Secure OS binary to PSP memory */ - psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c index 5b7b46d242c6..93253db5e2de 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c @@ -42,9 +42,10 @@ #include "oss/oss_1_0_d.h" #include "oss/oss_1_0_sh_mask.h" +#define VCE_V1_0_ALIGNMENT (32 * 1024) #define VCE_V1_0_FW_SIZE (256 * 1024) #define VCE_V1_0_STACK_SIZE (64 * 1024) -#define VCE_V1_0_DATA_SIZE (7808 * (AMDGPU_MAX_VCE_HANDLES + 1)) +#define VCE_V1_0_DATA_SIZE (ALIGN(7808 * (AMDGPU_MAX_VCE_HANDLES + 1), VCE_V1_0_ALIGNMENT)) #define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK 0x02 static void vce_v1_0_set_ring_funcs(struct amdgpu_device *adev); @@ -177,7 +178,7 @@ static void vce_v1_0_init_cg(struct amdgpu_device *adev) } /** - * vce_v1_0_load_fw_signature - load firmware signature into VCPU BO + * vce_v1_0_load_fw() - load firmware signature into VCPU BO * * @adev: amdgpu_device pointer * @@ -185,21 +186,26 @@ static void vce_v1_0_init_cg(struct amdgpu_device *adev) * This function finds the signature appropriate for the current * ASIC and writes that into the VCPU BO. */ -static int vce_v1_0_load_fw_signature(struct amdgpu_device *adev) +static int vce_v1_0_load_fw(struct amdgpu_device *adev) { const struct common_firmware_header *hdr; struct vce_v1_0_fw_signature *sign; - unsigned int ucode_offset; + u32 ucode_offset; + u32 ucode_size; uint32_t chip_id; u32 *cpu_addr; int i; hdr = (const struct common_firmware_header *)adev->vce.fw->data; ucode_offset = le32_to_cpu(hdr->ucode_array_offset_bytes); + ucode_size = hdr->ucode_size_bytes - sizeof(struct vce_v1_0_fw_signature *); cpu_addr = adev->vce.cpu_addr; sign = (void *)adev->vce.fw->data + ucode_offset; + if (ucode_size > VCE_V1_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET) + return -EINVAL; + switch (adev->asic_type) { case CHIP_TAHITI: chip_id = 0x01000014; @@ -226,12 +232,14 @@ static int vce_v1_0_load_fw_signature(struct amdgpu_device *adev) return -EINVAL; } + memset_io(&cpu_addr[0], 0, amdgpu_bo_size(adev->vce.vcpu_bo)); + cpu_addr += (256 - 64) / 4; memcpy_toio(&cpu_addr[0], &sign->val[i].nonce[0], 16); cpu_addr[4] = cpu_to_le32(le32_to_cpu(sign->length) + 64); memset_io(&cpu_addr[5], 0, 44); - memcpy_toio(&cpu_addr[16], &sign[1], hdr->ucode_size_bytes - sizeof(*sign)); + memcpy_toio(&cpu_addr[16], &sign[1], ucode_size); cpu_addr += (le32_to_cpu(sign->length) + 64) / 4; memcpy_toio(&cpu_addr[0], &sign->val[i].sigval[0], 16); @@ -312,18 +320,23 @@ static int vce_v1_0_mc_resume(struct amdgpu_device *adev) WREG32(mmVCE_VCPU_SCRATCH7, AMDGPU_MAX_VCE_HANDLES); offset = adev->vce.gpu_addr + AMDGPU_VCE_FIRMWARE_OFFSET; - size = VCE_V1_0_FW_SIZE; - WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset & 0x7fffffff); + size = VCE_V1_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET; + WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset); WREG32(mmVCE_VCPU_CACHE_SIZE0, size); offset += size; size = VCE_V1_0_STACK_SIZE; - WREG32(mmVCE_VCPU_CACHE_OFFSET1, offset & 0x7fffffff); + WARN_ON(!IS_ALIGNED(offset, VCE_V1_0_ALIGNMENT)); + WARN_ON(!IS_ALIGNED(size, VCE_V1_0_ALIGNMENT)); + WREG32(mmVCE_VCPU_CACHE_OFFSET1, offset); WREG32(mmVCE_VCPU_CACHE_SIZE1, size); offset += size; size = VCE_V1_0_DATA_SIZE; - WREG32(mmVCE_VCPU_CACHE_OFFSET2, offset & 0x7fffffff); + WARN_ON(!IS_ALIGNED(offset, VCE_V1_0_ALIGNMENT)); + WARN_ON(!IS_ALIGNED(size, VCE_V1_0_ALIGNMENT)); + WARN_ON((offset + size - adev->vce.gpu_addr) > amdgpu_bo_size(adev->vce.vcpu_bo)); + WREG32(mmVCE_VCPU_CACHE_OFFSET2, offset); WREG32(mmVCE_VCPU_CACHE_SIZE2, size); WREG32_P(mmVCE_LMI_CTRL2, 0x0, ~0x100); @@ -527,22 +540,31 @@ static int vce_v1_0_early_init(struct amdgpu_ip_block *ip_block) * To accomodate that, we put GART to the LOW address range * and reserve some GART pages where we map the VCPU BO, * so that it gets a 32-bit address. + * + * The BAR address is zero and we can't change it + * due to the firmware validation mechanism. + * It seems that it fails to initialize if the address is >= 128 MiB. */ static int vce_v1_0_ensure_vcpu_bo_32bit_addr(struct amdgpu_device *adev) { u64 bo_size = amdgpu_bo_size(adev->vce.vcpu_bo); - u64 max_vcpu_bo_addr = 0xffffffff - bo_size; + u64 max_vcpu_bo_addr = 0x07ffffff - bo_size; u64 num_pages = ALIGN(bo_size, AMDGPU_GPU_PAGE_SIZE) / AMDGPU_GPU_PAGE_SIZE; u64 pa = amdgpu_gmc_vram_pa(adev, adev->vce.vcpu_bo); u64 flags = AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE | AMDGPU_PTE_VALID; u64 vce_gart_start_offs; int r; - r = amdgpu_gtt_mgr_alloc_entries(&adev->mman.gtt_mgr, - &adev->vce.gart_node, num_pages, - DRM_MM_INSERT_LOW); - if (r) - return r; + if (adev->gmc.vram_start < adev->gmc.gart_start) + return amdgpu_bo_gpu_offset(adev->vce.vcpu_bo) <= max_vcpu_bo_addr ? 0 : -EINVAL; + + if (!drm_mm_node_allocated(&adev->vce.gart_node)) { + r = amdgpu_gtt_mgr_alloc_entries(&adev->mman.gtt_mgr, + &adev->vce.gart_node, num_pages, + DRM_MM_INSERT_LOW); + if (r) + return r; + } vce_gart_start_offs = amdgpu_gtt_node_to_byte_offset(&adev->vce.gart_node); @@ -553,8 +575,6 @@ static int vce_v1_0_ensure_vcpu_bo_32bit_addr(struct amdgpu_device *adev) amdgpu_gart_map_vram_range(adev, pa, adev->vce.gart_node.start, num_pages, flags, adev->gart.ptr); adev->vce.gpu_addr = adev->gmc.gart_start + vce_gart_start_offs; - if (adev->vce.gpu_addr > max_vcpu_bo_addr) - return -EINVAL; return 0; } @@ -574,10 +594,7 @@ static int vce_v1_0_sw_init(struct amdgpu_ip_block *ip_block) if (r) return r; - r = amdgpu_vce_resume(adev); - if (r) - return r; - r = vce_v1_0_load_fw_signature(adev); + r = vce_v1_0_load_fw(adev); if (r) return r; r = vce_v1_0_ensure_vcpu_bo_32bit_addr(adev); @@ -696,10 +713,7 @@ static int vce_v1_0_resume(struct amdgpu_ip_block *ip_block) struct amdgpu_device *adev = ip_block->adev; int r; - r = amdgpu_vce_resume(adev); - if (r) - return r; - r = vce_v1_0_load_fw_signature(adev); + r = vce_v1_0_load_fw(adev); if (r) return r; r = vce_v1_0_ensure_vcpu_bo_32bit_addr(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c index db149eda6204..3a6fc8604108 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c @@ -37,9 +37,14 @@ #include "oss/oss_2_0_d.h" #include "oss/oss_2_0_sh_mask.h" + +/* Use 24K to be safe. The FW supposedly only requires 23744 bytes. */ +#define VCE_V2_0_DATA_ENTRY_SIZE (24 * 1024) + #define VCE_V2_0_FW_SIZE (256 * 1024) #define VCE_V2_0_STACK_SIZE (64 * 1024) -#define VCE_V2_0_DATA_SIZE (23552 * AMDGPU_MAX_VCE_HANDLES) +#define VCE_V2_0_DATA_SIZE (VCE_V2_0_DATA_ENTRY_SIZE * (AMDGPU_MAX_VCE_HANDLES + 1)) + #define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK 0x02 static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev); @@ -183,7 +188,7 @@ static void vce_v2_0_mc_resume(struct amdgpu_device *adev) WREG32(mmVCE_LMI_VCPU_CACHE_40BIT_BAR, (adev->vce.gpu_addr >> 8)); offset = AMDGPU_VCE_FIRMWARE_OFFSET; - size = VCE_V2_0_FW_SIZE; + size = VCE_V2_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET; WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset & 0x7fffffff); WREG32(mmVCE_VCPU_CACHE_SIZE0, size); diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c index 03d79e464f04..c69f7d82060f 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c @@ -574,7 +574,7 @@ static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx) } else WREG32(mmVCE_LMI_VCPU_CACHE_40BIT_BAR, (adev->vce.gpu_addr >> 8)); offset = AMDGPU_VCE_FIRMWARE_OFFSET; - size = VCE_V3_0_FW_SIZE; + size = VCE_V3_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET; WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset & 0x7fffffff); WREG32(mmVCE_VCPU_CACHE_SIZE0, size); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 04ae3cb3a65c..81bcb16eb6dd 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -69,6 +69,21 @@ static const struct class kfd_class = { .name = kfd_dev_name, }; +/* + * Cache the address space of the chardev on first open so that the reset + * path can drop all userspace mappings of doorbell and MMIO ranges via + * unmap_mapping_range(). + */ +static struct address_space *kfd_dev_mapping; + +void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen) +{ + struct address_space *mapping = READ_ONCE(kfd_dev_mapping); + + if (mapping) + unmap_mapping_range(mapping, holebegin, holelen, 1); +} + static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id) { struct kfd_process_device *pdd; @@ -135,6 +150,13 @@ static int kfd_open(struct inode *inode, struct file *filep) if (iminor(inode) != 0) return -ENODEV; + /* + * /dev/kfd is a single chardev so all opens share one inode. Cache + * its address_space on the first open for use by the reset path. + */ + if (!READ_ONCE(kfd_dev_mapping)) + cmpxchg(&kfd_dev_mapping, NULL, inode->i_mapping); + is_32bit_user_mode = in_compat_syscall(); if (is_32bit_user_mode) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 5043b74b56ae..5cba592ba941 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -612,6 +612,9 @@ static int allocate_doorbell(struct qcm_process_device *qpd, } else { /* For CP queues on SOC15 */ if (restore_id) { + if (*restore_id >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) + return -EINVAL; + /* make sure that ID is free */ if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap)) return -EINVAL; @@ -1724,6 +1727,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, } if (restore_sdma_id) { + if (*restore_sdma_id >= get_num_sdma_queues(dqm)) + return -EINVAL; + /* Re-use existing sdma_id */ if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) { dev_err(dev, "SDMA queue already in use\n"); @@ -1750,6 +1756,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, return -ENOMEM; } if (restore_sdma_id) { + if (*restore_sdma_id >= get_num_xgmi_sdma_queues(dqm)) + return -EINVAL; + /* Re-use existing sdma_id */ if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) { dev_err(dev, "SDMA queue already in use\n"); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c index 56a7679ca98d..17bfb419b202 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c @@ -375,11 +375,15 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd, { struct v9_mqd *m; struct kfd_context_save_area_header header; + u32 cntl_stack_size; + u32 cntl_stack_offset; /* Control stack is located one page after MQD. */ void *mqd_ctl_stack = (void *)((uintptr_t)mqd + AMDGPU_GPU_PAGE_SIZE); m = get_mqd(mqd); + cntl_stack_size = min_t(u32, m->cp_hqd_cntl_stack_size, q->ctl_stack_size); + cntl_stack_offset = min_t(u32, m->cp_hqd_cntl_stack_offset, cntl_stack_size); *ctl_stack_used_size = m->cp_hqd_cntl_stack_size - m->cp_hqd_cntl_stack_offset; @@ -395,9 +399,10 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd, if (copy_to_user(ctl_stack, &header, sizeof(header.wave_state))) return -EFAULT; - if (copy_to_user(ctl_stack + m->cp_hqd_cntl_stack_offset, - mqd_ctl_stack + m->cp_hqd_cntl_stack_offset, - *ctl_stack_used_size)) + *ctl_stack_used_size = cntl_stack_size - cntl_stack_offset; + + if (copy_to_user(ctl_stack + cntl_stack_offset, mqd_ctl_stack + cntl_stack_offset, + *ctl_stack_used_size)) return -EFAULT; return 0; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 482bcfa10f82..acd0e41e744c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -399,6 +399,7 @@ enum kfd_mempool { /* Character device interface */ int kfd_chardev_init(void); void kfd_chardev_exit(void); +void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen); /** * enum kfd_unmap_queues_filter - Enum for queue filters. diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index 46db0d65d242..630f46091dc7 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -514,7 +514,8 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr, dev->node_props.capability |= HSA_CAP_AQL_QUEUE_DOUBLE_MAP; - if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0) && + if ((KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0) || + KFD_GC_VERSION(dev->gpu) == IP_VERSION(12, 1, 0)) && (dev->gpu->adev->sdma.supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE)) dev->node_props.capability2 |= HSA_CAP2_PER_SDMA_QUEUE_RESET_SUPPORTED; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 3ae2f330c6b4..ba7f98a87808 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -2330,9 +2330,16 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev) adev->dm.idle_workqueue = NULL; } - /* Disable ISM before dc_destroy() invalidates dm->dc */ + /* + * Disable ISM before dc_destroy() invalidates dm->dc. + * + * Quiesce workers first without dc_lock (they take dc_lock + * themselves, so syncing under it would deadlock), then drive the + * FSM back to FULL_POWER_RUNNING under dc_lock. + */ + amdgpu_dm_ism_disable(&adev->dm); scoped_guard(mutex, &adev->dm.dc_lock) - amdgpu_dm_ism_disable(&adev->dm); + amdgpu_dm_ism_force_full_power(&adev->dm); amdgpu_dm_destroy_drm_device(&adev->dm); @@ -3364,9 +3371,14 @@ static int dm_suspend(struct amdgpu_ip_block *ip_block) if (amdgpu_in_reset(adev)) { enum dc_status res; + /* Quiesce ISM workers before taking dc_lock (workers take + * dc_lock themselves; syncing under it would deadlock). + */ + amdgpu_dm_ism_disable(dm); + mutex_lock(&dm->dc_lock); - amdgpu_dm_ism_disable(dm); + amdgpu_dm_ism_force_full_power(dm); dc_allow_idle_optimizations(adev->dm.dc, false); dm->cached_dc_state = dc_state_create_copy(dm->dc->current_state); @@ -3400,8 +3412,13 @@ static int dm_suspend(struct amdgpu_ip_block *ip_block) amdgpu_dm_irq_suspend(adev); + /* + * Quiesce ISM workers before taking dc_lock (workers take dc_lock + * themselves; syncing under it would deadlock). + */ + amdgpu_dm_ism_disable(dm); scoped_guard(mutex, &dm->dc_lock) - amdgpu_dm_ism_disable(dm); + amdgpu_dm_ism_force_full_power(dm); hpd_rx_irq_work_suspend(dm); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c index d03ea3bafd46..7dcc587c45e9 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c @@ -451,8 +451,8 @@ void amdgpu_dm_ism_commit_event(struct amdgpu_dm_ism *ism, struct amdgpu_display_manager *dm = &adev->dm; struct dm_crtc_state *acrtc_state = to_dm_crtc_state(acrtc->base.state); - /* ISM transitions must be called with mutex acquired */ - ASSERT(mutex_is_locked(&dm->dc_lock)); + /* ISM transitions must be called with dc_lock held */ + lockdep_assert_held(&dm->dc_lock); /* ISM should not run after dc is destroyed */ ASSERT(dm->dc); @@ -516,13 +516,20 @@ static void dm_ism_sso_delayed_work_func(struct work_struct *work) } /** - * amdgpu_dm_ism_disable - Disable the ISM + * amdgpu_dm_ism_disable - Quiesce ISM workers * * @dm: The amdgpu display manager * - * Disable the idle state manager by disabling any ISM work, canceling pending - * work, and waiting for in-progress work to finish. After disabling, the system - * is left in DM_ISM_STATE_FULL_POWER_RUNNING state. + * Cancels and disables any pending or in-flight ISM delayed work and waits + * for in-progress work to finish. After this returns, no ISM worker can run + * and subsequent mod_delayed_work() calls become no-ops via + * clear_pending_if_disabled(). + * + * Must NOT be called with dc_lock held: the workers themselves take dc_lock, + * so a synchronous wait under dc_lock would deadlock. + * + * The caller is responsible for driving the FSM back to FULL_POWER_RUNNING + * (under dc_lock) by calling amdgpu_dm_ism_force_full_power(). */ void amdgpu_dm_ism_disable(struct amdgpu_display_manager *dm) { @@ -530,21 +537,54 @@ void amdgpu_dm_ism_disable(struct amdgpu_display_manager *dm) struct amdgpu_crtc *acrtc; struct amdgpu_dm_ism *ism; - ASSERT(mutex_is_locked(&dm->dc_lock)); + /* + * Caller must NOT hold dc_lock: the ISM delayed work handlers + * acquire dc_lock themselves, so waiting for them via + * disable_delayed_work_sync() while holding dc_lock would + * self-deadlock against an in-flight worker. + */ + lockdep_assert_not_held(&dm->dc_lock); drm_for_each_crtc(crtc, dm->ddev) { acrtc = to_amdgpu_crtc(crtc); ism = &acrtc->ism; - /* Cancel and disable any pending work */ disable_delayed_work_sync(&ism->delayed_work); disable_delayed_work_sync(&ism->sso_delayed_work); + } +} + +/** + * amdgpu_dm_ism_force_full_power - Force every CRTC's ISM FSM to FULL_POWER + * + * @dm: The amdgpu display manager + * + * Sends DM_ISM_EVENT_EXIT_IDLE_REQUESTED to every CRTC's ISM, leaving each + * FSM in FULL_POWER_RUNNING. Intended to be paired with + * amdgpu_dm_ism_disable(): callers should first quiesce workers (without + * dc_lock), then take dc_lock and call this helper. + * + * Must be called with dc_lock held. + */ +void amdgpu_dm_ism_force_full_power(struct amdgpu_display_manager *dm) +{ + struct drm_crtc *crtc; + struct amdgpu_crtc *acrtc; + + /* + * Caller must hold dc_lock: commit_event() drives the FSM and + * may touch dc state via dc_allow_idle_optimizations() etc. + */ + lockdep_assert_held(&dm->dc_lock); + + drm_for_each_crtc(crtc, dm->ddev) { + acrtc = to_amdgpu_crtc(crtc); /* * When disabled, leave in FULL_POWER_RUNNING state. - * EXIT_IDLE will not queue any work + * EXIT_IDLE will not queue any work. */ - amdgpu_dm_ism_commit_event(ism, + amdgpu_dm_ism_commit_event(&acrtc->ism, DM_ISM_EVENT_EXIT_IDLE_REQUESTED); } } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h index fde0ddc8d4e4..964408cd9a83 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h @@ -146,6 +146,7 @@ void amdgpu_dm_ism_fini(struct amdgpu_dm_ism *ism); void amdgpu_dm_ism_commit_event(struct amdgpu_dm_ism *ism, enum amdgpu_dm_ism_event event); void amdgpu_dm_ism_disable(struct amdgpu_display_manager *dm); +void amdgpu_dm_ism_force_full_power(struct amdgpu_display_manager *dm); void amdgpu_dm_ism_enable(struct amdgpu_display_manager *dm); #endif diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c index 1b09a7907cb4..ee6d52b87b02 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c @@ -493,6 +493,10 @@ static enum bp_result get_gpio_i2c_info( - sizeof(struct atom_common_table_header)) / sizeof(struct atom_gpio_pin_assignment); + if (!bios_get_image(&bp->base, DATA_TABLES(gpio_pin_lut), + le16_to_cpu(header->table_header.structuresize))) + return BP_RESULT_BADBIOSTABLE; + pin = (struct atom_gpio_pin_assignment *) header->gpio_pin; for (table_index = 0; table_index < count; table_index++) { @@ -681,6 +685,11 @@ static enum bp_result bios_parser_get_gpio_pin_info( count = (le16_to_cpu(header->table_header.structuresize) - sizeof(struct atom_common_table_header)) / sizeof(struct atom_gpio_pin_assignment); + + if (!bios_get_image(&bp->base, DATA_TABLES(gpio_pin_lut), + le16_to_cpu(header->table_header.structuresize))) + return BP_RESULT_BADBIOSTABLE; + for (i = 0; i < count; ++i) { if (header->gpio_pin[i].gpio_id != gpio_id) continue; diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c index 8d2cf95ae739..e00dc05c2d9d 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c @@ -37,10 +37,13 @@ uint8_t *bios_get_image(struct dc_bios *bp, uint32_t offset, uint32_t size) { - if (bp->bios && offset + size < bp->bios_size) - return bp->bios + offset; - else + if (!bp->bios) return NULL; + + if (offset > bp->bios_size || size > bp->bios_size - offset) + return NULL; + + return bp->bios + offset; } #include "reg_helper.h" diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c index d856a7a807b1..d01e5969a670 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c @@ -309,6 +309,8 @@ void dcn42_update_clocks(struct clk_mgr *clk_mgr_base, } // workaround: Limit dppclk to 100Mhz to avoid lower eDP panel switch to plus 4K monitor underflow. + if (new_clocks->dppclk_khz < 100000) + new_clocks->dppclk_khz = 100000; if (should_set_clock(safe_to_lower, new_clocks->dppclk_khz, clk_mgr->base.clks.dppclk_khz)) { if (clk_mgr->base.clks.dppclk_khz > new_clocks->dppclk_khz) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index dc9b7959d548..6d520d5b02e9 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -6614,7 +6614,11 @@ bool dc_process_dmub_aux_transfer_async(struct dc *dc, uint8_t action; union dmub_rb_cmd cmd = {0}; - ASSERT(payload->length <= 16); + if (link_index >= dc->link_count || !dc->links[link_index]) + return false; + + if (payload->length > sizeof(cmd.dp_aux_access.aux_control.dpaux.data)) + return false; cmd.dp_aux_access.header.type = DMUB_CMD__DP_AUX_ACCESS; cmd.dp_aux_access.header.payload_bytes = 0; diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index d0b6fad65bc0..75f59ca927ff 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -63,7 +63,7 @@ struct dcn_dsc_reg_state; struct dcn_optc_reg_state; struct dcn_dccg_reg_state; -#define DC_VER "3.2.382" +#define DC_VER "3.2.383" /** * MAX_SURFACES - representative of the upper bound of surfaces that can be piped to a single CRTC @@ -1220,6 +1220,7 @@ struct dc_debug_options { bool enable_otg_frame_sync_pwa; unsigned int min_deep_sleep_dcfclk_khz; unsigned int force_odm2to1_for_edp_pixclk_mhz; + bool enable_replay_esd_recovery; }; diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h index c2ca08d26e37..fa64bf6e711c 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h @@ -346,7 +346,9 @@ enum swizzle_mode_addr3_values { DC_ADDR3_SW_4KB_3D = 5, DC_ADDR3_SW_64KB_3D = 6, DC_ADDR3_SW_256KB_3D = 7, - DC_ADDR3_SW_MAX = 8, + DC_ADDR3_SW_64KB_2D_Z = 8, + DC_ADDR3_SW_256KB_2D_Z = 9, + DC_ADDR3_SW_MAX = 10, DC_ADDR3_SW_UNKNOWN = DC_ADDR3_SW_MAX }; diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c index 8faffc2993b4..2ebc738c37f2 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c @@ -603,6 +603,7 @@ static void dpp3_power_on_blnd_lut( if (power_on) { REG_UPDATE(CM_MEM_PWR_CTRL, BLNDGAM_MEM_PWR_FORCE, 0); REG_WAIT(CM_MEM_PWR_STATUS, BLNDGAM_MEM_PWR_STATE, 0, 1, 5); + dpp_base->deferred_reg_writes.bits.disable_blnd_lut = false; } else { dpp_base->ctx->dc->optimized_required = true; dpp_base->deferred_reg_writes.bits.disable_blnd_lut = true; @@ -620,6 +621,7 @@ static void dpp3_power_on_hdr3dlut( if (power_on) { REG_UPDATE(CM_MEM_PWR_CTRL2, HDR3DLUT_MEM_PWR_FORCE, 0); REG_WAIT(CM_MEM_PWR_STATUS2, HDR3DLUT_MEM_PWR_STATE, 0, 1, 5); + dpp_base->deferred_reg_writes.bits.disable_3dlut = false; } else { dpp_base->ctx->dc->optimized_required = true; dpp_base->deferred_reg_writes.bits.disable_3dlut = true; @@ -637,6 +639,7 @@ static void dpp3_power_on_shaper( if (power_on) { REG_UPDATE(CM_MEM_PWR_CTRL2, SHAPER_MEM_PWR_FORCE, 0); REG_WAIT(CM_MEM_PWR_STATUS2, SHAPER_MEM_PWR_STATE, 0, 1, 5); + dpp_base->deferred_reg_writes.bits.disable_shaper = false; } else { dpp_base->ctx->dc->optimized_required = true; dpp_base->deferred_reg_writes.bits.disable_shaper = true; diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c index 99581f35e54b..e01d8e9672fc 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c @@ -136,6 +136,7 @@ static void dpp3_power_on_gamcor_lut( if (dpp_base->ctx->dc->caps.ips_v2_support) REG_UPDATE(CM_MEM_PWR_CTRL, GAMCOR_MEM_PWR_DIS, 1); REG_WAIT(CM_MEM_PWR_STATUS, GAMCOR_MEM_PWR_STATE, 0, 1, 5); + dpp_base->deferred_reg_writes.bits.disable_gamcor = false; } else { dpp_base->ctx->dc->optimized_required = true; dpp_base->deferred_reg_writes.bits.disable_gamcor = true; diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c index df3c21ef57a1..62994aecf499 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c @@ -161,6 +161,7 @@ static void dpp401_power_on_dscl( REG_WAIT(DSCL_MEM_PWR_STATUS, LUT_MEM_PWR_STATE, 0, 1, 100); } else REG_WAIT(DSCL_MEM_PWR_STATUS, LUT_MEM_PWR_STATE, 0, 1, 5); + dpp->base.deferred_reg_writes.bits.disable_dscl = false; } else { if (dpp->base.ctx->dc->debug.enable_mem_low_power.bits.dscl) { dpp->base.ctx->dc->optimized_required = true; diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c index e12ed7591848..bf30a1bb61b7 100644 --- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c +++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c @@ -617,7 +617,9 @@ bool hubbub401_dcc_support_swizzle( swizzle_supported = true; break; case DC_ADDR3_SW_64KB_2D: + case DC_ADDR3_SW_64KB_2D_Z: case DC_ADDR3_SW_256KB_2D: + case DC_ADDR3_SW_256KB_2D_Z: swizzle_supported = true; break; default: diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c index 3a6e3a0d18b0..1733f551248e 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c @@ -181,6 +181,7 @@ void dcn10_set_wait_for_update_needed_for_pipe(struct dc *dc, struct pipe_ctx *p uint32_t vupdate_start, vupdate_end; struct crtc_position position; unsigned int vpos, cur_frame; + uint32_t max_frame_count; if (!pipe_ctx->stream || !pipe_ctx->stream_res.tg || @@ -197,7 +198,8 @@ void dcn10_set_wait_for_update_needed_for_pipe(struct dc *dc, struct pipe_ctx *p struct optc *optc1 = DCN10TG_FROM_TG(tg); - ASSERT(optc1->max_frame_count != 0); + max_frame_count = optc1->tg_mask->OTG_FRAME_COUNT >> optc1->tg_shift->OTG_FRAME_COUNT; + ASSERT(max_frame_count != 0); if (tg->funcs->is_tg_enabled && !tg->funcs->is_tg_enabled(tg)) return; @@ -209,8 +211,8 @@ void dcn10_set_wait_for_update_needed_for_pipe(struct dc *dc, struct pipe_ctx *p if (vpos < vupdate_start) { pipe_ctx->wait_frame_count = cur_frame; } else { - if (cur_frame + 1 > optc1->max_frame_count) - pipe_ctx->wait_frame_count = cur_frame + 1 - optc1->max_frame_count; + if (cur_frame + 1 > max_frame_count) + pipe_ctx->wait_frame_count = cur_frame + 1 - max_frame_count; else pipe_ctx->wait_frame_count = cur_frame + 1; } diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c index 079c226c1097..b5e82e190124 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c @@ -118,6 +118,8 @@ static const struct hwseq_private_funcs dcn10_private_funcs = { .dsc_pg_control = NULL, .set_hdr_multiplier = dcn10_set_hdr_multiplier, .verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high, + .wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed, + .set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe, }; void dcn10_hw_sequencer_construct(struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c index ad253c586ea1..1797a91b0186 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c @@ -136,6 +136,8 @@ static const struct hwseq_private_funcs dcn20_private_funcs = { .dccg_init = dcn20_dccg_init, .set_blend_lut = dcn20_set_blend_lut, .set_shaper_3dlut = dcn20_set_shaper_3dlut, + .wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed, + .set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe, }; void dcn20_hw_sequencer_construct(struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c index 5cbae0cdda96..9834d3075487 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c @@ -145,6 +145,8 @@ static const struct hwseq_private_funcs dcn30_private_funcs = { .wait_for_blank_complete = dcn20_wait_for_blank_complete, .set_blend_lut = dcn30_set_blend_lut, .set_shaper_3dlut = dcn20_set_shaper_3dlut, + .wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed, + .set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe, }; void dcn30_hw_sequencer_construct(struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c index 33cc48cd0196..a570333aeac1 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c @@ -142,6 +142,8 @@ static const struct hwseq_private_funcs dcn301_private_funcs = { .wait_for_blank_complete = dcn20_wait_for_blank_complete, .set_blend_lut = dcn30_set_blend_lut, .set_shaper_3dlut = dcn20_set_shaper_3dlut, + .wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed, + .set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe, }; void dcn301_hw_sequencer_construct(struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c index e56b9a46aecf..b14e6e60b878 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c @@ -147,6 +147,8 @@ static const struct hwseq_private_funcs dcn31_private_funcs = { .set_blend_lut = dcn30_set_blend_lut, .set_shaper_3dlut = dcn20_set_shaper_3dlut, .setup_hpo_hw_control = dcn31_setup_hpo_hw_control, + .wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed, + .set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe, }; void dcn31_hw_sequencer_construct(struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c index 9900c87b4567..d782080883ab 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c @@ -154,6 +154,8 @@ static const struct hwseq_private_funcs dcn314_private_funcs = { .setup_hpo_hw_control = dcn31_setup_hpo_hw_control, .calculate_dccg_k1_k2_values = dcn314_calculate_dccg_k1_k2_values, .resync_fifo_dccg_dio = dcn314_resync_fifo_dccg_dio, + .wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed, + .set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe, }; void dcn314_hw_sequencer_construct(struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c index 849dae18b738..c68b20104773 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c @@ -163,6 +163,8 @@ static const struct hwseq_private_funcs dcn32_private_funcs = { .is_dp_dig_pixel_rate_div_policy = dcn32_is_dp_dig_pixel_rate_div_policy, .apply_single_controller_ctx_to_hw = dce110_apply_single_controller_ctx_to_hw, .reset_back_end_for_pipe = dcn20_reset_back_end_for_pipe, + .wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed, + .set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe, }; void dcn32_hw_sequencer_init_functions(struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c index 5d0dfb36f3e1..0908a791832b 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c @@ -167,6 +167,8 @@ static const struct hwseq_private_funcs dcn401_private_funcs = { .perform_3dlut_wa_unlock = dcn401_perform_3dlut_wa_unlock, .program_pipe_sequence = dcn401_program_pipe_sequence, .dc_ip_request_cntl = dcn401_dc_ip_request_cntl, + .wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed, + .set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe, }; void dcn401_hw_sequencer_init_functions(struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c index cbc20f214f9e..cabfac151940 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c @@ -470,9 +470,9 @@ static bool is_rmcm_3dlut_fl_supported(struct dc *dc, enum dc_cm2_gpu_mem_size s if (!dc->caps.color.mpc.rmcm_3d_lut_caps.dma_3d_lut) return false; if (size == DC_CM2_GPU_MEM_SIZE_171717) - return (dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_17); + return dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_17 != 0u; else if (size == DC_CM2_GPU_MEM_SIZE_333333) - return (dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_33); + return dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_33 != 0u; return false; } @@ -945,6 +945,7 @@ bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx, void dcn42_hardware_release(struct dc *dc) { dcn35_hardware_release(dc); + dc_dmub_srv_release_hw(dc); } static int count_active_streams(const struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h index 0d5a8358a778..7f371cbb35cd 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h @@ -68,7 +68,6 @@ struct optc { int pstate_keepout; struct dc_crtc_timing orginal_patched_timing; enum signal_type signal; - uint32_t max_frame_count; }; void optc1_read_otg_state(struct timing_generator *optc, struct dcn_otg_state *s); diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c index d52155e86f0f..21f64946b993 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c @@ -2097,10 +2097,10 @@ static enum dc_status enable_link( return status; } -static bool allocate_usb4_bandwidth_for_stream(struct dc_stream_state *stream, int bw) +static bool allocate_usb4_bandwidth_for_stream(struct dc_stream_state *stream, int stream_bw) { struct dc_link *link = stream->sink->link; - int req_bw = bw; + int req_bw = stream_bw; DC_LOGGER_INIT(link->ctx->logger); @@ -2108,39 +2108,38 @@ static bool allocate_usb4_bandwidth_for_stream(struct dc_stream_state *stream, i return false; if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { - int sink_index = 0; + int sink_index = -1; unsigned int i = 0; for (i = 0; i < link->sink_count; i++) { if (link->remote_sinks[i] == NULL) continue; - if (stream->sink->sink_id != link->remote_sinks[i]->sink_id) + if (stream->sink->sink_id != link->remote_sinks[i]->sink_id) { + DC_LOG_DEBUG("%s: add remote_sink=%s, request_bw=%d\n", __func__, + (const char *)(&link->remote_sinks[i]->edid_caps.display_name[0]), + link->dpia_bw_alloc_config.remote_sink_req_bw[i]); + req_bw += link->dpia_bw_alloc_config.remote_sink_req_bw[i]; - else + } else sink_index = i; } - link->dpia_bw_alloc_config.remote_sink_req_bw[sink_index] = bw; - } + if (sink_index >= 0) + link->dpia_bw_alloc_config.remote_sink_req_bw[sink_index] = stream_bw; + else + DC_LOG_WARNING("%s: stream sink_id=%u not found in remote_sinks[]\n", + __func__, stream->sink->sink_id); - link->dpia_bw_alloc_config.dp_overhead = link_dpia_get_dp_overhead(link); - req_bw += link->dpia_bw_alloc_config.dp_overhead; + if (req_bw) { + link->dpia_bw_alloc_config.dp_overhead = link_dpia_get_dp_overhead(link); + req_bw += link->dpia_bw_alloc_config.dp_overhead; + } else + link->dpia_bw_alloc_config.dp_overhead = 0; + } link_dp_dpia_allocate_usb4_bandwidth_for_stream(link, req_bw); - if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { - unsigned int i = 0; - - for (i = 0; i < link->sink_count; i++) { - if (link->remote_sinks[i] == NULL) - continue; - DC_LOG_DEBUG("%s, remote_sink=%s, request_bw=%d\n", __func__, - (const char *)(&link->remote_sinks[i]->edid_caps.display_name[0]), - link->dpia_bw_alloc_config.remote_sink_req_bw[i]); - } - } - return true; } diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c index f9e0ba24d401..54ce768ae6ad 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c @@ -312,12 +312,14 @@ static void handle_hpd_irq_replay_sink(struct dc_link *link, bool *need_re_enabl } } - if (!link->replay_settings.replay_allow_active && - replay_sink_status.bits.SINK_DEVICE_REPLAY_STATUS == 0x7) { - /* If sink device replay status is 0x7 and replay is disabled, - * it means sink is in a bad state and link retraining is needed to recover - */ - *replay_esd_detection_needed = true; + if (link->ctx->dc->debug.enable_replay_esd_recovery) { + if (!link->replay_settings.replay_allow_active && + replay_sink_status.bits.SINK_DEVICE_REPLAY_STATUS == 0x7) { + /* If sink device replay status is 0x7 and replay is disabled, + * it means sink is in a bad state and link retraining is needed to recover + */ + *replay_esd_detection_needed = true; + } } } diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c index 72d6e6011a09..d87f87a02d63 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c @@ -321,6 +321,19 @@ bool dp_pr_copy_settings(struct dc_link *link, struct replay_context *replay_con cmd.pr_copy_settings.data.flags.bitfields.dsc_enable_status = (pipe_ctx->stream->timing.flags.DSC == 1); cmd.pr_copy_settings.data.debug.u32All = link->replay_settings.config.debug_flags; + // ALPM settings + cmd.pr_copy_settings.data.flags.bitfields.alpm_mode = (enum dmub_alpm_mode)link->replay_settings.config.alpm_mode; + if (link->replay_settings.config.alpm_mode == DC_ALPM_AUXLESS) { + cmd.pr_copy_settings.data.auxless_alpm_data.lfps_setup_ns = (uint16_t)dc->debug.auxless_alpm_lfps_setup_ns; + cmd.pr_copy_settings.data.auxless_alpm_data.lfps_period_ns = (uint16_t)dc->debug.auxless_alpm_lfps_period_ns; + cmd.pr_copy_settings.data.auxless_alpm_data.lfps_silence_ns = (uint16_t)dc->debug.auxless_alpm_lfps_silence_ns; + cmd.pr_copy_settings.data.auxless_alpm_data.lfps_t1_t2_override_us = + (uint16_t)dc->debug.auxless_alpm_lfps_t1t2_us; + cmd.pr_copy_settings.data.auxless_alpm_data.lfps_t1_t2_offset_us = + (uint16_t)dc->debug.auxless_alpm_lfps_t1t2_offset_us; + cmd.pr_copy_settings.data.auxless_alpm_data.lttpr_count = link->dc->link_srv->dp_get_lttpr_count(link); + } + cmd.pr_copy_settings.data.su_granularity_needed = link->dpcd_caps.vesa_replay_caps.bits.PR_SU_GRANULARITY_NEEDED; cmd.pr_copy_settings.data.su_x_granularity = link->dpcd_caps.vesa_replay_su_info.pr_su_x_granularity; cmd.pr_copy_settings.data.su_y_granularity = link->dpcd_caps.vesa_replay_su_info.pr_su_y_granularity; diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c index 72b5921227d2..e06a9ac65286 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c @@ -462,7 +462,7 @@ bool edp_receiver_ready_T9(struct dc_link *link) do { sinkstatus = 1; result = core_link_read_dpcd(link, DP_SINK_STATUS, &sinkstatus, sizeof(sinkstatus)); - if (sinkstatus == 0) + if (!(sinkstatus & DP_RECEIVE_PORT_0_STATUS)) break; if (result != DC_OK) break; @@ -492,7 +492,7 @@ bool edp_receiver_ready_T7(struct dc_link *link) do { sinkstatus = 0; result = core_link_read_dpcd(link, DP_SINK_STATUS, &sinkstatus, sizeof(sinkstatus)); - if (sinkstatus == 1) + if (sinkstatus & DP_RECEIVE_PORT_0_STATUS) break; if (result != DC_OK) break; diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c index a880e4a6d165..62f45c156c32 100644 --- a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c +++ b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c @@ -621,7 +621,6 @@ void dcn35_timing_generator_init(struct optc *optc1) optc1->min_v_blank_interlace = 5; optc1->min_h_sync_width = 4; optc1->min_v_sync_width = 1; - optc1->max_frame_count = 0xFFFFFF; dcn35_timing_generator_set_fgcg( optc1, CTX->dc->debug.enable_fine_grain_clock_gating.bits.optc); diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c index ed66a2bbb8ae..a3431ec2f058 100644 --- a/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c +++ b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c @@ -283,9 +283,7 @@ void dcn42_timing_generator_init(struct optc *optc1) optc1->min_v_blank_interlace = 5; optc1->min_h_sync_width = 4; optc1->min_v_sync_width = 1; - optc1->max_frame_count = 0xFFFFFF; dcn35_timing_generator_set_fgcg( optc1, CTX->dc->debug.enable_fine_grain_clock_gating.bits.optc); } - diff --git a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c index 4a096fa3fc10..729c2b653161 100644 --- a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c +++ b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c @@ -27,9 +27,6 @@ static bool pg_cntl42_dsc_pg_status(struct pg_cntl *pg_cntl, unsigned int dsc_in struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl); uint32_t pwr_status = 0; - if (pg_cntl->ctx->dc->debug.ignore_pg) - return true; - switch (dsc_inst) { case 0: /* DSC0 */ REG_GET(DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, &pwr_status); @@ -64,9 +61,11 @@ void pg_cntl42_dsc_pg_control(struct pg_cntl *pg_cntl, unsigned int dsc_inst, bo pg_cntl->ctx->dc->res_pool->dccg->funcs->enable_dsc( pg_cntl->ctx->dc->res_pool->dccg, dsc_inst); - if (pg_cntl->ctx->dc->debug.ignore_pg || - pg_cntl->ctx->dc->debug.disable_dsc_power_gate || - pg_cntl->ctx->dc->idle_optimizations_allowed) + bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg || + pg_cntl->ctx->dc->debug.disable_dsc_power_gate || + pg_cntl->ctx->dc->idle_optimizations_allowed; + + if (skip_pg && !power_on) return; block_enabled = pg_cntl42_dsc_pg_status(pg_cntl, dsc_inst); @@ -261,11 +260,12 @@ void pg_cntl42_hpo_pg_control(struct pg_cntl *pg_cntl, bool power_on) uint32_t power_forceon; bool block_enabled; - if (pg_cntl->ctx->dc->debug.ignore_pg || - pg_cntl->ctx->dc->debug.disable_hpo_power_gate || - pg_cntl->ctx->dc->idle_optimizations_allowed) - return; + bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg || + pg_cntl->ctx->dc->debug.disable_hpo_power_gate || + pg_cntl->ctx->dc->idle_optimizations_allowed; + if (skip_pg && !power_on) + return; block_enabled = pg_cntl42_hpo_pg_status(pg_cntl); if (power_on) { if (block_enabled) @@ -316,9 +316,11 @@ void pg_cntl42_io_clk_pg_control(struct pg_cntl *pg_cntl, bool power_on) uint32_t power_forceon; bool block_enabled; - if (pg_cntl->ctx->dc->debug.ignore_pg || - pg_cntl->ctx->dc->idle_optimizations_allowed || - pg_cntl->ctx->dc->debug.disable_io_clk_power_gate) + bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg || + pg_cntl->ctx->dc->idle_optimizations_allowed || + pg_cntl->ctx->dc->debug.disable_io_clk_power_gate; + + if (skip_pg && !power_on) return; block_enabled = pg_cntl42_io_clk_status(pg_cntl); @@ -412,9 +414,11 @@ void pg_cntl42_mem_pg_control(struct pg_cntl *pg_cntl, bool power_on) uint32_t power_forceon; bool block_enabled; - if (pg_cntl->ctx->dc->debug.ignore_pg || - pg_cntl->ctx->dc->idle_optimizations_allowed || - pg_cntl->ctx->dc->debug.disable_mem_power_gate) + bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg || + pg_cntl->ctx->dc->idle_optimizations_allowed || + pg_cntl->ctx->dc->debug.disable_mem_power_gate; + + if (skip_pg && !power_on) return; block_enabled = pg_cntl42_mem_status(pg_cntl); @@ -464,9 +468,10 @@ void pg_cntl42_dio_pg_control(struct pg_cntl *pg_cntl, bool power_on) uint32_t org_ip_request_cntl; bool block_enabled; - if (pg_cntl->ctx->dc->debug.ignore_pg || - pg_cntl->ctx->dc->idle_optimizations_allowed || - pg_cntl->ctx->dc->debug.disable_dio_power_gate) + bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg || + pg_cntl->ctx->dc->idle_optimizations_allowed || + pg_cntl->ctx->dc->debug.disable_dio_power_gate; + if (skip_pg && !power_on) return; block_enabled = pg_cntl42_dio_pg_status(pg_cntl); diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c index 52a1996a654f..1fc457cec293 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c @@ -1520,6 +1520,10 @@ static void dcn42_resource_destruct(struct dcn42_resource_pool *pool) if (pool->base.pg_cntl != NULL) dcn_pg_cntl_destroy(&pool->base.pg_cntl); + + if (pool->base.replay != NULL) + dmub_replay_destroy(&pool->base.replay); + if (pool->base.dccg != NULL) dcn_dccg_destroy(&pool->base.dccg); @@ -1921,12 +1925,6 @@ static bool dcn42_resource_construct( dc->caps.max_v_total = (1 << 15) - 1; dc->caps.vtotal_limited_by_fp2 = true; - dc->caps.seamless_odm = true; - dc->caps.zstate_support = true; - dc->caps.ips_support = true; - dc->caps.max_v_total = (1 << 15) - 1; - dc->caps.vtotal_limited_by_fp2 = true; - /* Color pipeline capabilities */ dc->caps.color.dpp.dcn_arch = 1; dc->caps.color.dpp.input_lut_shared = 0; diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h index 60acf0e423d9..fe960542d7bd 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h @@ -456,6 +456,8 @@ SRI_ARR(OTG_V_SYNC_A, OTG, inst), \ SRI_ARR(OTG_V_SYNC_A_CNTL, OTG, inst), \ SRI_ARR(OTG_CONTROL, OTG, inst), \ + SRI_ARR(OTG_MASTER_EN, OTG, inst), \ + SRI_ARR(OTG_LONG_VBLANK_STATUS, OTG, inst), \ SRI_ARR(OTG_STEREO_CONTROL, OTG, inst), \ SRI_ARR(OTG_3D_STRUCTURE_CONTROL, OTG, inst), \ SRI_ARR(OTG_STEREO_STATUS, OTG, inst), \ @@ -472,6 +474,11 @@ SRI_ARR(OTG_M_CONST_DTO0, OTG, inst), \ SRI_ARR(OTG_M_CONST_DTO1, OTG, inst), \ SRI_ARR(OTG_CLOCK_CONTROL, OTG, inst), \ + SRI_ARR(OTG_COUNT_CONTROL, OTG, inst), \ + SRI_ARR(OTG_COUNT_RESET, OTG, inst), \ + SRI_ARR(OTG_CRC_SIG_BLUE_CONTROL_MASK, OTG, inst), \ + SRI_ARR(OTG_CRC_SIG_RED_GREEN_MASK, OTG, inst), \ + SRI_ARR(OTG_DRR_TIMING_INT_STATUS, OTG, inst), \ SRI_ARR(OTG_VERTICAL_INTERRUPT0_CONTROL, OTG, inst), \ SRI_ARR(OTG_VERTICAL_INTERRUPT0_POSITION, OTG, inst), \ SRI_ARR(OTG_VERTICAL_INTERRUPT1_CONTROL, OTG, inst), \ diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h index c1becd664cb9..2f68438000ea 100644 --- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h +++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h @@ -1008,30 +1008,15 @@ union dmub_fw_boot_status { * enum dmub_fw_boot_status_bit - Enum bit definitions for SCRATCH0. */ enum dmub_fw_boot_status_bit { - /** @DMUB_FW_BOOT_STATUS_BIT_DAL_FIRMWARE: 1 if DAL FW */ - DMUB_FW_BOOT_STATUS_BIT_DAL_FIRMWARE = (1 << 0), - /** @DMUB_FW_BOOT_STATUS_BIT_MAILBOX_READY: 1 if mailbox ready */ - DMUB_FW_BOOT_STATUS_BIT_MAILBOX_READY = (1 << 1), - /** @DMUB_FW_BOOT_STATUS_BIT_OPTIMIZED_INIT_DONE: 1 if init done */ - DMUB_FW_BOOT_STATUS_BIT_OPTIMIZED_INIT_DONE = (1 << 2), - /** @DMUB_FW_BOOT_STATUS_BIT_RESTORE_REQUIRED: 1 if driver should call restore */ - DMUB_FW_BOOT_STATUS_BIT_RESTORE_REQUIRED = (1 << 3), - /** - * @DMUB_FW_BOOT_STATUS_BIT_DEFERRED_LOADED: 1 if VBIOS data is - * deferred programmed - */ - DMUB_FW_BOOT_STATUS_BIT_DEFERRED_LOADED = (1 << 4), - /** @DMUB_FW_BOOT_STATUS_BIT_FAMS_ENABLED: 1 if FAMS is enabled */ - DMUB_FW_BOOT_STATUS_BIT_FAMS_ENABLED = (1 << 5), - /** - * @DMUB_FW_BOOT_STATUS_BIT_DETECTION_REQUIRED: 1 if detection needs - * to be triggered by driver - */ - DMUB_FW_BOOT_STATUS_BIT_DETECTION_REQUIRED = (1 << 6), - /** @DMUB_FW_BOOT_STATUS_BIT_HW_POWER_INIT_DONE: 1 if hw power init is completed */ - DMUB_FW_BOOT_STATUS_BIT_HW_POWER_INIT_DONE = (1 << 7), - /** @DMUB_FW_BOOT_STATUS_BIT_ONO_REGIONS_ENABLED: if ONO regions are enabled */ - DMUB_FW_BOOT_STATUS_BIT_ONO_REGIONS_ENABLED = (1 << 8), + DMUB_FW_BOOT_STATUS_BIT_DAL_FIRMWARE = (1 << 0), /**< 1 if DAL FW */ + DMUB_FW_BOOT_STATUS_BIT_MAILBOX_READY = (1 << 1), /**< 1 if mailbox ready */ + DMUB_FW_BOOT_STATUS_BIT_OPTIMIZED_INIT_DONE = (1 << 2), /**< 1 if init done */ + DMUB_FW_BOOT_STATUS_BIT_RESTORE_REQUIRED = (1 << 3), /**< 1 if driver should call restore */ + DMUB_FW_BOOT_STATUS_BIT_DEFERRED_LOADED = (1 << 4), /**< 1 if VBIOS data is deferred programmed */ + DMUB_FW_BOOT_STATUS_BIT_FAMS_ENABLED = (1 << 5), /**< 1 if FAMS is enabled*/ + DMUB_FW_BOOT_STATUS_BIT_DETECTION_REQUIRED = (1 << 6), /**< 1 if detection need to be triggered by driver*/ + DMUB_FW_BOOT_STATUS_BIT_HW_POWER_INIT_DONE = (1 << 7), /**< 1 if hw power init is completed */ + DMUB_FW_BOOT_STATUS_BIT_ONO_REGIONS_ENABLED = (1 << 8), /**< 1 if ONO regions are enabled */ }; /* Register bit definition for SCRATCH5 */ @@ -1095,22 +1080,18 @@ enum dmub_ips_comand_type { */ enum dmub_cursor_offload_comand_type { /** - * @DMUB_CMD__CURSOR_OFFLOAD_INIT: * Initializes the cursor offload feature. */ DMUB_CMD__CURSOR_OFFLOAD_INIT = 0, /** - * @DMUB_CMD__CURSOR_OFFLOAD_STREAM_ENABLE: Enables cursor offloading - * for a stream and updates the timing parameters. + * Enables cursor offloading for a stream and updates the timing parameters. */ DMUB_CMD__CURSOR_OFFLOAD_STREAM_ENABLE = 1, /** - * @DMUB_CMD__CURSOR_OFFLOAD_STREAM_DISABLE: * Disables cursor offloading for a given stream. */ DMUB_CMD__CURSOR_OFFLOAD_STREAM_DISABLE = 2, /** - * @DMUB_CMD__CURSOR_OFFLOAD_STREAM_PROGRAM: * Programs the latest data for a given stream. */ DMUB_CMD__CURSOR_OFFLOAD_STREAM_PROGRAM = 3, @@ -1176,7 +1157,7 @@ enum dmub_fw_boot_options_bit { * [Header (256 Bytes)][Feature 1 (256 Bytes)][Feature 2 (256 Bytes)]... */ -/* +/** * enum dmub_shared_state_feature_id - List of shared state features. */ enum dmub_shared_state_feature_id { @@ -1933,36 +1914,30 @@ enum dmub_cmd_type { */ enum dmub_out_cmd_type { /** - * @DMUB_OUT_CMD__NULL: Invalid outbox command, ignored. + * Invalid outbox command, ignored. */ DMUB_OUT_CMD__NULL = 0, /** - * @DMUB_OUT_CMD__DP_AUX_REPLY: * Command type used for DP AUX Reply data notification */ DMUB_OUT_CMD__DP_AUX_REPLY = 1, /** - * @DMUB_OUT_CMD__DP_HPD_NOTIFY: * Command type used for DP HPD event notification */ DMUB_OUT_CMD__DP_HPD_NOTIFY = 2, /** - * @DMUB_OUT_CMD__SET_CONFIG_REPLY: * Command type used for SET_CONFIG Reply notification */ DMUB_OUT_CMD__SET_CONFIG_REPLY = 3, /** - * @DMUB_OUT_CMD__DPIA_NOTIFICATION: * Command type used for USB4 DPIA notification */ DMUB_OUT_CMD__DPIA_NOTIFICATION = 5, /** - * @DMUB_OUT_CMD__HPD_SENSE_NOTIFY: * Command type used for HPD redetect notification */ DMUB_OUT_CMD__HPD_SENSE_NOTIFY = 6, /** - * @DMUB_OUT_CMD__FUSED_IO: * Command type used for Fused IO notification */ DMUB_OUT_CMD__FUSED_IO = 7, @@ -2154,19 +2129,19 @@ struct dmub_rb_cmd_PLAT_54186_wa { */ enum dmub_cmd_mall_type { /** - * @DMUB_CMD__MALL_ACTION_ALLOW: Allows display refresh from MALL. + * Allows display refresh from MALL. */ DMUB_CMD__MALL_ACTION_ALLOW = 0, /** - * @DMUB_CMD__MALL_ACTION_DISALLOW: Disallows display refresh from MALL. + * Disallows display refresh from MALL. */ DMUB_CMD__MALL_ACTION_DISALLOW = 1, /** - * @DMUB_CMD__MALL_ACTION_COPY_CURSOR: Cursor copy for MALL. + * Cursor copy for MALL. */ DMUB_CMD__MALL_ACTION_COPY_CURSOR = 2, /** - * @DMUB_CMD__MALL_ACTION_NO_DF_REQ: Controls DF requests. + * Controls DF requests. */ DMUB_CMD__MALL_ACTION_NO_DF_REQ = 3, }; @@ -2195,21 +2170,18 @@ struct dmub_rb_cmd_mall { */ enum dmub_cmd_cab_type { /** - * @DMUB_CMD__CAB_NO_IDLE_OPTIMIZATION: * No idle optimizations (i.e. no CAB) */ DMUB_CMD__CAB_NO_IDLE_OPTIMIZATION = 0, /** - * @DMUB_CMD__CAB_NO_DCN_REQ: No DCN requests for memory + * No DCN requests for memory */ DMUB_CMD__CAB_NO_DCN_REQ = 1, /** - * @DMUB_CMD__CAB_DCN_SS_FIT_IN_CAB: * Fit surfaces in CAB (i.e. CAB enable) */ DMUB_CMD__CAB_DCN_SS_FIT_IN_CAB = 2, /** - * @DMUB_CMD__CAB_DCN_SS_NOT_FIT_IN_CAB: * Do not fit surfaces in CAB (i.e. no CAB) */ DMUB_CMD__CAB_DCN_SS_NOT_FIT_IN_CAB = 3, @@ -2224,7 +2196,7 @@ struct dmub_rb_cmd_cab_for_ss { uint8_t debug_bits; /* debug bits */ }; -/* +/** * Enum for indicating which MCLK switch mode per pipe */ enum mclk_switch_mode { @@ -2702,18 +2674,11 @@ struct dmub_fams2_stream_static_state_v1 { * p-state request to allow latency */ enum dmub_fams2_allow_delay_check_mode { - /** - * @FAMS2_ALLOW_DELAY_CHECK_NONE: No check for request to allow delay - */ + /* No check for request to allow delay */ FAMS2_ALLOW_DELAY_CHECK_NONE = 0, - /** - * @FAMS2_ALLOW_DELAY_CHECK_FROM_START: Check for request to allow delay - */ + /* Check for request to allow delay */ FAMS2_ALLOW_DELAY_CHECK_FROM_START = 1, - /** - * @FAMS2_ALLOW_DELAY_CHECK_FROM_PREPARE: - * Check for prepare to allow delay - */ + /* Check for prepare to allow delay */ FAMS2_ALLOW_DELAY_CHECK_FROM_PREPARE = 2, }; @@ -2785,28 +2750,27 @@ struct dmub_rb_cmd_ib { */ enum dmub_cmd_idle_opt_type { /** - * @DMUB_CMD__IDLE_OPT_DCN_RESTORE: DCN hardware restore. + * DCN hardware restore. */ DMUB_CMD__IDLE_OPT_DCN_RESTORE = 0, /** - * @DMUB_CMD__IDLE_OPT_DCN_SAVE_INIT: DCN hardware save. + * DCN hardware save. */ DMUB_CMD__IDLE_OPT_DCN_SAVE_INIT = 1, /** - * @DMUB_CMD__IDLE_OPT_DCN_NOTIFY_IDLE: DCN hardware notify idle. + * DCN hardware notify idle. */ DMUB_CMD__IDLE_OPT_DCN_NOTIFY_IDLE = 2, /** - * @DMUB_CMD__IDLE_OPT_SET_DC_POWER_STATE: * DCN hardware notify power state. */ DMUB_CMD__IDLE_OPT_SET_DC_POWER_STATE = 3, /** - * @DMUB_CMD__IDLE_OPT_RELEASE_HW: DCN notify to release HW. + * DCN notify to release HW. */ DMUB_CMD__IDLE_OPT_RELEASE_HW = 4, }; @@ -2835,7 +2799,7 @@ struct dmub_rb_cmd_idle_opt_dcn_notify_idle { struct dmub_dcn_notify_idle_cntl_data cntl_data; }; -/* +/** * enum dmub_idle_opt_dc_power_state - DC power states. */ enum dmub_idle_opt_dc_power_state { @@ -2877,7 +2841,7 @@ struct dmub_clocks { */ enum dmub_cmd_clk_mgr_type { /** - * @DMUB_CMD__CLK_MGR_NOTIFY_CLOCKS: Notify DMCUB of clock update. + * Notify DMCUB of clock update. */ DMUB_CMD__CLK_MGR_NOTIFY_CLOCKS = 0, }; @@ -3122,40 +3086,21 @@ struct dmub_rb_cmd_dpphy_init { * 4 AUX request command bits are shifted to high nibble. */ enum dp_aux_request_action { - /** - * @DP_AUX_REQ_ACTION_I2C_WRITE: I2C-over-AUX write request - */ + /** I2C-over-AUX write request */ DP_AUX_REQ_ACTION_I2C_WRITE = 0x00, - /** - * @DP_AUX_REQ_ACTION_I2C_READ: I2C-over-AUX read request - */ + /** I2C-over-AUX read request */ DP_AUX_REQ_ACTION_I2C_READ = 0x10, - /** - * @DP_AUX_REQ_ACTION_I2C_STATUS_REQ: I2C-over-AUX write status request - */ + /** I2C-over-AUX write status request */ DP_AUX_REQ_ACTION_I2C_STATUS_REQ = 0x20, - /** - * @DP_AUX_REQ_ACTION_I2C_WRITE_MOT: - * I2C-over-AUX write request with MOT=1 - */ + /** I2C-over-AUX write request with MOT=1 */ DP_AUX_REQ_ACTION_I2C_WRITE_MOT = 0x40, - /** - * @DP_AUX_REQ_ACTION_I2C_READ_MOT: - * I2C-over-AUX read request with MOT=1 - */ + /** I2C-over-AUX read request with MOT=1 */ DP_AUX_REQ_ACTION_I2C_READ_MOT = 0x50, - /** - * @DP_AUX_REQ_ACTION_I2C_STATUS_REQ_MOT: - * I2C-over-AUX write status request with MOT=1 - */ + /** I2C-over-AUX write status request with MOT=1 */ DP_AUX_REQ_ACTION_I2C_STATUS_REQ_MOT = 0x60, - /** - * @DP_AUX_REQ_ACTION_DPCD_WRITE: Native AUX write request - */ + /** Native AUX write request */ DP_AUX_REQ_ACTION_DPCD_WRITE = 0x80, - /** - * @DP_AUX_REQ_ACTION_DPCD_READ: Native AUX read request - */ + /** Native AUX read request */ DP_AUX_REQ_ACTION_DPCD_READ = 0x90 }; @@ -3163,21 +3108,21 @@ enum dp_aux_request_action { * enum aux_return_code_type - DP AUX process return code listing. */ enum aux_return_code_type { - /** @AUX_RET_SUCCESS: AUX process succeeded */ + /** AUX process succeeded */ AUX_RET_SUCCESS = 0, - /** @AUX_RET_ERROR_UNKNOWN: AUX process failed with unknown reason */ + /** AUX process failed with unknown reason */ AUX_RET_ERROR_UNKNOWN, - /** @AUX_RET_ERROR_INVALID_REPLY: AUX process completed with invalid reply */ + /** AUX process completed with invalid reply */ AUX_RET_ERROR_INVALID_REPLY, - /** @AUX_RET_ERROR_TIMEOUT: AUX process timed out */ + /** AUX process timed out */ AUX_RET_ERROR_TIMEOUT, - /** @AUX_RET_ERROR_HPD_DISCON: HPD was low during AUX process */ + /** HPD was low during AUX process */ AUX_RET_ERROR_HPD_DISCON, - /** @AUX_RET_ERROR_ENGINE_ACQUIRE: Failed to acquire AUX engine */ + /** Failed to acquire AUX engine */ AUX_RET_ERROR_ENGINE_ACQUIRE, - /** @AUX_RET_ERROR_INVALID_OPERATION: AUX request not supported */ + /** AUX request not supported */ AUX_RET_ERROR_INVALID_OPERATION, - /** @AUX_RET_ERROR_PROTOCOL_ERROR: AUX process not available */ + /** AUX process not available */ AUX_RET_ERROR_PROTOCOL_ERROR, }; @@ -3185,9 +3130,9 @@ enum aux_return_code_type { * enum aux_channel_type - DP AUX channel type listing. */ enum aux_channel_type { - /** @AUX_CHANNEL_LEGACY_DDC: AUX thru Legacy DP AUX */ + /** AUX thru Legacy DP AUX */ AUX_CHANNEL_LEGACY_DDC, - /** @AUX_CHANNEL_DPIA: AUX thru DPIA DP tunneling */ + /** AUX thru DPIA DP tunneling */ AUX_CHANNEL_DPIA }; @@ -3672,7 +3617,7 @@ enum phy_link_rate { */ }; -/* +/** * enum dmub_phy_fsm_state - PHY FSM states. * PHY FSM state to transit to during PSR enable/disable. */ @@ -5949,11 +5894,10 @@ struct dmub_rb_cmd_abm_query_caps { */ enum dmub_abm_ace_curve_type { /** - * @ABM_ACE_CURVE_TYPE__SW: ACE curve as defined by the SW layer. + * ACE curve as defined by the SW layer. */ ABM_ACE_CURVE_TYPE__SW = 0, /** - * @ABM_ACE_CURVE_TYPE__SW_IF: * ACE curve as defined by the SW to HW translation interface layer. */ ABM_ACE_CURVE_TYPE__SW_IF = 1, @@ -5964,11 +5908,10 @@ enum dmub_abm_ace_curve_type { */ enum dmub_abm_histogram_type { /** - * @ABM_HISTOGRAM_TYPE__SW: ACE curve as defined by the SW layer. + * ACE curve as defined by the SW layer. */ ABM_HISTOGRAM_TYPE__SW = 0, /** - * @ABM_HISTOGRAM_TYPE__SW_IF: * ACE curve as defined by the SW to HW translation interface layer. */ ABM_HISTOGRAM_TYPE__SW_IF = 1, @@ -6174,17 +6117,14 @@ struct dmub_rb_cmd_get_visual_confirm_color { */ enum dmub_cmd_panel_cntl_type { /** - * @DMUB_CMD__PANEL_CNTL_HW_INIT: * Initializes embedded panel hardware blocks. */ DMUB_CMD__PANEL_CNTL_HW_INIT = 0, /** - * @DMUB_CMD__PANEL_CNTL_QUERY_BACKLIGHT_INFO: * Queries backlight info for the embedded panel. */ DMUB_CMD__PANEL_CNTL_QUERY_BACKLIGHT_INFO = 1, /** - * @DMUB_CMD__PANEL_DEBUG_PWM_FREQ: * Sets the PWM Freq as per user's requirement. */ DMUB_CMD__PANEL_DEBUG_PWM_FREQ = 2, diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_power.h b/drivers/gpu/drm/amd/display/modules/inc/mod_power.h index 440e4284b6f0..f9814cf7bbdb 100644 --- a/drivers/gpu/drm/amd/display/modules/inc/mod_power.h +++ b/drivers/gpu/drm/amd/display/modules/inc/mod_power.h @@ -414,27 +414,4 @@ bool mod_power_backlight_nits_to_percent(struct mod_power *mod_power, unsigned int backlight_millinit, unsigned int *backlight_millipercent); -void initialize_backlight_caps(struct core_power *core_power, unsigned int inst); - -unsigned int backlight_millipercent_to_pwm( - struct core_power *core_power, unsigned int millipercent, unsigned int inst); - -unsigned int backlight_millipercent_to_millinit( - struct core_power *core_power, unsigned int millipercent, unsigned int inst); - -void fill_backlight_level_params(struct core_power *core_power, - struct set_backlight_level_params *backlight_level_params, - int panel_inst, uint8_t aux_inst, unsigned int backlight_pwm, - enum backlight_control_type backlight_control_type, - unsigned int backlight_millinit, unsigned int transition_time_millisec, - bool is_hdr); - -bool mod_power_hw_init_backlight(struct mod_power *mod_power); - -void mod_power_update_backlight_on_mode_change( - struct core_power *core_power, - struct dc_link *link, - unsigned int panel_inst, - uint8_t aux_inst, - bool is_hdr); #endif /* MODULES_INC_MOD_POWER_H_ */ diff --git a/drivers/gpu/drm/amd/display/modules/power/Makefile b/drivers/gpu/drm/amd/display/modules/power/Makefile index dd11f7d5617d..3000f392bdbc 100644 --- a/drivers/gpu/drm/amd/display/modules/power/Makefile +++ b/drivers/gpu/drm/amd/display/modules/power/Makefile @@ -23,9 +23,9 @@ # Makefile for the 'power' sub-module of DAL. # -MOD_POWER = power_helpers.o power.o power_abm.o +MOD_POWER = power_helpers.o power.o power_abm.o power_psr.o power_replay.o AMD_DAL_MOD_POWER = $(addprefix $(AMDDALPATH)/modules/power/,$(MOD_POWER)) #$(info ************ DAL POWER MODULE MAKEFILE ************) -AMD_DISPLAY_FILES += $(AMD_DAL_MOD_POWER) +AMD_DISPLAY_FILES += $(AMD_DAL_MOD_POWER) \ No newline at end of file diff --git a/drivers/gpu/drm/amd/display/modules/power/power.c b/drivers/gpu/drm/amd/display/modules/power/power.c index f8a0b252c745..1dd65bdf8cc9 100644 --- a/drivers/gpu/drm/amd/display/modules/power/power.c +++ b/drivers/gpu/drm/amd/display/modules/power/power.c @@ -42,101 +42,6 @@ #define SMOOTH_BRIGHTNESS_ADJUSTMENT_TIME_IN_MS 500 #define LOW_REFRESH_RATE_DURATION_US_UPPER_BOUND 25000 - -struct backlight_state { - /* HW uses u16.16 format for backlight PWM */ - unsigned int backlight_pwm; - /* DM may call power module to set backlight - * targeting percent brightness - */ - unsigned int backlight_millipercent; - /* DM may call power module to set backlight based on an explicit - * nits value. - */ - unsigned int backlight_millinit; - unsigned int frame_ramp; - bool smooth_brightness_enabled; - bool isHDR; -}; -struct power_entity { - struct dc_stream_state *stream; - struct psr_caps *caps; - struct mod_power_psr_context *psr_context; - - /*PSR cached properties*/ - bool psr_enabled; - unsigned int psr_events; - unsigned int psr_power_opt; - unsigned int replay_events; -}; - -struct pwr_backlight_properties { - bool use_nits_based_brightness; - bool disable_fractional_pwm; - - unsigned int min_abm_backlight; - unsigned int num_backlight_levels; - - bool backlight_ramping_override; - unsigned int backlight_ramping_reduction; - unsigned int backlight_ramping_start; - - /* Backlight cached properties */ - unsigned int ac_backlight_percent; - unsigned int dc_backlight_percent; - - /* backlight LUT stored in HW u16.16 format*/ - unsigned int *backlight_lut; - unsigned int min_backlight_pwm; - unsigned int max_backlight_pwm; - unsigned int backlight_range; - - /* Describes the panel's min and max luminance in millinits measured - * on full white screen, in min and max backlight settings. - */ - unsigned int min_brightness_millinits; - unsigned int max_brightness_millinits; - unsigned int nits_range; - - bool backlight_caps_valid; - bool use_custom_backlight_caps; - unsigned int custom_backlight_caps_config_no; - bool use_linear_backlight_curve; -}; - -struct dmcu_varibright_cached_properties { - unsigned int varibright_config_setting; - unsigned int varibright_level; - unsigned int varibright_hw_level; - unsigned int def_varibright_level; - bool varibright_user_enable; - bool varibright_active; -}; - -struct core_power { - struct mod_power mod_public; - struct dc *dc; - struct power_entity *map; - struct dmcu_varibright_cached_properties varibright_prop; - struct pwr_backlight_properties bl_prop[MAX_NUM_EDP]; - struct backlight_state bl_state[MAX_NUM_EDP]; - unsigned int edp_num; - - bool psr_smu_optimizations_support; - bool multi_disp_optimizations_support; - - unsigned int num_entities; -}; - -union dmcu_abm_set_bl_params { - struct { - unsigned int gradual_change : 1; /* [0:0] */ - unsigned int reserved : 15; /* [15:1] */ - unsigned int frame_ramp : 16; /* [31:16] */ - } bits; - unsigned int u32All; -}; - /* If system or panel does not report some sort of brightness percent to nits * mapping, we will use following default values so backlight control using * nits based interfaces will still work, but might not describe panel @@ -155,24 +60,10 @@ static const unsigned int default_dc_backlight_percent = 70; #define MOD_POWER_TO_CORE(mod_power)\ container_of(mod_power, struct core_power, mod_public) -static unsigned int calc_psr_num_static_frames(unsigned int vsync_rate_hz) -{ - /* Initialize fail-safe to 2 static frames. */ - unsigned int num_frames_static = 2; - - /* Calculate number of frames such that at least 30 ms has passed. - * Round up to ensure the static period is not shorter than 30 ms. - */ - if (vsync_rate_hz != 0) - num_frames_static = DIV_ROUND_UP(30000 * vsync_rate_hz, 1000000); - - return num_frames_static; -} - /* Given a specific dc_stream* this function finds its equivalent * on the core_freesync->map and returns the corresponding index */ -static unsigned int map_index_from_stream(struct core_power *core_power, +unsigned int map_index_from_stream(struct core_power *core_power, const struct dc_stream_state *stream) { unsigned int index = 0; @@ -564,7 +455,6 @@ bool mod_power_replace_stream(struct mod_power *mod_power, return true; } - bool mod_power_notify_mode_change(struct mod_power *mod_power, const struct dc_stream_state *stream, bool is_hdr) @@ -572,12 +462,8 @@ bool mod_power_notify_mode_change(struct mod_power *mod_power, unsigned int stream_index = 0; struct core_power *core_power = NULL; struct dc_link *link = NULL; - struct psr_config psr_config = {0}; - struct psr_context psr_context = {0}; struct dc *dc = NULL; unsigned int panel_inst = 0; - int active_psr_events = 0; - int active_replay_events = 0; if ((mod_power == NULL) || (stream == NULL)) return false; @@ -594,1236 +480,19 @@ bool mod_power_notify_mode_change(struct mod_power *mod_power, dc = core_power->dc; link = dc_stream_get_link(stream); - active_psr_events = core_power->map[stream_index].psr_events; - active_replay_events = core_power->map[stream_index].replay_events; + if (link != NULL && dc_get_edp_link_panel_inst(dc, link, &panel_inst)) { ASSERT(link->ddc->ddc_pin->hw_info.ddc_channel <= 0xFF); uint8_t aux_inst = (uint8_t)link->ddc->ddc_pin->hw_info.ddc_channel; mod_power_update_backlight_on_mode_change(core_power, link, panel_inst, aux_inst, is_hdr); - mod_power_calc_psr_configs(&psr_config, link, stream); + /* Handle PSR notification */ + mod_power_psr_notify_mode_change(mod_power, stream, link, stream_index); - psr_config.psr_exit_link_training_required = core_power->map[stream_index].caps->psr_exit_link_training_required; - - if (dc->ctx->asic_id.chip_family >= AMDGPU_FAMILY_GC_11_0_1) - psr_config.allow_smu_optimizations = - core_power->psr_smu_optimizations_support && dc_is_embedded_signal(stream->signal); - else - psr_config.allow_smu_optimizations = - core_power->psr_smu_optimizations_support && mod_power_only_edp(dc->current_state, stream); - - psr_config.allow_multi_disp_optimizations = core_power->multi_disp_optimizations_support; - - psr_config.rate_control_caps = core_power->map[stream_index].caps->rate_control_caps; - - if (active_psr_events & psr_event_os_request_force_ffu) { - psr_config.os_request_force_ffu = true; - } - /* - * DSC support: - * DSC slice height value must be 'mod' by su_y_granularity. - * According to Panel Vendor, there might be varied conditions to fulfill. - * Right now, DSC slice height value must be multiple of su_y_granularity. - * - * The value of DSC slice height is determined in DSC Driver but it does not - * propagated out here, so we need to calculate it as below 'slice_height'. - */ - psr_su_set_dsc_slice_height(dc, link, - (struct dc_stream_state *) stream, - &psr_config); - - dc_link_setup_psr(link, stream, &psr_config, &psr_context); - - link->replay_settings.replay_smu_opt_enable = - (link->replay_settings.config.replay_smu_opt_supported && - mod_power_only_edp(dc->current_state, stream)); - - if (active_replay_events & replay_event_os_request_force_ffu) { - link->replay_settings.config.os_request_force_ffu = true; - } - - if (dc_is_embedded_signal(stream->signal)) - dc->link_srv->dp_setup_replay(link, stream); + /* Handle Replay notification */ + mod_power_replay_notify_mode_change(mod_power, dc, link, stream, stream_index); } return true; } - -static void mod_power_psr_set_power_opt(struct mod_power *mod_power, - struct dc_stream_state *stream, - unsigned int active_psr_events, - bool psr_enable_request) -{ - (void)psr_enable_request; - struct core_power *core_power = NULL; - struct dc_link *link = NULL; - unsigned int stream_index = 0; - unsigned int power_opt = 0; - - if (!stream) - return; - - core_power = MOD_POWER_TO_CORE(mod_power); - stream_index = map_index_from_stream(core_power, stream); - if (!core_power->map[stream_index].caps->psr_version) - return; - - link = dc_stream_get_link(stream); - - if (active_psr_events == 0) { - /* Static Screen */ - power_opt |= (psr_power_opt_smu_opt_static_screen | psr_power_opt_z10_static_screen | - psr_power_opt_ds_disable_allow); - } - - /* psr_power_opt_flag is a configuration parameter into the module that determines - * which optimizations to enable during psr - */ - power_opt &= core_power->map[stream_index].caps->psr_power_opt_flag; - if (core_power->map[stream_index].psr_power_opt != power_opt) { - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_VERBOSE, - WPP_BIT_FLAG_Firmware_PsrState, - "mod_power set_power_opt: psr_power_opt=0x%04x, power_opt=0x%04x" - "active_psr_events=0x%04x, psr_power_opt_flag=0x%04x", - core_power->map[stream_index].psr_power_opt, - power_opt, - active_psr_events, - core_power->map[stream_index].caps->psr_power_opt_flag); - dc_link_set_psr_allow_active(link, NULL, false, false, &power_opt); - core_power->map[stream_index].psr_power_opt = power_opt; - } -} - -static bool set_psr_enable(struct mod_power *mod_power, - struct dc_stream_state *stream, - bool psr_enable, - bool wait, - bool force_static) -{ - struct core_power *core_power = NULL; - enum dc_psr_state state = PSR_STATE0; - unsigned int retry_count; - const unsigned int max_retry = 1000; - struct dc_link *link = NULL; - - if (mod_power == NULL) - return false; - - core_power = MOD_POWER_TO_CORE(mod_power); - - if (core_power->num_entities == 0) { - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, - WPP_BIT_FLAG_Firmware_PsrState, - "set psr enable: ERROR: stream=%p num_entities=%u", - stream, - core_power->num_entities); - return false; - } - - if (psr_enable) { - unsigned int vsync_rate_hz; - struct dc_static_screen_params params = {0}; - - vsync_rate_hz = (unsigned int)div_u64(div_u64(( - stream->timing.pix_clk_100hz * 100), - stream->timing.v_total), - stream->timing.h_total); - - params.triggers.cursor_update = true; - params.triggers.overlay_update = true; - params.triggers.surface_update = true; - params.num_frames = calc_psr_num_static_frames(vsync_rate_hz); - - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, - WPP_BIT_FLAG_Firmware_PsrState, - "set psr enable: CALCS: pix_clk_100hz=%u v_total=%u h_total=%u vsync_rate_hz=%u num_frames=%u", - stream->timing.pix_clk_100hz, - stream->timing.v_total, - stream->timing.h_total, - vsync_rate_hz, - params.num_frames); - - dc_stream_set_static_screen_params(core_power->dc, - &stream, 1, - ¶ms); - } - - link = dc_stream_get_link(stream); - - if (!dc_link_set_psr_allow_active(link, &psr_enable, false, force_static, NULL)) { - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, - WPP_BIT_FLAG_Firmware_PsrState, - "set psr enable: ERROR: stream=%p link=%p psr_enable=%d", - stream, - link, - psr_enable); - return false; - } - - if (wait == true) { - - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, - WPP_BIT_FLAG_Firmware_PsrState, - "set psr enable: BEGIN WAIT: psr_enable=%d", - (int)psr_enable); - - for (retry_count = 0; retry_count <= max_retry; retry_count++) { - dc_link_get_psr_state(link, &state); - if (psr_enable) { - if (state != PSR_STATE0 && - (!force_static || state == PSR_STATE3)) - break; - } else { - if (state == PSR_STATE0) - break; - } - udelay(500); - } - - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, - WPP_BIT_FLAG_Firmware_PsrState, - "set psr enable: END WAIT: psr_enable=%d", - (int)psr_enable); - - /* assert if max retry hit */ - if (retry_count >= max_retry) { - ASSERT(0); - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, - WPP_BIT_FLAG_Firmware_PsrState, - "set psr enable: ERROR: retry_count=%u: Unexpectedly long wait for PSR state change.", - retry_count); - } - } else { - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, - WPP_BIT_FLAG_Firmware_PsrState, - "set psr enable: PSR state change initiated (wait=false): psr_enable=%d", - (int)psr_enable); - } - - return true; -} - -bool mod_power_get_psr_event(struct mod_power *mod_power, - struct dc_stream_state *stream, - unsigned int *active_psr_events) -{ - struct core_power *core_power = NULL; - unsigned int stream_index = 0; - - if (mod_power == NULL) - return false; - - core_power = MOD_POWER_TO_CORE(mod_power); - - if (core_power->num_entities == 0) - return false; - - stream_index = map_index_from_stream(core_power, stream); - - if (!core_power->map[stream_index].caps->psr_version) - return false; - - *active_psr_events = core_power->map[stream_index].psr_events; - - return true; -} - -bool mod_power_set_psr_event(struct mod_power *mod_power, - struct dc_stream_state *stream, bool set_event, - enum psr_event event, bool wait) -{ - struct core_power *core_power = NULL; - unsigned int stream_index = 0; - unsigned int active_psr_events = 0; - bool psr_enable_request = false; - bool force_static = false; - - if (mod_power == NULL || stream == NULL) - return false; - - core_power = MOD_POWER_TO_CORE(mod_power); - stream_index = map_index_from_stream(core_power, stream); - - if (core_power->num_entities == 0) { - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, - WPP_BIT_FLAG_Firmware_PsrState, - "mod_power set_psr_event: ERROR: stream=%p event=%d num_entities=%u", - stream, - (int)event, - core_power->num_entities); - return false; - } - - if (!core_power->map[stream_index].caps->psr_version) - return false; - - if (set_event) - core_power->map[stream_index].psr_events |= event; - else - core_power->map[stream_index].psr_events &= ~event; - - active_psr_events = core_power->map[stream_index].psr_events; - - // ignore other events when we're in forced psr enabled state - if (active_psr_events & psr_event_dynamic_display_switch && - event != psr_event_dynamic_display_switch) - return false; - - // ignore other events when we're in forced psr enabled state - if (active_psr_events & psr_event_os_override_hold && - event != psr_event_os_override_hold) - return false; - - // ignore other events when we're in forced psr enabled state - // dds events need to be processed while in dynamic_link_rate_control - if (active_psr_events & psr_event_dynamic_link_rate_control && - event != psr_event_dynamic_link_rate_control && - event != psr_event_dds_defer_stream_enable && - event != psr_event_dynamic_display_switch) - return false; - - if (active_psr_events & (psr_event_test_harness_disable_psr | psr_event_os_request_disable)) - psr_enable_request = false; - else if (active_psr_events & psr_event_pause) - psr_enable_request = false; - else if (active_psr_events & psr_event_test_harness_enable_psr) - psr_enable_request = true; - else if (active_psr_events & psr_event_dynamic_display_switch) { - psr_enable_request = true; - force_static = true; - } else if (active_psr_events & psr_event_dynamic_link_rate_control) { - psr_enable_request = true; - force_static = true; - } else if (active_psr_events & psr_event_edp_panel_off_disable_psr) - psr_enable_request = false; - else if (active_psr_events & (psr_event_hw_programming | - psr_event_defer_enable | - psr_event_dds_defer_stream_enable | - psr_event_vrr_transition | - psr_event_immediate_flip)) - psr_enable_request = false; - else if (active_psr_events & psr_event_big_screen_video) - psr_enable_request = true; - else if (active_psr_events & psr_event_full_screen) - psr_enable_request = false; - else if (active_psr_events & psr_event_mpo_video_selective_update) - psr_enable_request = true; - else if (active_psr_events & psr_event_vsync) - psr_enable_request = false; - else if (active_psr_events & psr_event_crc_window_active) - psr_enable_request = false; - else - psr_enable_request = true; - - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_VERBOSE, - WPP_BIT_FLAG_Firmware_PsrState, - "mod_power set_psr_event: before: psr_enabled=%d -> request: set_event=%d event=0x%04x -> result: psr_events=0x%04x psr_enable_request=%d", - (int)core_power->map[stream_index].psr_enabled, - (int)set_event, - (unsigned int)event, - (unsigned int)core_power->map[stream_index].psr_events, - (int)psr_enable_request); - mod_power_psr_set_power_opt(mod_power, stream, active_psr_events, psr_enable_request); - - if (core_power->map[stream_index].psr_enabled != psr_enable_request || force_static) { - if (set_psr_enable(mod_power, stream, psr_enable_request, wait, force_static)) { - core_power->map[stream_index].psr_enabled = psr_enable_request; - } - } - - return true; -} - -bool mod_power_get_psr_state(struct mod_power *mod_power, - const struct dc_stream_state *stream, - enum dc_psr_state *state) -{ - struct core_power *core_power = NULL; - const struct dc_link *link = NULL; - - if (!stream) - return false; - - if (mod_power == NULL) - return false; - - core_power = MOD_POWER_TO_CORE(mod_power); - - if (core_power->num_entities == 0) - return false; - - link = dc_stream_get_link(stream); - return dc_link_get_psr_state(link, state); -} - -bool mod_power_get_psr_enabled_status(struct mod_power *mod_power, - const struct dc_stream_state *stream, - bool *psr_enabled) -{ - struct core_power *core_power = NULL; - unsigned int stream_index = 0; - - if (mod_power == NULL) - return false; - - core_power = MOD_POWER_TO_CORE(mod_power); - - if (core_power->num_entities == 0) - return false; - - stream_index = map_index_from_stream(core_power, stream); - - if (!core_power->map[stream_index].caps->psr_version) - return false; - - *psr_enabled = core_power->map[stream_index].psr_enabled; - - return true; -} - -void mod_power_psr_residency(struct mod_power *mod_power, - const struct dc_stream_state *stream, - unsigned int *residency, - const uint8_t mode) -{ - struct core_power *core_power = NULL; - const struct dc_link *link = NULL; - - if (!stream) - return; - - if (mod_power == NULL) - return; - - core_power = MOD_POWER_TO_CORE(mod_power); - - if (core_power->num_entities == 0) - return; - - link = dc_stream_get_link(stream); - - if (link != NULL) - link->dc->link_srv->edp_get_psr_residency(link, residency, mode); -} -bool mod_power_psr_get_active_psr_events(struct mod_power *mod_power, - const struct dc_stream_state *stream, unsigned int *active_psr_events) -{ - struct core_power *core_power = NULL; - unsigned int stream_index = 0; - - if (!stream) - return false; - - if (mod_power == NULL) - return false; - - if (active_psr_events == NULL) - return false; - - core_power = MOD_POWER_TO_CORE(mod_power); - - if (core_power->num_entities == 0) - return false; - - stream_index = map_index_from_stream(core_power, stream); - - *active_psr_events = core_power->map[stream_index].psr_events; - return true; -} - -bool mod_power_psr_set_sink_vtotal_in_psr_active(struct mod_power *mod_power, - const struct dc_stream_state *stream, - uint16_t psr_vtotal_idle, - uint16_t psr_vtotal_su) -{ - struct core_power *core_power = NULL; - unsigned int stream_index = 0; - const struct dc_link *link = NULL; - - if (!stream) - return false; - - if (mod_power == NULL) - return false; - - core_power = MOD_POWER_TO_CORE(mod_power); - - if (core_power->num_entities == 0) - return false; - - stream_index = map_index_from_stream(core_power, stream); - - if (!core_power->map[stream_index].caps->psr_version) - return false; - - link = dc_stream_get_link(stream); - - return link->dc->link_srv->edp_set_sink_vtotal_in_psr_active( - link, psr_vtotal_idle, psr_vtotal_su); -} - -static bool mod_power_set_replay_active(struct dc_stream_state *stream, - bool replay_active, - bool wait, - bool force_static) -{ - uint64_t state; - unsigned int retry_count; - const unsigned int max_retry = 1000; - struct dc_link *link = NULL; - - if (!stream) - return false; - - link = dc_stream_get_link(stream); - - if (!link) - return false; - - if (!dc_link_set_replay_allow_active(link, &replay_active, false, force_static, NULL)) - return false; - - if (wait == true) { - - for (retry_count = 0; retry_count <= max_retry; retry_count++) { - dc_link_get_replay_state(link, &state); - if (replay_active) { - if (state != REPLAY_STATE_0 && - (!force_static || state == REPLAY_STATE_3)) - break; - } else { - if (state == REPLAY_STATE_0) - break; - } - udelay(500); - } - - /* assert if max retry hit */ - if (retry_count >= max_retry) - ASSERT(0); - } else { - /* To-do: Add trace log */ - } - - return true; -} - -static unsigned int mod_power_replay_setup_power_opt(struct dc_link *link, - unsigned int active_replay_events, bool is_ultra_sleep_mode) -{ - unsigned int power_opt = 0; - - if (is_ultra_sleep_mode) { - /* Static Screen */ - power_opt |= (replay_power_opt_smu_opt_static_screen | replay_power_opt_z10_static_screen); - } else if (active_replay_events & replay_event_test_harness_ultra_sleep) { - power_opt |= replay_power_opt_z10_static_screen; - } - - /* replay_power_opt_flag is a configuration parameter into the module that determines - * which optimizations to enable during replay - */ - power_opt &= link->replay_settings.config.replay_power_opt_supported; - - return power_opt; -} - -static bool mod_power_replay_set_power_opt(struct mod_power *mod_power, - struct dc_stream_state *stream, - unsigned int active_replay_events, - bool is_ultra_sleep_mode) -{ - (void)mod_power; - struct dc_link *link = NULL; - unsigned int power_opt = 0; - - if (!stream) - return false; - - link = dc_stream_get_link(stream); - - if (!link || !link->replay_settings.replay_feature_enabled) - return false; - - power_opt = mod_power_replay_setup_power_opt(link, active_replay_events, is_ultra_sleep_mode); - - if (!dc_link_set_replay_allow_active(link, NULL, false, false, &power_opt)) - return false; - - return true; -} - -bool mod_power_get_replay_event(struct mod_power *mod_power, - struct dc_stream_state *stream, - unsigned int *active_replay_events) -{ - struct core_power *core_power = NULL; - unsigned int stream_index = 0; - - if (mod_power == NULL) - return false; - - core_power = MOD_POWER_TO_CORE(mod_power); - - if (core_power->num_entities == 0) - return false; - - stream_index = map_index_from_stream(core_power, stream); - - *active_replay_events = core_power->map[stream_index].replay_events; - - return true; -} - -static bool mod_power_update_replay_active_status(unsigned int active_replay_events, - struct dc_link *link, uint32_t *coasting_vtotal, bool *is_full_screen_video, bool *is_ultra_sleep_mode, uint16_t *frame_skip_number, bool *is_video_playback) -{ - if (!link || !coasting_vtotal || !is_full_screen_video || !is_video_playback) - return false; - - // Check coasting_vtotal_table has been updated. - if (!link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_STATIC] - || !link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM]) - return false; - - unsigned int replay_enable_option = - link->replay_settings.config.replay_enable_option; - - /* TODO: To support test harness and DDS event */ - - *coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM]; - ASSERT(link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM] <= 0xFFFF); - *frame_skip_number = (uint16_t)link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM]; - - link->replay_settings.config.replay_timing_sync_supported = false; - - *is_full_screen_video = false; - - *is_ultra_sleep_mode = false; - - *is_video_playback = false; - - /* DSAT test scenario */ - if (active_replay_events & replay_event_test_harness_mode) { - if (link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]) - *coasting_vtotal = - link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]; - if (link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]) { - ASSERT(link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS] <= 0xFFFF); - *frame_skip_number = - (uint16_t)link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]; - } - - /* During the ultra sleep mode testing, disable the timing sync in short vblank mode */ - if (active_replay_events & (replay_event_test_harness_enable_replay)) { - if ((active_replay_events & replay_event_test_harness_ultra_sleep) && - !link->replay_settings.config.replay_support_fast_resync_in_ultra_sleep_mode) - link->replay_settings.config.replay_timing_sync_supported = false; - return true; - } else - return false; - } else if (active_replay_events & (replay_event_test_harness_enable_replay)) { - if (link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]) - *coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]; - if (link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]) { - uint32_t frame_skip_val = - link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]; - - ASSERT(frame_skip_val <= 0xFFFF); - *frame_skip_number = (uint16_t)frame_skip_val; - } - - /* During the ultra sleep mode testing, disable the timing sync in short vblank mode */ - if ((active_replay_events & replay_event_test_harness_ultra_sleep) && - !link->replay_settings.config.replay_support_fast_resync_in_ultra_sleep_mode) - link->replay_settings.config.replay_timing_sync_supported = false; - return true; - } else if (active_replay_events & (replay_event_test_harness_disable_replay | replay_event_os_request_disable)) { - // set last set coasting vtotal - if (link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]) - *coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]; - if (link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]) { - uint32_t frame_skip_val = - link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]; - - ASSERT(frame_skip_val <= 0xFFFF); - *frame_skip_number = (uint16_t)frame_skip_val; - } - return false; - } - - /* Inactive conditions */ - if (active_replay_events & (replay_event_edp_panel_off_disable_psr | - replay_event_hw_programming | - replay_event_vrr | - replay_event_immediate_flip | - replay_event_prepare_vtotal | - replay_event_vrr_transition | - replay_event_pause | - replay_event_disable_replay_while_DPMS | - replay_event_sleep_resume | - replay_event_disable_in_AC | - replay_event_disable_replay_while_detect_display | - replay_event_infopacket | - replay_event_crc_window_active)) - return false; - - // Full screen scenario - if (active_replay_events & replay_event_full_screen) { - if (!(replay_enable_option & pr_enable_option_full_screen)) - return false; - } - - /* Full screen video scenario */ - if (active_replay_events & replay_event_big_screen_video) { - - link->replay_settings.config.replay_timing_sync_supported = false; - - if (replay_enable_option & pr_enable_option_full_screen_video_coasting) { - unsigned int fsn_vid = - link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_FULL_SCREEN_VIDEO]; - - *coasting_vtotal = - link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_FULL_SCREEN_VIDEO]; - ASSERT(fsn_vid <= 0xFFFF); - *frame_skip_number = (uint16_t)fsn_vid; - } - - *is_video_playback = true; - - if ((replay_enable_option & pr_enable_option_full_screen_video) && - (replay_enable_option & pr_enable_option_full_screen_video_coasting)) { - *is_full_screen_video = true; - return true; - } else - return false; - } - - /* MPO video scenario - * Some of the cases may contain a full screen UI layer in MPO video scenario which is - * not the expected case to enable Replay. - */ - if ((active_replay_events & replay_event_mpo_video_selective_update) && - !(active_replay_events & replay_event_full_screen)) { - - link->replay_settings.config.replay_timing_sync_supported = false; - - if (replay_enable_option & pr_enable_option_mpo_video_coasting) { - *coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM]; - { - uint32_t frame_skip_val = - link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM]; - - ASSERT(frame_skip_val <= 0xFFFF); - *frame_skip_number = (uint16_t)frame_skip_val; - } - } - - *is_video_playback = true; - - if (replay_enable_option & pr_enable_option_mpo_video) - return true; - else - return false; - } - - /* Static screen scenario */ - if (!(active_replay_events & replay_event_vsync)) { - - if (replay_enable_option & pr_enable_option_static_screen_coasting) { - // Do not adjust eDP refresh rate if static screen + normal sleep mode - if ((!(link->replay_settings.config.replay_power_opt_supported & - replay_power_opt_z10_static_screen)) || - (active_replay_events & replay_event_cursor_updating)) { - // normal sleep mode - *coasting_vtotal = - link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM]; - { - uint32_t frame_skip_val = - link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM]; - - ASSERT(frame_skip_val <= 0xFFFF); - *frame_skip_number = (uint16_t)frame_skip_val; - } - } else { - // ultra sleep mode - *coasting_vtotal = - link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_STATIC]; - { - uint32_t frame_skip_val = - link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_STATIC]; - - ASSERT(frame_skip_val <= 0xFFFF); - *frame_skip_number = (uint16_t)frame_skip_val; - } - *is_ultra_sleep_mode = true; - } - } - - if (replay_enable_option & pr_enable_option_static_screen) { - if (!link->replay_settings.config.replay_support_fast_resync_in_ultra_sleep_mode) - link->replay_settings.config.replay_timing_sync_supported = false; - return true; - } else - return false; - } - - /* General UI scenario */ - if (active_replay_events & replay_event_general_ui) { - if (replay_enable_option & pr_enable_option_general_ui) - return true; - else - return false; - } - - return false; -} - -bool mod_power_replay_set_coasting_vtotal(struct mod_power *mod_power, - const struct dc_stream_state *stream, - uint32_t coasting_vtotal, - uint16_t frame_skip_number) -{ - struct core_power *core_power = NULL; - struct dc_link *link = NULL; - - if (!stream) - return false; - - link = dc_stream_get_link(stream); - if (!link || !link->replay_settings.replay_feature_enabled) - return false; - - if (mod_power == NULL) - return false; - - core_power = MOD_POWER_TO_CORE(mod_power); - - if (core_power->num_entities == 0) - return false; - - return link->dc->link_srv->edp_set_coasting_vtotal(link, coasting_vtotal, frame_skip_number); -} - -void mod_power_replay_set_timing_sync_supported(struct mod_power *mod_power, - const struct dc_stream_state *stream) -{ - struct core_power *core_power = NULL; - struct dc_link *link = NULL; - unsigned int stream_index = 0; - union dmub_replay_cmd_set cmd_data = { 0 }; - - if (!stream || mod_power == NULL) - return; - - core_power = MOD_POWER_TO_CORE(mod_power); - if (core_power->num_entities == 0) - return; - - stream_index = map_index_from_stream(core_power, stream); - if (stream_index > core_power->num_entities) //invalid index - return; - - link = dc_stream_get_link(stream); - if (!link || !link->replay_settings.replay_feature_enabled) - return; - - cmd_data.sync_data.timing_sync_supported = link->replay_settings.config.replay_timing_sync_supported; - - link->dc->link_srv->edp_send_replay_cmd(link, Replay_Set_Timing_Sync_Supported, - &cmd_data); -} - -void mod_power_replay_disabled_adaptive_sync_sdp(struct mod_power *mod_power, - const struct dc_stream_state *stream, bool force_disabled) -{ - struct core_power *core_power = NULL; - struct dc_link *link = NULL; - unsigned int stream_index = 0; - union dmub_replay_cmd_set cmd_data = { 0 }; - - if (!stream || mod_power == NULL) - return; - - core_power = MOD_POWER_TO_CORE(mod_power); - if (core_power->num_entities == 0) - return; - - stream_index = map_index_from_stream(core_power, stream); - if (stream_index > core_power->num_entities) //invalid index - return; - - link = dc_stream_get_link(stream); - if (!link || !link->replay_settings.replay_feature_enabled) - return; - - cmd_data.disabled_adaptive_sync_sdp_data.force_disabled = force_disabled; - - link->dc->link_srv->edp_send_replay_cmd(link, Replay_Disabled_Adaptive_Sync_SDP, - &cmd_data); -} - -static void mod_power_replay_set_general_cmd(struct mod_power *mod_power, - const struct dc_stream_state *stream, - const enum dmub_cmd_replay_general_subtype general_cmd_type, - const uint32_t param1, const uint32_t param2) -{ - struct core_power *core_power = NULL; - struct dc_link *link = NULL; - unsigned int stream_index = 0; - union dmub_replay_cmd_set cmd_data = { 0 }; - - if (!stream || mod_power == NULL) - return; - - core_power = MOD_POWER_TO_CORE(mod_power); - if (core_power->num_entities == 0) - return; - - stream_index = map_index_from_stream(core_power, stream); - if (stream_index > core_power->num_entities) //invalid index - return; - - link = dc_stream_get_link(stream); - if (!link || !link->replay_settings.replay_feature_enabled) - return; - - cmd_data.set_general_cmd_data.subtype = general_cmd_type; - cmd_data.set_general_cmd_data.param1 = param1; - cmd_data.set_general_cmd_data.param2 = param2; - link->dc->link_srv->edp_send_replay_cmd(link, Replay_Set_General_Cmd, - &cmd_data); -} - -void mod_power_replay_disabled_desync_error_detection(struct mod_power *mod_power, - const struct dc_stream_state *stream, bool force_disabled) -{ - mod_power_replay_set_general_cmd(mod_power, stream, - REPLAY_GENERAL_CMD_DISABLED_DESYNC_ERROR_DETECTION, - force_disabled, 0); -} - -static void mod_power_replay_set_pseudo_vtotal(struct mod_power *mod_power, - const struct dc_stream_state *stream, uint16_t vtotal) -{ - struct core_power *core_power = NULL; - struct dc_link *link = NULL; - unsigned int stream_index = 0; - union dmub_replay_cmd_set cmd_data = { 0 }; - - if (!stream || mod_power == NULL) - return; - - core_power = MOD_POWER_TO_CORE(mod_power); - if (core_power->num_entities == 0) - return; - - stream_index = map_index_from_stream(core_power, stream); - if (stream_index > core_power->num_entities) //invalid index - return; - - link = dc_stream_get_link(stream); - if (!link || !link->replay_settings.replay_feature_enabled) - return; - - cmd_data.pseudo_vtotal_data.vtotal = vtotal; - - if (link->replay_settings.last_pseudo_vtotal != vtotal) { - link->replay_settings.last_pseudo_vtotal = vtotal; - link->dc->link_srv->edp_send_replay_cmd(link, Replay_Set_Pseudo_VTotal, &cmd_data); - } -} - -static void mod_power_update_error_status(struct mod_power *mod_power, - const struct dc_stream_state *stream) -{ - struct dc_link *link = NULL; - union replay_debug_flags *pDebug = NULL; - - if (mod_power == NULL || stream == NULL) - return; - - link = dc_stream_get_link(stream); - - if (!link) - return; - - pDebug = (union replay_debug_flags *)&link->replay_settings.config.debug_flags; - - if (0 == pDebug->bitfields.enable_visual_confirm_debug) - return; - - mod_power_replay_set_general_cmd(mod_power, stream, - REPLAY_GENERAL_CMD_UPDATE_ERROR_STATUS, - link->replay_settings.config.replay_error_status.raw, 0); -} - -void mod_power_set_low_rr_activate(struct mod_power *mod_power, - const struct dc_stream_state *stream, bool low_rr_supported) -{ - struct dc_link *link = NULL; - - if (mod_power == NULL || stream == NULL) - return; - - link = dc_stream_get_link(stream); - - if (!link) - return; - - mod_power_replay_set_general_cmd(mod_power, stream, - REPLAY_GENERAL_CMD_SET_LOW_RR_ACTIVATE, - low_rr_supported, 0); -} - -void mod_power_set_video_conferencing_activate(struct mod_power *mod_power, - const struct dc_stream_state *stream, bool video_conferencing_activate) -{ - struct dc_link *link = NULL; - - if (mod_power == NULL || stream == NULL) - return; - - link = dc_stream_get_link(stream); - if (!link || !link->replay_settings.replay_feature_enabled) - return; - - mod_power_replay_set_general_cmd(mod_power, stream, - REPLAY_GENERAL_CMD_VIDEO_CONFERENCING, - video_conferencing_activate, 0); -} - -void mod_power_set_coasting_vtotal_without_frame_update(struct mod_power *mod_power, - const struct dc_stream_state *stream, uint32_t coasting_vtotal) -{ - struct dc_link *link = NULL; - - if (mod_power == NULL || stream == NULL) - return; - - link = dc_stream_get_link(stream); - if (!link || !link->replay_settings.replay_feature_enabled) - return; - - mod_power_replay_set_general_cmd(mod_power, stream, - REPLAY_GENERAL_CMD_SET_COASTING_VTOTAL_WITHOUT_FRAME_UPDATE, - coasting_vtotal, 0); -} - -void mod_power_set_replay_continuously_resync(struct mod_power *mod_power, - const struct dc_stream_state *stream, bool enable) -{ - struct dc_link *link = NULL; - - if (mod_power == NULL || stream == NULL) - return; - - link = dc_stream_get_link(stream); - if (!link || !link->replay_settings.replay_feature_enabled) - return; - - mod_power_replay_set_general_cmd(mod_power, stream, - REPLAY_GENERAL_CMD_SET_CONTINUOUSLY_RESYNC, - enable, 0); -} - -void mod_power_set_live_capture_with_cvt_activate(struct mod_power *mod_power, - const struct dc_stream_state *stream, bool live_capture_with_cvt_activate) -{ - struct dc_link *link = NULL; - - if (mod_power == NULL || stream == NULL) - return; - - link = dc_stream_get_link(stream); - if (!link || !link->replay_settings.replay_feature_enabled) - return; - - // Check if LIVE_CAPTURE_WITH_CVT bit is enabled in DalRegKey_ReplayOptimization - if (!link->replay_settings.config.replay_optimization.bits.LIVE_CAPTURE_WITH_CVT) - return; - - if (link->replay_settings.config.live_capture_with_cvt_activated != live_capture_with_cvt_activate) { - link->replay_settings.config.live_capture_with_cvt_activated = live_capture_with_cvt_activate; - mod_power_replay_set_general_cmd(mod_power, stream, - REPLAY_GENERAL_CMD_LIVE_CAPTURE_WITH_CVT, - live_capture_with_cvt_activate, 0); - } -} - -bool mod_power_set_replay_event(struct mod_power *mod_power, - struct dc_stream_state *stream, bool set_event, - enum replay_event event, bool wait_for_disable) -{ - struct core_power *core_power = NULL; - struct dc_link *link = NULL; - unsigned int stream_index = 0; - unsigned int active_replay_events = 0; - bool replay_active_request = false; - bool force_static = false; - uint32_t coasting_vtotal = 0; - bool current_timing_sync_status = false; - bool is_full_screen_video = false; - bool is_ultra_sleep_mode = false; - unsigned int sink_duration_us = 0; - bool low_rr_active = false; - uint16_t frame_skip_number = 0; - bool is_video_playback = false; - - if (!stream) - return false; - - if (mod_power == NULL) - return false; - - core_power = MOD_POWER_TO_CORE(mod_power); - - if (core_power->num_entities == 0) - return false; - - stream_index = map_index_from_stream(core_power, stream); - - if (set_event) - core_power->map[stream_index].replay_events |= event; - else - core_power->map[stream_index].replay_events &= ~event; - - link = dc_stream_get_link(stream); - if (!link || !link->replay_settings.replay_feature_enabled) - return false; - - if ((core_power->map[stream_index].replay_events & replay_event_disable_replay_while_switching_mux) != 0) - return false; - - if ((core_power->map[stream_index].replay_events & replay_event_os_override_hold) != 0) - return false; - - active_replay_events = core_power->map[stream_index].replay_events; - - current_timing_sync_status = - link->replay_settings.config.replay_timing_sync_supported; - - replay_active_request = mod_power_update_replay_active_status(active_replay_events, - link, &coasting_vtotal, &is_full_screen_video, &is_ultra_sleep_mode, &frame_skip_number, &is_video_playback); - - if (is_full_screen_video) - mod_power_replay_set_pseudo_vtotal(mod_power, stream, - link->replay_settings.low_rr_full_screen_video_pseudo_vtotal); - else - mod_power_replay_set_pseudo_vtotal(mod_power, stream, 0); - - //If timing_sync_status change, then re-enabled set timing_sync_supported value and re-enabled replay - if (current_timing_sync_status != link->replay_settings.config.replay_timing_sync_supported) - mod_power_replay_set_timing_sync_supported(mod_power, stream); - - if (link->replay_settings.config.low_rr_supported) { - sink_duration_us = - (unsigned int)(div_u64(((unsigned long long)(coasting_vtotal) - * 10000) * stream->timing.h_total, - stream->timing.pix_clk_100hz)); - low_rr_active = sink_duration_us < LOW_REFRESH_RATE_DURATION_US_UPPER_BOUND ? false : true; - if (low_rr_active != link->replay_settings.config.low_rr_activated) { - mod_power_set_low_rr_activate(mod_power, stream, low_rr_active); - link->replay_settings.config.low_rr_activated = low_rr_active; - } - } - - // The function return fail when - // 1. DMUB function is not support (for backward compatible). - // 2. active_replay_events or coasting_vtotal is not updated in the same time - if (!mod_power_replay_set_power_opt_and_coasting_vtotal(mod_power, - stream, active_replay_events, coasting_vtotal, is_ultra_sleep_mode, frame_skip_number)) { - if (!mod_power_replay_set_power_opt(mod_power, stream, active_replay_events, is_ultra_sleep_mode)) - return false; - - if (!mod_power_replay_set_coasting_vtotal(mod_power, stream, coasting_vtotal, frame_skip_number)) - return false; - } - - mod_power_set_live_capture_with_cvt_activate(mod_power, stream, is_video_playback); - - mod_power_update_error_status(mod_power, stream); - - // If Replay is going to be enable (No matter is disable -> enable or enable -> enable), we don't need to wait. - // If Replay is going to be disable - // if disable -> disable - // -> Replay DMUB state should be state 0. - // So no matter wait_for_disable is true or not, it should makes no difference. - // if enable -> disable -> We should wait if wait_for_disable is true. - if (replay_active_request) - wait_for_disable = false; - - if (!mod_power_set_replay_active(stream, replay_active_request, wait_for_disable, force_static)) - return false; - - return true; -} - -bool mod_power_get_replay_active_status(const struct dc_stream_state *stream, - bool *replay_active) -{ - const struct dc_link *link = NULL; - - if (!stream) - return false; - - link = dc_stream_get_link(stream); - *replay_active = link->replay_settings.replay_allow_active; - - return true; -} - -void mod_power_replay_residency(const struct dc_stream_state *stream, - unsigned int *residency, const bool is_start, const bool is_alpm) -{ - const struct dc_link *link = NULL; - enum pr_residency_mode mode; - - if (!stream) - return; - - link = dc_stream_get_link(stream); - - if (is_alpm) - mode = PR_RESIDENCY_MODE_ALPM; - else - mode = PR_RESIDENCY_MODE_PHY; - - if (link && link->dc && link->dc->link_srv) - link->dc->link_srv->edp_replay_residency(link, residency, is_start, mode); -} - -bool mod_power_replay_set_power_opt_and_coasting_vtotal(struct mod_power *mod_power, - const struct dc_stream_state *stream, unsigned int active_replay_events, uint32_t coasting_vtotal, - bool is_ultra_sleep_mode, uint16_t frame_skip_number) -{ - struct core_power *core_power = NULL; - struct dc_link *link = NULL; - unsigned int power_opt = 0; - - if (!stream) - return false; - - if (mod_power == NULL) - return false; - - core_power = MOD_POWER_TO_CORE(mod_power); - - if (core_power->num_entities == 0) - return false; - - link = dc_stream_get_link(stream); - - if (!link || !link->replay_settings.replay_feature_enabled) - return false; - - power_opt = mod_power_replay_setup_power_opt(link, active_replay_events, is_ultra_sleep_mode); - - return link->dc->link_srv->edp_set_replay_power_opt_and_coasting_vtotal(link, &power_opt, coasting_vtotal, frame_skip_number); -} - - - - - diff --git a/drivers/gpu/drm/amd/display/modules/power/power_abm.c b/drivers/gpu/drm/amd/display/modules/power/power_abm.c index c41ace406519..b94dbb9e64a7 100644 --- a/drivers/gpu/drm/amd/display/modules/power/power_abm.c +++ b/drivers/gpu/drm/amd/display/modules/power/power_abm.c @@ -233,102 +233,6 @@ struct iram_table_v_2_2 { #define MOD_POWER_MAX_CONCURRENT_STREAMS 32 #define SMOOTH_BRIGHTNESS_ADJUSTMENT_TIME_IN_MS 500 - - -struct backlight_state { - /* HW uses u16.16 format for backlight PWM */ - unsigned int backlight_pwm; - /* DM may call power module to set backlight - * targeting percent brightness - */ - unsigned int backlight_millipercent; - /* DM may call power module to set backlight based on an explicit - * nits value. - */ - unsigned int backlight_millinit; - unsigned int frame_ramp; - bool smooth_brightness_enabled; - bool isHDR; -}; -struct power_entity { - struct dc_stream_state *stream; - struct psr_caps *caps; - struct mod_power_psr_context *psr_context; - - /*PSR cached properties*/ - bool psr_enabled; - unsigned int psr_events; - unsigned int psr_power_opt; - unsigned int replay_events; -}; - -struct pwr_backlight_properties { - bool use_nits_based_brightness; - bool disable_fractional_pwm; - - unsigned int min_abm_backlight; - unsigned int num_backlight_levels; - - bool backlight_ramping_override; - unsigned int backlight_ramping_reduction; - unsigned int backlight_ramping_start; - - /* Backlight cached properties */ - unsigned int ac_backlight_percent; - unsigned int dc_backlight_percent; - - /* backlight LUT stored in HW u16.16 format*/ - unsigned int *backlight_lut; - unsigned int min_backlight_pwm; - unsigned int max_backlight_pwm; - unsigned int backlight_range; - - /* Describes the panel's min and max luminance in millinits measured - * on full white screen, in min and max backlight settings. - */ - unsigned int min_brightness_millinits; - unsigned int max_brightness_millinits; - unsigned int nits_range; - - bool backlight_caps_valid; - bool use_custom_backlight_caps; - unsigned int custom_backlight_caps_config_no; - bool use_linear_backlight_curve; -}; - -struct dmcu_varibright_cached_properties { - unsigned int varibright_config_setting; - unsigned int varibright_level; - unsigned int varibright_hw_level; - unsigned int def_varibright_level; - bool varibright_user_enable; - bool varibright_active; -}; - -struct core_power { - struct mod_power mod_public; - struct dc *dc; - struct power_entity *map; - struct dmcu_varibright_cached_properties varibright_prop; - struct pwr_backlight_properties bl_prop[MAX_NUM_EDP]; - struct backlight_state bl_state[MAX_NUM_EDP]; - unsigned int edp_num; - - bool psr_smu_optimizations_support; - bool multi_disp_optimizations_support; - - unsigned int num_entities; -}; - -union dmcu_abm_set_bl_params { - struct { - unsigned int gradual_change : 1; /* [0:0] */ - unsigned int reserved : 15; /* [15:1] */ - unsigned int frame_ramp : 16; /* [31:16] */ - } bits; - unsigned int u32All; -}; - /* If system or panel does not report some sort of brightness percent to nits * mapping, we will use following default values so backlight control using * nits based interfaces will still work, but might not describe panel @@ -341,8 +245,6 @@ union dmcu_abm_set_bl_params { #define MOD_POWER_TO_CORE(mod_power)\ container_of(mod_power, struct core_power, mod_public) - - static uint16_t backlight_8_to_16(unsigned int backlight_8bit) { return (uint16_t)(backlight_8bit * 0x101); diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c index 1046fc35f8f9..bf0c5901b4ee 100644 --- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c +++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c @@ -33,257 +33,7 @@ #define bswap16_based_on_endian(big_endian, value) \ ((big_endian) ? cpu_to_be16(value) : cpu_to_le16(value)) -/* - * is_psr_su_specific_panel() - check if sink is AMD vendor-specific PSR-SU - * supported eDP device. - * - * @link: dc link pointer - * - * Return: true if AMDGPU vendor specific PSR-SU eDP panel - */ -bool is_psr_su_specific_panel(struct dc_link *link) -{ - bool isPSRSUSupported = false; - struct dpcd_caps *dpcd_caps = &link->dpcd_caps; - - if (dpcd_caps->edp_rev >= DP_EDP_14) { - if (dpcd_caps->psr_info.psr_version >= DP_PSR2_WITH_Y_COORD_ET_SUPPORTED) - isPSRSUSupported = true; - /* - * Some panels will report PSR capabilities over additional DPCD bits. - * Such panels are approved despite reporting only PSR v3, as long as - * the additional bits are reported. - */ - if (dpcd_caps->sink_dev_id == DP_BRANCH_DEVICE_ID_001CF8) { - /* - * This is the temporary workaround to disable PSRSU when system turned on - * DSC function on the sepcific sink. - */ - if (dpcd_caps->psr_info.psr_version < DP_PSR2_WITH_Y_COORD_IS_SUPPORTED) - isPSRSUSupported = false; - else if (dpcd_caps->dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT && - ((dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x08) || - (dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x07))) - isPSRSUSupported = false; - else if (dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x03) - isPSRSUSupported = false; - else if (dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x01) - isPSRSUSupported = false; - else if (dpcd_caps->psr_info.force_psrsu_cap == 0x1) - isPSRSUSupported = true; - } - } - - return isPSRSUSupported; -} - -/** - * mod_power_calc_psr_configs() - calculate/update generic psr configuration fields. - * @psr_config: [output], psr configuration structure to be updated - * @link: [input] dc link pointer - * @stream: [input] dc stream state pointer - * - * calculate and update the psr configuration fields that are not DM specific, i.e. such - * fields which are based on DPCD caps or timing information. To setup PSR in DMUB FW, - * this helper is assumed to be called before the call of the DC helper dc_link_setup_psr(). - * - * PSR config fields to be updated within the helper: - * - psr_rfb_setup_time - * - psr_sdp_transmit_line_num_deadline - * - line_time_in_us - * - su_y_granularity - * - su_granularity_required - * - psr_frame_capture_indication_req - * - psr_exit_link_training_required - * - * PSR config fields that are DM specific and NOT updated within the helper: - * - allow_smu_optimizations - * - allow_multi_disp_optimizations - */ -void mod_power_calc_psr_configs(struct psr_config *psr_config, - struct dc_link *link, - const struct dc_stream_state *stream) -{ - unsigned int num_vblank_lines = 0; - unsigned int vblank_time_in_us = 0; - unsigned int sdp_tx_deadline_in_us = 0; - unsigned int line_time_in_us = 0; - struct dpcd_caps *dpcd_caps = &link->dpcd_caps; - const int psr_setup_time_step_in_us = 55; /* refer to eDP spec DPCD 0x071h */ - - /* timing parameters */ - num_vblank_lines = stream->timing.v_total - - stream->timing.v_addressable - - stream->timing.v_border_top - - stream->timing.v_border_bottom; - - vblank_time_in_us = (stream->timing.h_total * num_vblank_lines * 1000) / (stream->timing.pix_clk_100hz / 10); - - line_time_in_us = ((stream->timing.h_total * 1000) / (stream->timing.pix_clk_100hz / 10)) + 1; - - /** - * psr configuration fields - * - * as per eDP 1.5 pg. 377 of 459, DPCD 0x071h bits [3:1], psr setup time bits interpreted as below - * 000b <--> 330 us (default) - * 001b <--> 275 us - * 010b <--> 220 us - * 011b <--> 165 us - * 100b <--> 110 us - * 101b <--> 055 us - * 110b <--> 000 us - */ - psr_config->psr_rfb_setup_time = - (6 - dpcd_caps->psr_info.psr_dpcd_caps.bits.PSR_SETUP_TIME) * psr_setup_time_step_in_us; - - if (psr_config->psr_rfb_setup_time > vblank_time_in_us) { - link->psr_settings.psr_frame_capture_indication_req = true; - link->psr_settings.psr_sdp_transmit_line_num_deadline = num_vblank_lines; - } else { - sdp_tx_deadline_in_us = vblank_time_in_us - psr_config->psr_rfb_setup_time; - - /* Set the last possible line SDP may be transmitted without violating the RFB setup time */ - link->psr_settings.psr_frame_capture_indication_req = false; - link->psr_settings.psr_sdp_transmit_line_num_deadline = sdp_tx_deadline_in_us / line_time_in_us; - } - - psr_config->psr_sdp_transmit_line_num_deadline = link->psr_settings.psr_sdp_transmit_line_num_deadline; - psr_config->line_time_in_us = line_time_in_us; - psr_config->su_y_granularity = dpcd_caps->psr_info.psr2_su_y_granularity_cap; - psr_config->su_granularity_required = dpcd_caps->psr_info.psr_dpcd_caps.bits.SU_GRANULARITY_REQUIRED; - psr_config->psr_frame_capture_indication_req = link->psr_settings.psr_frame_capture_indication_req; - psr_config->psr_exit_link_training_required = - !link->dpcd_caps.psr_info.psr_dpcd_caps.bits.LINK_TRAINING_ON_EXIT_NOT_REQUIRED; -} - -void init_replay_config(struct dc_link *link, struct replay_config *pr_config) -{ - link->replay_settings.config = *pr_config; -} - bool mod_power_only_edp(const struct dc_state *context, const struct dc_stream_state *stream) { return context && context->stream_count == 1 && dc_is_embedded_signal(stream->signal); } - -bool psr_su_set_dsc_slice_height(struct dc *dc, struct dc_link *link, - struct dc_stream_state *stream, - struct psr_config *config) -{ - uint32_t pic_height; - uint32_t slice_height; - - config->dsc_slice_height = 0; - if (!(link->connector_signal & SIGNAL_TYPE_EDP) || - !dc->caps.edp_dsc_support || - link->panel_config.dsc.disable_dsc_edp || - !link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT || - !stream->timing.dsc_cfg.num_slices_v) - return true; - - pic_height = stream->timing.v_addressable + - stream->timing.v_border_top + stream->timing.v_border_bottom; - - if (stream->timing.dsc_cfg.num_slices_v == 0) - return false; - - slice_height = pic_height / stream->timing.dsc_cfg.num_slices_v; - config->dsc_slice_height = (uint16_t)slice_height; - - if (slice_height) { - if (config->su_y_granularity && - (slice_height % config->su_y_granularity)) { - ASSERT(0); - return false; - } - } - - return true; -} - -void set_replay_frame_skip_number(struct dc_link *link, - enum replay_coasting_vtotal_type type, - uint32_t coasting_vtotal_refresh_rate_uhz, - uint32_t flicker_free_refresh_rate_uhz, - bool is_defer) -{ - uint32_t *frame_skip_number_array = NULL; - uint32_t frame_skip_number = 0; - - if (link == NULL) - return; - - if (false == link->replay_settings.config.frame_skip_supported) - return; - - if (flicker_free_refresh_rate_uhz == 0 || coasting_vtotal_refresh_rate_uhz == 0) - return; - - if (is_defer) - frame_skip_number_array = link->replay_settings.defer_frame_skip_number_table; - else - frame_skip_number_array = link->replay_settings.frame_skip_number_table; - - if (frame_skip_number_array == NULL) - return; - - frame_skip_number = (coasting_vtotal_refresh_rate_uhz + 500000) / flicker_free_refresh_rate_uhz; - - if (frame_skip_number >= 1) - frame_skip_number_array[type] = frame_skip_number - 1; - else - frame_skip_number_array[type] = 0; -} - -void set_replay_defer_update_coasting_vtotal(struct dc_link *link, - enum replay_coasting_vtotal_type type, - uint32_t vtotal) -{ - link->replay_settings.defer_update_coasting_vtotal_table[type] = vtotal; -} - -void update_replay_coasting_vtotal_from_defer(struct dc_link *link, - enum replay_coasting_vtotal_type type) -{ - link->replay_settings.coasting_vtotal_table[type] = - link->replay_settings.defer_update_coasting_vtotal_table[type]; - link->replay_settings.frame_skip_number_table[type] = - link->replay_settings.defer_frame_skip_number_table[type]; -} - -void set_replay_coasting_vtotal(struct dc_link *link, - enum replay_coasting_vtotal_type type, - uint32_t vtotal) -{ - link->replay_settings.coasting_vtotal_table[type] = vtotal; -} - -void set_replay_low_rr_full_screen_video_src_vtotal(struct dc_link *link, uint16_t vtotal) -{ - link->replay_settings.low_rr_full_screen_video_pseudo_vtotal = vtotal; -} - -void calculate_replay_link_off_frame_count(struct dc_link *link, - uint16_t vtotal, uint16_t htotal) -{ - uint32_t max_link_off_frame_count = 0; - uint16_t max_deviation_line = 0, pixel_deviation_per_line = 0; - - if (!link || link->replay_settings.config.replay_version != DC_FREESYNC_REPLAY) - return; - - max_deviation_line = link->dpcd_caps.pr_info.max_deviation_line; - pixel_deviation_per_line = link->dpcd_caps.pr_info.pixel_deviation_per_line; - - if (htotal != 0 && vtotal != 0 && pixel_deviation_per_line != 0) - max_link_off_frame_count = htotal * max_deviation_line / (pixel_deviation_per_line * vtotal); - else - ASSERT(0); - - link->replay_settings.link_off_frame_count = max_link_off_frame_count; -} - -void reset_replay_dsync_error_count(struct dc_link *link) -{ - link->replay_settings.replay_desync_error_fail_count = 0; -} diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.h b/drivers/gpu/drm/amd/display/modules/power/power_helpers.h index 94d2521355ce..600da3e33126 100644 --- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.h +++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.h @@ -47,7 +47,99 @@ struct dmcu_iram_parameters { unsigned int min_abm_backlight; unsigned int set; }; +struct backlight_state { + /* HW uses u16.16 format for backlight PWM */ + unsigned int backlight_pwm; + /* DM may call power module to set backlight + * targeting percent brightness + */ + unsigned int backlight_millipercent; + /* DM may call power module to set backlight based on an explicit + * nits value. + */ + unsigned int backlight_millinit; + unsigned int frame_ramp; + bool smooth_brightness_enabled; + bool isHDR; +}; +struct power_entity { + struct dc_stream_state *stream; + struct psr_caps *caps; + struct mod_power_psr_context *psr_context; + /*PSR cached properties*/ + bool psr_enabled; + unsigned int psr_events; + unsigned int psr_power_opt; + unsigned int replay_events; +}; + +struct pwr_backlight_properties { + bool use_nits_based_brightness; + bool disable_fractional_pwm; + + unsigned int min_abm_backlight; + unsigned int num_backlight_levels; + + bool backlight_ramping_override; + unsigned int backlight_ramping_reduction; + unsigned int backlight_ramping_start; + + /* Backlight cached properties */ + unsigned int ac_backlight_percent; + unsigned int dc_backlight_percent; + + /* backlight LUT stored in HW u16.16 format*/ + unsigned int *backlight_lut; + unsigned int min_backlight_pwm; + unsigned int max_backlight_pwm; + unsigned int backlight_range; + + /* Describes the panel's min and max luminance in millinits measured + * on full white screen, in min and max backlight settings. + */ + unsigned int min_brightness_millinits; + unsigned int max_brightness_millinits; + unsigned int nits_range; + + bool backlight_caps_valid; + bool use_custom_backlight_caps; + unsigned int custom_backlight_caps_config_no; + bool use_linear_backlight_curve; +}; + +struct dmcu_varibright_cached_properties { + unsigned int varibright_config_setting; + unsigned int varibright_level; + unsigned int varibright_hw_level; + unsigned int def_varibright_level; + bool varibright_user_enable; + bool varibright_active; +}; + +struct core_power { + struct mod_power mod_public; + struct dc *dc; + struct power_entity *map; + struct dmcu_varibright_cached_properties varibright_prop; + struct pwr_backlight_properties bl_prop[MAX_NUM_EDP]; + struct backlight_state bl_state[MAX_NUM_EDP]; + unsigned int edp_num; + + bool psr_smu_optimizations_support; + bool multi_disp_optimizations_support; + + unsigned int num_entities; +}; + +union dmcu_abm_set_bl_params { + struct { + unsigned int gradual_change : 1; /* [0:0] */ + unsigned int reserved : 15; /* [15:1] */ + unsigned int frame_ramp : 16; /* [31:16] */ + } bits; + unsigned int u32All; +}; bool dmcu_load_iram(struct dmcu *dmcu, struct dmcu_iram_parameters params); bool dmub_init_abm_config(struct resource_pool *res_pool, @@ -87,4 +179,33 @@ bool fill_custom_backlight_caps(unsigned int config_no, void reset_replay_dsync_error_count(struct dc_link *link); void change_replay_to_psr(struct dc_link *link); void change_psr_to_replay(struct dc_link *link); +void initialize_backlight_caps(struct core_power *core_power, unsigned int inst); +unsigned int backlight_millipercent_to_pwm( + struct core_power *core_power, unsigned int millipercent, unsigned int inst); +unsigned int backlight_millipercent_to_millinit( + struct core_power *core_power, unsigned int millipercent, unsigned int inst); +void fill_backlight_level_params(struct core_power *core_power, + struct set_backlight_level_params *backlight_level_params, + int panel_inst, uint8_t aux_inst, unsigned int backlight_pwm, + enum backlight_control_type backlight_control_type, + unsigned int backlight_millinit, unsigned int transition_time_millisec, + bool is_hdr); +bool mod_power_hw_init_backlight(struct mod_power *mod_power); +void mod_power_update_backlight_on_mode_change( + struct core_power *core_power, + struct dc_link *link, + unsigned int panel_inst, + uint8_t aux_inst, + bool is_hdr); +unsigned int map_index_from_stream(struct core_power *core_power, + const struct dc_stream_state *stream); +bool mod_power_psr_notify_mode_change(struct mod_power *mod_power, + const struct dc_stream_state *stream, + struct dc_link *link, + unsigned int stream_index); +void mod_power_replay_notify_mode_change(struct mod_power *mod_power, + struct dc *dc, + struct dc_link *link, + const struct dc_stream_state *stream, + unsigned int stream_index); #endif /* MODULES_POWER_POWER_HELPERS_H_ */ diff --git a/drivers/gpu/drm/amd/display/modules/power/power_psr.c b/drivers/gpu/drm/amd/display/modules/power/power_psr.c new file mode 100644 index 000000000000..5ecb570c204e --- /dev/null +++ b/drivers/gpu/drm/amd/display/modules/power/power_psr.c @@ -0,0 +1,665 @@ +// SPDX-License-Identifier: MIT +// +// Copyright 2026 Advanced Micro Devices, Inc. + +#include "dm_services.h" +#include "dc.h" +#include "mod_power.h" +#include "core_types.h" +#include "dmcu.h" +#include "abm.h" +#include "power_helpers.h" +#include "dce/dmub_psr.h" +#include "dal_asic_id.h" +#include "link_service.h" +#include + +#define DC_TRACE_LEVEL_MESSAGE(...) /* do nothing */ +#define DC_TRACE_LEVEL_MESSAGEP(...) /* do nothing */ +#include "dc/inc/hw/dmcu.h" +#include "dc/inc/hw/abm.h" +#include "dmub_cmd.h" + +#define MOD_POWER_TO_CORE(mod_power)\ + container_of(mod_power, struct core_power, mod_public) + +static unsigned int calc_psr_num_static_frames(unsigned int vsync_rate_hz) +{ + /* Initialize fail-safe to 2 static frames. */ + unsigned int num_frames_static = 2; + + /* Calculate number of frames such that at least 30 ms has passed. + * Round up to ensure the static period is not shorter than 30 ms. + */ + if (vsync_rate_hz != 0) + num_frames_static = DIV_ROUND_UP(30000 * vsync_rate_hz, 1000000); + + return num_frames_static; +} + +bool mod_power_psr_notify_mode_change(struct mod_power *mod_power, + const struct dc_stream_state *stream, + struct dc_link *link, + unsigned int stream_index) +{ + struct core_power *core_power = NULL; + struct dc *dc = NULL; + struct psr_config psr_config = {0}; + struct psr_context psr_context = {0}; + int active_psr_events = 0; + + if ((mod_power == NULL) || (stream == NULL) || (link == NULL)) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + dc = core_power->dc; + + // NO num_entities check here - already validated by caller + // stream_index is passed as validated parameter + active_psr_events = core_power->map[stream_index].psr_events; + + /* Calculate PSR configurations */ + mod_power_calc_psr_configs(&psr_config, link, stream); + + psr_config.psr_exit_link_training_required = + core_power->map[stream_index].caps->psr_exit_link_training_required; + if (dc->ctx->asic_id.chip_family >= AMDGPU_FAMILY_GC_11_0_1) + psr_config.allow_smu_optimizations = + core_power->psr_smu_optimizations_support && dc_is_embedded_signal(stream->signal); + else + psr_config.allow_smu_optimizations = + core_power->psr_smu_optimizations_support && + mod_power_only_edp(dc->current_state, stream); + + psr_config.allow_multi_disp_optimizations = core_power->multi_disp_optimizations_support; + + psr_config.rate_control_caps = core_power->map[stream_index].caps->rate_control_caps; + + if (active_psr_events & psr_event_os_request_force_ffu) + psr_config.os_request_force_ffu = true; + + /* + * DSC support: + * DSC slice height value must be 'mod' by su_y_granularity. + * According to Panel Vendor, there might be varied conditions to fulfill. + * Right now, DSC slice height value must be multiple of su_y_granularity. + * + * The value of DSC slice height is determined in DSC Driver but it does not + * propagated out here, so we need to calculate it as below 'slice_height'. + */ + psr_su_set_dsc_slice_height(dc, link, + (struct dc_stream_state *) stream, + &psr_config); + + dc_link_setup_psr(link, stream, &psr_config, &psr_context); + + return true; +} + +static void mod_power_psr_set_power_opt(struct mod_power *mod_power, + struct dc_stream_state *stream, + unsigned int active_psr_events, + bool psr_enable_request) +{ + (void)psr_enable_request; + struct core_power *core_power = NULL; + struct dc_link *link = NULL; + unsigned int stream_index = 0; + unsigned int power_opt = 0; + + if (!stream) + return; + + core_power = MOD_POWER_TO_CORE(mod_power); + stream_index = map_index_from_stream(core_power, stream); + if (!core_power->map[stream_index].caps->psr_version) + return; + + link = dc_stream_get_link(stream); + + if (active_psr_events == 0) { + /* Static Screen */ + power_opt |= (psr_power_opt_smu_opt_static_screen | psr_power_opt_z10_static_screen | + psr_power_opt_ds_disable_allow); + } + + /* psr_power_opt_flag is a configuration parameter into the module that determines + * which optimizations to enable during psr + */ + power_opt &= core_power->map[stream_index].caps->psr_power_opt_flag; + if (core_power->map[stream_index].psr_power_opt != power_opt) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_VERBOSE, + WPP_BIT_FLAG_Firmware_PsrState, + "mod_power set_power_opt: psr_power_opt=0x%04x, power_opt=0x%04x active_psr_events=0x%04x, psr_power_opt_flag=0x%04x", + core_power->map[stream_index].psr_power_opt, + power_opt, + active_psr_events, + core_power->map[stream_index].caps->psr_power_opt_flag); + dc_link_set_psr_allow_active(link, NULL, false, false, &power_opt); + core_power->map[stream_index].psr_power_opt = power_opt; + } +} + +static bool set_psr_enable(struct mod_power *mod_power, + struct dc_stream_state *stream, + bool psr_enable, + bool wait, + bool force_static) +{ + struct core_power *core_power = NULL; + enum dc_psr_state state = PSR_STATE0; + unsigned int retry_count; + const unsigned int max_retry = 1000; + struct dc_link *link = NULL; + + if (mod_power == NULL) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + + if (core_power->num_entities == 0) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + WPP_BIT_FLAG_Firmware_PsrState, + "set psr enable: ERROR: stream=%p num_entities=%u", + stream, + core_power->num_entities); + return false; + } + + if (psr_enable) { + unsigned int vsync_rate_hz; + struct dc_static_screen_params params = {0}; + + vsync_rate_hz = (unsigned int)div_u64(div_u64(( + stream->timing.pix_clk_100hz * 100), + stream->timing.v_total), + stream->timing.h_total); + + params.triggers.cursor_update = true; + params.triggers.overlay_update = true; + params.triggers.surface_update = true; + params.num_frames = calc_psr_num_static_frames(vsync_rate_hz); + + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + WPP_BIT_FLAG_Firmware_PsrState, + "set psr enable: CALCS: pix_clk_100hz=%u v_total=%u h_total=%u vsync_rate_hz=%u num_frames=%u", + stream->timing.pix_clk_100hz, + stream->timing.v_total, + stream->timing.h_total, + vsync_rate_hz, + params.num_frames); + + dc_stream_set_static_screen_params(core_power->dc, + &stream, 1, + ¶ms); + } + + link = dc_stream_get_link(stream); + + if (!dc_link_set_psr_allow_active(link, &psr_enable, false, force_static, NULL)) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + WPP_BIT_FLAG_Firmware_PsrState, + "set psr enable: ERROR: stream=%p link=%p psr_enable=%d", + stream, + link, + psr_enable); + return false; + } + + if (wait == true) { + + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + WPP_BIT_FLAG_Firmware_PsrState, + "set psr enable: BEGIN WAIT: psr_enable=%d", + (int)psr_enable); + + for (retry_count = 0; retry_count <= max_retry; retry_count++) { + dc_link_get_psr_state(link, &state); + if (psr_enable) { + if (state != PSR_STATE0 && + (!force_static || state == PSR_STATE3)) + break; + } else { + if (state == PSR_STATE0) + break; + } + udelay(500); + } + + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + WPP_BIT_FLAG_Firmware_PsrState, + "set psr enable: END WAIT: psr_enable=%d", + (int)psr_enable); + + /* assert if max retry hit */ + if (retry_count >= max_retry) { + ASSERT(0); + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + WPP_BIT_FLAG_Firmware_PsrState, + "set psr enable: ERROR: retry_count=%u: Unexpectedly long wait for PSR state change.", + retry_count); + } + } else { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, + WPP_BIT_FLAG_Firmware_PsrState, + "set psr enable: PSR state change initiated (wait=false): psr_enable=%d", + (int)psr_enable); + } + + return true; +} + +bool mod_power_get_psr_event(struct mod_power *mod_power, + struct dc_stream_state *stream, + unsigned int *active_psr_events) +{ + struct core_power *core_power = NULL; + unsigned int stream_index = 0; + + if (mod_power == NULL) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + + if (core_power->num_entities == 0) + return false; + + stream_index = map_index_from_stream(core_power, stream); + + if (!core_power->map[stream_index].caps->psr_version) + return false; + + *active_psr_events = core_power->map[stream_index].psr_events; + + return true; +} + +bool mod_power_set_psr_event(struct mod_power *mod_power, + struct dc_stream_state *stream, bool set_event, + enum psr_event event, bool wait) +{ + struct core_power *core_power = NULL; + unsigned int stream_index = 0; + unsigned int active_psr_events = 0; + bool psr_enable_request = false; + bool force_static = false; + + if (mod_power == NULL || stream == NULL) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + stream_index = map_index_from_stream(core_power, stream); + + if (core_power->num_entities == 0) { + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, + WPP_BIT_FLAG_Firmware_PsrState, + "mod_power set_psr_event: ERROR: stream=%p event=%d num_entities=%u", + stream, + (int)event, + core_power->num_entities); + return false; + } + + if (!core_power->map[stream_index].caps->psr_version) + return false; + + if (set_event) + core_power->map[stream_index].psr_events |= event; + else + core_power->map[stream_index].psr_events &= ~event; + + active_psr_events = core_power->map[stream_index].psr_events; + + // ignore other events when we're in forced psr enabled state + if (active_psr_events & psr_event_dynamic_display_switch && + event != psr_event_dynamic_display_switch) + return false; + + // ignore other events when we're in forced psr enabled state + if (active_psr_events & psr_event_os_override_hold && + event != psr_event_os_override_hold) + return false; + + // ignore other events when we're in forced psr enabled state + // dds events need to be processed while in dynamic_link_rate_control + if (active_psr_events & psr_event_dynamic_link_rate_control && + event != psr_event_dynamic_link_rate_control && + event != psr_event_dds_defer_stream_enable && + event != psr_event_dynamic_display_switch) + return false; + + if (active_psr_events & (psr_event_test_harness_disable_psr | psr_event_os_request_disable)) + psr_enable_request = false; + else if (active_psr_events & psr_event_pause) + psr_enable_request = false; + else if (active_psr_events & psr_event_test_harness_enable_psr) + psr_enable_request = true; + else if (active_psr_events & psr_event_dynamic_display_switch) { + psr_enable_request = true; + force_static = true; + } else if (active_psr_events & psr_event_dynamic_link_rate_control) { + psr_enable_request = true; + force_static = true; + } else if (active_psr_events & psr_event_edp_panel_off_disable_psr) + psr_enable_request = false; + else if (active_psr_events & (psr_event_hw_programming | + psr_event_defer_enable | + psr_event_dds_defer_stream_enable | + psr_event_vrr_transition | + psr_event_immediate_flip)) + psr_enable_request = false; + else if (active_psr_events & psr_event_big_screen_video) + psr_enable_request = true; + else if (active_psr_events & psr_event_full_screen) + psr_enable_request = false; + else if (active_psr_events & psr_event_mpo_video_selective_update) + psr_enable_request = true; + else if (active_psr_events & psr_event_vsync) + psr_enable_request = false; + else if (active_psr_events & psr_event_crc_window_active) + psr_enable_request = false; + else + psr_enable_request = true; + + DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_VERBOSE, + WPP_BIT_FLAG_Firmware_PsrState, + "mod_power set_psr_event: before: psr_enabled=%d -> request: set_event=%d event=0x%04x -> result: psr_events=0x%04x psr_enable_request=%d", + (int)core_power->map[stream_index].psr_enabled, + (int)set_event, + (unsigned int)event, + (unsigned int)core_power->map[stream_index].psr_events, + (int)psr_enable_request); + mod_power_psr_set_power_opt(mod_power, stream, active_psr_events, psr_enable_request); + + if (core_power->map[stream_index].psr_enabled != psr_enable_request || force_static) { + if (set_psr_enable(mod_power, stream, psr_enable_request, wait, force_static)) + core_power->map[stream_index].psr_enabled = psr_enable_request; + } + + return true; +} + +bool mod_power_get_psr_state(struct mod_power *mod_power, + const struct dc_stream_state *stream, + enum dc_psr_state *state) +{ + struct core_power *core_power = NULL; + const struct dc_link *link = NULL; + + if (!stream) + return false; + + if (mod_power == NULL) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + + if (core_power->num_entities == 0) + return false; + + link = dc_stream_get_link(stream); + return dc_link_get_psr_state(link, state); +} + +bool mod_power_get_psr_enabled_status(struct mod_power *mod_power, + const struct dc_stream_state *stream, + bool *psr_enabled) +{ + struct core_power *core_power = NULL; + unsigned int stream_index = 0; + + if (mod_power == NULL) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + + if (core_power->num_entities == 0) + return false; + + stream_index = map_index_from_stream(core_power, stream); + + if (!core_power->map[stream_index].caps->psr_version) + return false; + + *psr_enabled = core_power->map[stream_index].psr_enabled; + + return true; +} + +void mod_power_psr_residency(struct mod_power *mod_power, + const struct dc_stream_state *stream, + unsigned int *residency, + const uint8_t mode) +{ + struct core_power *core_power = NULL; + const struct dc_link *link = NULL; + + if (!stream) + return; + + if (mod_power == NULL) + return; + + core_power = MOD_POWER_TO_CORE(mod_power); + + if (core_power->num_entities == 0) + return; + + link = dc_stream_get_link(stream); + + if (link != NULL) + link->dc->link_srv->edp_get_psr_residency(link, residency, mode); +} +bool mod_power_psr_get_active_psr_events(struct mod_power *mod_power, + const struct dc_stream_state *stream, unsigned int *active_psr_events) +{ + struct core_power *core_power = NULL; + unsigned int stream_index = 0; + + if (!stream) + return false; + + if (mod_power == NULL) + return false; + + if (active_psr_events == NULL) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + + if (core_power->num_entities == 0) + return false; + + stream_index = map_index_from_stream(core_power, stream); + + *active_psr_events = core_power->map[stream_index].psr_events; + return true; +} + +bool mod_power_psr_set_sink_vtotal_in_psr_active(struct mod_power *mod_power, + const struct dc_stream_state *stream, + uint16_t psr_vtotal_idle, + uint16_t psr_vtotal_su) +{ + struct core_power *core_power = NULL; + unsigned int stream_index = 0; + const struct dc_link *link = NULL; + + if (!stream) + return false; + + if (mod_power == NULL) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + + if (core_power->num_entities == 0) + return false; + + stream_index = map_index_from_stream(core_power, stream); + + if (!core_power->map[stream_index].caps->psr_version) + return false; + + link = dc_stream_get_link(stream); + + return link->dc->link_srv->edp_set_sink_vtotal_in_psr_active( + link, psr_vtotal_idle, psr_vtotal_su); +} +/* + * is_psr_su_specific_panel() - check if sink is AMD vendor-specific PSR-SU + * supported eDP device. + * + * @link: dc link pointer + * + * Return: true if AMDGPU vendor specific PSR-SU eDP panel + */ +bool is_psr_su_specific_panel(struct dc_link *link) +{ + bool isPSRSUSupported = false; + struct dpcd_caps *dpcd_caps = &link->dpcd_caps; + + if (dpcd_caps->edp_rev >= DP_EDP_14) { + if (dpcd_caps->psr_info.psr_version >= DP_PSR2_WITH_Y_COORD_ET_SUPPORTED) + isPSRSUSupported = true; + /* + * Some panels will report PSR capabilities over additional DPCD bits. + * Such panels are approved despite reporting only PSR v3, as long as + * the additional bits are reported. + */ + if (dpcd_caps->sink_dev_id == DP_BRANCH_DEVICE_ID_001CF8) { + /* + * This is the temporary workaround to disable PSRSU when system turned on + * DSC function on the sepcific sink. + */ + if (dpcd_caps->psr_info.psr_version < DP_PSR2_WITH_Y_COORD_IS_SUPPORTED) + isPSRSUSupported = false; + else if (dpcd_caps->dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT && + ((dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x08) || + (dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x07))) + isPSRSUSupported = false; + else if (dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x03) + isPSRSUSupported = false; + else if (dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x01) + isPSRSUSupported = false; + else if (dpcd_caps->psr_info.force_psrsu_cap == 0x1) + isPSRSUSupported = true; + } + } + + return isPSRSUSupported; +} + +/** + * mod_power_calc_psr_configs() - calculate/update generic psr configuration fields. + * @psr_config: [output], psr configuration structure to be updated + * @link: [input] dc link pointer + * @stream: [input] dc stream state pointer + * + * calculate and update the psr configuration fields that are not DM specific, i.e. such + * fields which are based on DPCD caps or timing information. To setup PSR in DMUB FW, + * this helper is assumed to be called before the call of the DC helper dc_link_setup_psr(). + * + * PSR config fields to be updated within the helper: + * - psr_rfb_setup_time + * - psr_sdp_transmit_line_num_deadline + * - line_time_in_us + * - su_y_granularity + * - su_granularity_required + * - psr_frame_capture_indication_req + * - psr_exit_link_training_required + * + * PSR config fields that are DM specific and NOT updated within the helper: + * - allow_smu_optimizations + * - allow_multi_disp_optimizations + */ +void mod_power_calc_psr_configs(struct psr_config *psr_config, + struct dc_link *link, + const struct dc_stream_state *stream) +{ + unsigned int num_vblank_lines = 0; + unsigned int vblank_time_in_us = 0; + unsigned int sdp_tx_deadline_in_us = 0; + unsigned int line_time_in_us = 0; + struct dpcd_caps *dpcd_caps = &link->dpcd_caps; + const int psr_setup_time_step_in_us = 55; /* refer to eDP spec DPCD 0x071h */ + + /* timing parameters */ + num_vblank_lines = stream->timing.v_total - + stream->timing.v_addressable - + stream->timing.v_border_top - + stream->timing.v_border_bottom; + + vblank_time_in_us = (stream->timing.h_total * num_vblank_lines * 1000) / (stream->timing.pix_clk_100hz / 10); + + line_time_in_us = ((stream->timing.h_total * 1000) / (stream->timing.pix_clk_100hz / 10)) + 1; + + /** + * psr configuration fields + * + * as per eDP 1.5 pg. 377 of 459, DPCD 0x071h bits [3:1], psr setup time bits interpreted as below + * 000b <--> 330 us (default) + * 001b <--> 275 us + * 010b <--> 220 us + * 011b <--> 165 us + * 100b <--> 110 us + * 101b <--> 055 us + * 110b <--> 000 us + */ + psr_config->psr_rfb_setup_time = + (6 - dpcd_caps->psr_info.psr_dpcd_caps.bits.PSR_SETUP_TIME) * psr_setup_time_step_in_us; + + if (psr_config->psr_rfb_setup_time > vblank_time_in_us) { + link->psr_settings.psr_frame_capture_indication_req = true; + link->psr_settings.psr_sdp_transmit_line_num_deadline = num_vblank_lines; + } else { + sdp_tx_deadline_in_us = vblank_time_in_us - psr_config->psr_rfb_setup_time; + + /* Set the last possible line SDP may be transmitted without violating the RFB setup time */ + link->psr_settings.psr_frame_capture_indication_req = false; + link->psr_settings.psr_sdp_transmit_line_num_deadline = sdp_tx_deadline_in_us / line_time_in_us; + } + + psr_config->psr_sdp_transmit_line_num_deadline = link->psr_settings.psr_sdp_transmit_line_num_deadline; + psr_config->line_time_in_us = line_time_in_us; + psr_config->su_y_granularity = dpcd_caps->psr_info.psr2_su_y_granularity_cap; + psr_config->su_granularity_required = dpcd_caps->psr_info.psr_dpcd_caps.bits.SU_GRANULARITY_REQUIRED; + psr_config->psr_frame_capture_indication_req = link->psr_settings.psr_frame_capture_indication_req; + psr_config->psr_exit_link_training_required = + !link->dpcd_caps.psr_info.psr_dpcd_caps.bits.LINK_TRAINING_ON_EXIT_NOT_REQUIRED; +} + +bool psr_su_set_dsc_slice_height(struct dc *dc, struct dc_link *link, + struct dc_stream_state *stream, + struct psr_config *config) +{ + uint32_t pic_height; + uint32_t slice_height; + + config->dsc_slice_height = 0; + if (!(link->connector_signal & SIGNAL_TYPE_EDP) || + !dc->caps.edp_dsc_support || + link->panel_config.dsc.disable_dsc_edp || + !link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT || + !stream->timing.dsc_cfg.num_slices_v) + return true; + + pic_height = stream->timing.v_addressable + + stream->timing.v_border_top + stream->timing.v_border_bottom; + + if (stream->timing.dsc_cfg.num_slices_v == 0) + return false; + + slice_height = pic_height / stream->timing.dsc_cfg.num_slices_v; + config->dsc_slice_height = (uint16_t)slice_height; + + if (slice_height) { + if (config->su_y_granularity && + (slice_height % config->su_y_granularity)) { + ASSERT(0); + return false; + } + } + + return true; +} diff --git a/drivers/gpu/drm/amd/display/modules/power/power_replay.c b/drivers/gpu/drm/amd/display/modules/power/power_replay.c new file mode 100644 index 000000000000..983be9759e74 --- /dev/null +++ b/drivers/gpu/drm/amd/display/modules/power/power_replay.c @@ -0,0 +1,911 @@ +// SPDX-License-Identifier: MIT +// +// Copyright 2026 Advanced Micro Devices, Inc. + +#include "dm_services.h" +#include "dc.h" +#include "mod_power.h" +#include "core_types.h" +#include "dmcu.h" +#include "abm.h" +#include "power_helpers.h" +#include "dce/dmub_psr.h" +#include "dal_asic_id.h" +#include "link_service.h" +#include + +#define DC_TRACE_LEVEL_MESSAGE(...) /* do nothing */ +#define DC_TRACE_LEVEL_MESSAGEP(...) /* do nothing */ +#include "power_helpers.h" +#include "dc/inc/hw/dmcu.h" +#include "dc/inc/hw/abm.h" +#include "dc.h" +#include "core_types.h" +#include "dmub_cmd.h" + +#define MOD_POWER_TO_CORE(mod_power)\ + container_of(mod_power, struct core_power, mod_public) + +#define LOW_REFRESH_RATE_DURATION_US_UPPER_BOUND 25000 + +static bool mod_power_set_replay_active(struct dc_stream_state *stream, + bool replay_active, + bool wait, + bool force_static) +{ + uint64_t state; + unsigned int retry_count; + const unsigned int max_retry = 1000; + struct dc_link *link = NULL; + + if (!stream) + return false; + + link = dc_stream_get_link(stream); + + if (!link) + return false; + + if (!dc_link_set_replay_allow_active(link, &replay_active, false, force_static, NULL)) + return false; + + if (wait == true) { + + for (retry_count = 0; retry_count <= max_retry; retry_count++) { + dc_link_get_replay_state(link, &state); + if (replay_active) { + if (state != REPLAY_STATE_0 && + (!force_static || state == REPLAY_STATE_3)) + break; + } else { + if (state == REPLAY_STATE_0) + break; + } + udelay(500); + } + + /* assert if max retry hit */ + if (retry_count >= max_retry) + ASSERT(0); + } else { + /* To-do: Add trace log */ + } + + return true; +} + +static unsigned int mod_power_replay_setup_power_opt(struct dc_link *link, + unsigned int active_replay_events, bool is_ultra_sleep_mode) +{ + unsigned int power_opt = 0; + + if (is_ultra_sleep_mode) { + /* Static Screen */ + power_opt |= (replay_power_opt_smu_opt_static_screen | replay_power_opt_z10_static_screen); + } else if (active_replay_events & replay_event_test_harness_ultra_sleep) { + power_opt |= replay_power_opt_z10_static_screen; + } + + /* replay_power_opt_flag is a configuration parameter into the module that determines + * which optimizations to enable during replay + */ + power_opt &= link->replay_settings.config.replay_power_opt_supported; + + return power_opt; +} + +static bool mod_power_replay_set_power_opt(struct mod_power *mod_power, + struct dc_stream_state *stream, + unsigned int active_replay_events, + bool is_ultra_sleep_mode) +{ + (void)mod_power; + struct dc_link *link = NULL; + unsigned int power_opt = 0; + + if (!stream) + return false; + + link = dc_stream_get_link(stream); + + if (!link || !link->replay_settings.replay_feature_enabled) + return false; + + power_opt = mod_power_replay_setup_power_opt(link, active_replay_events, is_ultra_sleep_mode); + + if (!dc_link_set_replay_allow_active(link, NULL, false, false, &power_opt)) + return false; + + return true; +} + +bool mod_power_get_replay_event(struct mod_power *mod_power, + struct dc_stream_state *stream, + unsigned int *active_replay_events) +{ + struct core_power *core_power = NULL; + unsigned int stream_index = 0; + + if (mod_power == NULL) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + + if (core_power->num_entities == 0) + return false; + + stream_index = map_index_from_stream(core_power, stream); + + *active_replay_events = core_power->map[stream_index].replay_events; + + return true; +} + +static bool mod_power_update_replay_active_status(unsigned int active_replay_events, + struct dc_link *link, uint32_t *coasting_vtotal, bool *is_full_screen_video, + bool *is_ultra_sleep_mode, uint16_t *frame_skip_number, bool *is_video_playback) +{ + if (!link || !coasting_vtotal || !is_full_screen_video || !is_video_playback) + return false; + + // Check coasting_vtotal_table has been updated. + if (!link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_STATIC] + || !link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM]) + return false; + + unsigned int replay_enable_option = + link->replay_settings.config.replay_enable_option; + + /* TODO: To support test harness and DDS event */ + + *coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM]; + ASSERT(link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM] <= 0xFFFF); + *frame_skip_number = (uint16_t)link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM]; + + link->replay_settings.config.replay_timing_sync_supported = false; + + *is_full_screen_video = false; + + *is_ultra_sleep_mode = false; + + *is_video_playback = false; + + /* DSAT test scenario */ + if (active_replay_events & replay_event_test_harness_mode) { + if (link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]) + *coasting_vtotal = + link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]; + if (link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]) { + ASSERT(link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS] <= 0xFFFF); + *frame_skip_number = + (uint16_t)link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]; + } + + /* During the ultra sleep mode testing, disable the timing sync in short vblank mode */ + if (active_replay_events & (replay_event_test_harness_enable_replay)) { + if ((active_replay_events & replay_event_test_harness_ultra_sleep) && + !link->replay_settings.config.replay_support_fast_resync_in_ultra_sleep_mode) + link->replay_settings.config.replay_timing_sync_supported = false; + return true; + } else + return false; + } else if (active_replay_events & (replay_event_test_harness_enable_replay)) { + if (link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]) + *coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]; + if (link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]) { + uint32_t frame_skip_val = + link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]; + + ASSERT(frame_skip_val <= 0xFFFF); + *frame_skip_number = (uint16_t)frame_skip_val; + } + + /* During the ultra sleep mode testing, disable the timing sync in short vblank mode */ + if ((active_replay_events & replay_event_test_harness_ultra_sleep) && + !link->replay_settings.config.replay_support_fast_resync_in_ultra_sleep_mode) { + link->replay_settings.config.replay_timing_sync_supported = false; + } + return true; + } else if (active_replay_events & + (replay_event_test_harness_disable_replay | replay_event_os_request_disable)) { + // set last set coasting vtotal + if (link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]) + *coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS]; + if (link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]) { + uint32_t frame_skip_val = + link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]; + + ASSERT(frame_skip_val <= 0xFFFF); + *frame_skip_number = (uint16_t)frame_skip_val; + } + return false; + } + + /* Inactive conditions */ + if (active_replay_events & (replay_event_edp_panel_off_disable_psr | + replay_event_hw_programming | + replay_event_vrr | + replay_event_immediate_flip | + replay_event_prepare_vtotal | + replay_event_vrr_transition | + replay_event_pause | + replay_event_disable_replay_while_DPMS | + replay_event_sleep_resume | + replay_event_disable_in_AC | + replay_event_disable_replay_while_detect_display | + replay_event_infopacket | + replay_event_crc_window_active)) + return false; + + // Full screen scenario + if (active_replay_events & replay_event_full_screen) { + if (!(replay_enable_option & pr_enable_option_full_screen)) + return false; + } + + /* Full screen video scenario */ + if (active_replay_events & replay_event_big_screen_video) { + + link->replay_settings.config.replay_timing_sync_supported = false; + + if (replay_enable_option & pr_enable_option_full_screen_video_coasting) { + unsigned int fsn_vid = + link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_FULL_SCREEN_VIDEO]; + + *coasting_vtotal = + link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_FULL_SCREEN_VIDEO]; + ASSERT(fsn_vid <= 0xFFFF); + *frame_skip_number = (uint16_t)fsn_vid; + } + + *is_video_playback = true; + + if ((replay_enable_option & pr_enable_option_full_screen_video) && + (replay_enable_option & pr_enable_option_full_screen_video_coasting)) { + *is_full_screen_video = true; + return true; + } else + return false; + } + + /* MPO video scenario + * Some of the cases may contain a full screen UI layer in MPO video scenario which is + * not the expected case to enable Replay. + */ + if ((active_replay_events & replay_event_mpo_video_selective_update) && + !(active_replay_events & replay_event_full_screen)) { + + link->replay_settings.config.replay_timing_sync_supported = false; + + if (replay_enable_option & pr_enable_option_mpo_video_coasting) { + *coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM]; + { + uint32_t frame_skip_val = + link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM]; + + ASSERT(frame_skip_val <= 0xFFFF); + *frame_skip_number = (uint16_t)frame_skip_val; + } + } + + *is_video_playback = true; + + if (replay_enable_option & pr_enable_option_mpo_video) + return true; + else + return false; + } + + /* Static screen scenario */ + if (!(active_replay_events & replay_event_vsync)) { + + if (replay_enable_option & pr_enable_option_static_screen_coasting) { + // Do not adjust eDP refresh rate if static screen + normal sleep mode + if ((!(link->replay_settings.config.replay_power_opt_supported & + replay_power_opt_z10_static_screen)) || + (active_replay_events & replay_event_cursor_updating)) { + // normal sleep mode + *coasting_vtotal = + link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM]; + { + uint32_t frame_skip_val = + link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM]; + + ASSERT(frame_skip_val <= 0xFFFF); + *frame_skip_number = (uint16_t)frame_skip_val; + } + } else { + // ultra sleep mode + *coasting_vtotal = + link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_STATIC]; + { + uint32_t frame_skip_val = + link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_STATIC]; + + ASSERT(frame_skip_val <= 0xFFFF); + *frame_skip_number = (uint16_t)frame_skip_val; + } + *is_ultra_sleep_mode = true; + } + } + + if (replay_enable_option & pr_enable_option_static_screen) { + if (!link->replay_settings.config.replay_support_fast_resync_in_ultra_sleep_mode) + link->replay_settings.config.replay_timing_sync_supported = false; + return true; + } else + return false; + } + + /* General UI scenario */ + if (active_replay_events & replay_event_general_ui) { + if (replay_enable_option & pr_enable_option_general_ui) + return true; + else + return false; + } + + return false; +} + +bool mod_power_replay_set_coasting_vtotal(struct mod_power *mod_power, + const struct dc_stream_state *stream, + uint32_t coasting_vtotal, + uint16_t frame_skip_number) +{ + struct core_power *core_power = NULL; + struct dc_link *link = NULL; + + if (!stream) + return false; + + link = dc_stream_get_link(stream); + if (!link || !link->replay_settings.replay_feature_enabled) + return false; + + if (mod_power == NULL) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + + if (core_power->num_entities == 0) + return false; + + return link->dc->link_srv->edp_set_coasting_vtotal(link, coasting_vtotal, frame_skip_number); +} + +void mod_power_replay_set_timing_sync_supported(struct mod_power *mod_power, + const struct dc_stream_state *stream) +{ + struct core_power *core_power = NULL; + struct dc_link *link = NULL; + unsigned int stream_index = 0; + union dmub_replay_cmd_set cmd_data = { 0 }; + + if (!stream || mod_power == NULL) + return; + + core_power = MOD_POWER_TO_CORE(mod_power); + if (core_power->num_entities == 0) + return; + + stream_index = map_index_from_stream(core_power, stream); + if (stream_index > core_power->num_entities) //invalid index + return; + + link = dc_stream_get_link(stream); + if (!link || !link->replay_settings.replay_feature_enabled) + return; + + cmd_data.sync_data.timing_sync_supported = link->replay_settings.config.replay_timing_sync_supported; + + link->dc->link_srv->edp_send_replay_cmd(link, Replay_Set_Timing_Sync_Supported, + &cmd_data); +} + +void mod_power_replay_disabled_adaptive_sync_sdp(struct mod_power *mod_power, + const struct dc_stream_state *stream, bool force_disabled) +{ + struct core_power *core_power = NULL; + struct dc_link *link = NULL; + unsigned int stream_index = 0; + union dmub_replay_cmd_set cmd_data = { 0 }; + + if (!stream || mod_power == NULL) + return; + + core_power = MOD_POWER_TO_CORE(mod_power); + if (core_power->num_entities == 0) + return; + + stream_index = map_index_from_stream(core_power, stream); + if (stream_index > core_power->num_entities) //invalid index + return; + + link = dc_stream_get_link(stream); + if (!link || !link->replay_settings.replay_feature_enabled) + return; + + cmd_data.disabled_adaptive_sync_sdp_data.force_disabled = force_disabled; + + link->dc->link_srv->edp_send_replay_cmd(link, Replay_Disabled_Adaptive_Sync_SDP, + &cmd_data); +} + +static void mod_power_replay_set_general_cmd(struct mod_power *mod_power, + const struct dc_stream_state *stream, + const enum dmub_cmd_replay_general_subtype general_cmd_type, + const uint32_t param1, const uint32_t param2) +{ + struct core_power *core_power = NULL; + struct dc_link *link = NULL; + unsigned int stream_index = 0; + union dmub_replay_cmd_set cmd_data = { 0 }; + + if (!stream || mod_power == NULL) + return; + + core_power = MOD_POWER_TO_CORE(mod_power); + if (core_power->num_entities == 0) + return; + + stream_index = map_index_from_stream(core_power, stream); + if (stream_index > core_power->num_entities) //invalid index + return; + + link = dc_stream_get_link(stream); + if (!link || !link->replay_settings.replay_feature_enabled) + return; + + cmd_data.set_general_cmd_data.subtype = general_cmd_type; + cmd_data.set_general_cmd_data.param1 = param1; + cmd_data.set_general_cmd_data.param2 = param2; + link->dc->link_srv->edp_send_replay_cmd(link, Replay_Set_General_Cmd, + &cmd_data); +} + +void mod_power_replay_disabled_desync_error_detection(struct mod_power *mod_power, + const struct dc_stream_state *stream, bool force_disabled) +{ + mod_power_replay_set_general_cmd(mod_power, stream, + REPLAY_GENERAL_CMD_DISABLED_DESYNC_ERROR_DETECTION, + force_disabled, 0); +} + +static void mod_power_replay_set_pseudo_vtotal(struct mod_power *mod_power, + const struct dc_stream_state *stream, uint16_t vtotal) +{ + struct core_power *core_power = NULL; + struct dc_link *link = NULL; + unsigned int stream_index = 0; + union dmub_replay_cmd_set cmd_data = { 0 }; + + if (!stream || mod_power == NULL) + return; + + core_power = MOD_POWER_TO_CORE(mod_power); + if (core_power->num_entities == 0) + return; + + stream_index = map_index_from_stream(core_power, stream); + if (stream_index > core_power->num_entities) //invalid index + return; + + link = dc_stream_get_link(stream); + if (!link || !link->replay_settings.replay_feature_enabled) + return; + + cmd_data.pseudo_vtotal_data.vtotal = vtotal; + + if (link->replay_settings.last_pseudo_vtotal != vtotal) { + link->replay_settings.last_pseudo_vtotal = vtotal; + link->dc->link_srv->edp_send_replay_cmd(link, Replay_Set_Pseudo_VTotal, &cmd_data); + } +} + +static void mod_power_update_error_status(struct mod_power *mod_power, + const struct dc_stream_state *stream) +{ + struct dc_link *link = NULL; + union replay_debug_flags *pDebug = NULL; + + if (mod_power == NULL || stream == NULL) + return; + + link = dc_stream_get_link(stream); + + if (!link) + return; + + pDebug = (union replay_debug_flags *)&link->replay_settings.config.debug_flags; + + if (pDebug->bitfields.enable_visual_confirm_debug == 0) + return; + + mod_power_replay_set_general_cmd(mod_power, stream, + REPLAY_GENERAL_CMD_UPDATE_ERROR_STATUS, + link->replay_settings.config.replay_error_status.raw, 0); +} + +void mod_power_set_low_rr_activate(struct mod_power *mod_power, + const struct dc_stream_state *stream, bool low_rr_supported) +{ + struct dc_link *link = NULL; + + if (mod_power == NULL || stream == NULL) + return; + + link = dc_stream_get_link(stream); + + if (!link) + return; + + mod_power_replay_set_general_cmd(mod_power, stream, + REPLAY_GENERAL_CMD_SET_LOW_RR_ACTIVATE, + low_rr_supported, 0); +} + +void mod_power_set_video_conferencing_activate(struct mod_power *mod_power, + const struct dc_stream_state *stream, bool video_conferencing_activate) +{ + struct dc_link *link = NULL; + + if (mod_power == NULL || stream == NULL) + return; + + link = dc_stream_get_link(stream); + if (!link || !link->replay_settings.replay_feature_enabled) + return; + + mod_power_replay_set_general_cmd(mod_power, stream, + REPLAY_GENERAL_CMD_VIDEO_CONFERENCING, + video_conferencing_activate, 0); +} + +void mod_power_set_coasting_vtotal_without_frame_update(struct mod_power *mod_power, + const struct dc_stream_state *stream, uint32_t coasting_vtotal) +{ + struct dc_link *link = NULL; + + if (mod_power == NULL || stream == NULL) + return; + + link = dc_stream_get_link(stream); + if (!link || !link->replay_settings.replay_feature_enabled) + return; + + mod_power_replay_set_general_cmd(mod_power, stream, + REPLAY_GENERAL_CMD_SET_COASTING_VTOTAL_WITHOUT_FRAME_UPDATE, + coasting_vtotal, 0); +} + +void mod_power_set_replay_continuously_resync(struct mod_power *mod_power, + const struct dc_stream_state *stream, bool enable) +{ + struct dc_link *link = NULL; + + if (mod_power == NULL || stream == NULL) + return; + + link = dc_stream_get_link(stream); + if (!link || !link->replay_settings.replay_feature_enabled) + return; + + mod_power_replay_set_general_cmd(mod_power, stream, + REPLAY_GENERAL_CMD_SET_CONTINUOUSLY_RESYNC, + enable, 0); +} + +void mod_power_set_live_capture_with_cvt_activate(struct mod_power *mod_power, + const struct dc_stream_state *stream, bool live_capture_with_cvt_activate) +{ + struct dc_link *link = NULL; + + if (mod_power == NULL || stream == NULL) + return; + + link = dc_stream_get_link(stream); + if (!link || !link->replay_settings.replay_feature_enabled) + return; + + // Check if LIVE_CAPTURE_WITH_CVT bit is enabled in replay optimization config + if (!link->replay_settings.config.replay_optimization.bits.LIVE_CAPTURE_WITH_CVT) + return; + + if (link->replay_settings.config.live_capture_with_cvt_activated != live_capture_with_cvt_activate) { + link->replay_settings.config.live_capture_with_cvt_activated = live_capture_with_cvt_activate; + mod_power_replay_set_general_cmd(mod_power, stream, + REPLAY_GENERAL_CMD_LIVE_CAPTURE_WITH_CVT, + live_capture_with_cvt_activate, 0); + } +} + +bool mod_power_set_replay_event(struct mod_power *mod_power, + struct dc_stream_state *stream, bool set_event, + enum replay_event event, bool wait_for_disable) +{ + struct core_power *core_power = NULL; + struct dc_link *link = NULL; + unsigned int stream_index = 0; + unsigned int active_replay_events = 0; + bool replay_active_request = false; + bool force_static = false; + uint32_t coasting_vtotal = 0; + bool current_timing_sync_status = false; + bool is_full_screen_video = false; + bool is_ultra_sleep_mode = false; + unsigned int sink_duration_us = 0; + bool low_rr_active = false; + uint16_t frame_skip_number = 0; + bool is_video_playback = false; + + if (!stream) + return false; + + if (mod_power == NULL) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + + if (core_power->num_entities == 0) + return false; + + stream_index = map_index_from_stream(core_power, stream); + + if (set_event) + core_power->map[stream_index].replay_events |= event; + else + core_power->map[stream_index].replay_events &= ~event; + + link = dc_stream_get_link(stream); + if (!link || !link->replay_settings.replay_feature_enabled) + return false; + + if ((core_power->map[stream_index].replay_events & replay_event_disable_replay_while_switching_mux) != 0) + return false; + + if ((core_power->map[stream_index].replay_events & replay_event_os_override_hold) != 0) + return false; + + active_replay_events = core_power->map[stream_index].replay_events; + + current_timing_sync_status = + link->replay_settings.config.replay_timing_sync_supported; + + replay_active_request = mod_power_update_replay_active_status(active_replay_events, + link, &coasting_vtotal, &is_full_screen_video, &is_ultra_sleep_mode, &frame_skip_number, &is_video_playback); + + if (is_full_screen_video) + mod_power_replay_set_pseudo_vtotal(mod_power, stream, + link->replay_settings.low_rr_full_screen_video_pseudo_vtotal); + else + mod_power_replay_set_pseudo_vtotal(mod_power, stream, 0); + + //If timing_sync_status change, then re-enabled set timing_sync_supported value and re-enabled replay + if (current_timing_sync_status != link->replay_settings.config.replay_timing_sync_supported) + mod_power_replay_set_timing_sync_supported(mod_power, stream); + + if (link->replay_settings.config.low_rr_supported) { + sink_duration_us = + (unsigned int)(div_u64(((unsigned long long)(coasting_vtotal) + * 10000) * stream->timing.h_total, + stream->timing.pix_clk_100hz)); + low_rr_active = sink_duration_us < LOW_REFRESH_RATE_DURATION_US_UPPER_BOUND ? false : true; + if (low_rr_active != link->replay_settings.config.low_rr_activated) { + mod_power_set_low_rr_activate(mod_power, stream, low_rr_active); + link->replay_settings.config.low_rr_activated = low_rr_active; + } + } + + // The function return fail when + // 1. DMUB function is not support (for backward compatible). + // 2. active_replay_events or coasting_vtotal is not updated in the same time + if (!mod_power_replay_set_power_opt_and_coasting_vtotal(mod_power, + stream, active_replay_events, coasting_vtotal, is_ultra_sleep_mode, frame_skip_number)) { + if (!mod_power_replay_set_power_opt(mod_power, stream, active_replay_events, is_ultra_sleep_mode)) + return false; + + if (!mod_power_replay_set_coasting_vtotal(mod_power, stream, coasting_vtotal, frame_skip_number)) + return false; + } + + mod_power_set_live_capture_with_cvt_activate(mod_power, stream, is_video_playback); + + mod_power_update_error_status(mod_power, stream); + + // If Replay is going to be enable (No matter is disable -> enable or enable -> enable), we don't need to wait. + // If Replay is going to be disable + // if disable -> disable + // -> Replay DMUB state should be state 0. + // So no matter wait_for_disable is true or not, it should makes no difference. + // if enable -> disable -> We should wait if wait_for_disable is true. + if (replay_active_request) + wait_for_disable = false; + + if (!mod_power_set_replay_active(stream, replay_active_request, wait_for_disable, force_static)) + return false; + + return true; +} + +bool mod_power_get_replay_active_status(const struct dc_stream_state *stream, + bool *replay_active) +{ + const struct dc_link *link = NULL; + + if (!stream) + return false; + + link = dc_stream_get_link(stream); + *replay_active = link->replay_settings.replay_allow_active; + + return true; +} + +void mod_power_replay_residency(const struct dc_stream_state *stream, + unsigned int *residency, const bool is_start, const bool is_alpm) +{ + const struct dc_link *link = NULL; + enum pr_residency_mode mode; + + if (!stream) + return; + + link = dc_stream_get_link(stream); + + if (is_alpm) + mode = PR_RESIDENCY_MODE_ALPM; + else + mode = PR_RESIDENCY_MODE_PHY; + + if (link && link->dc && link->dc->link_srv) + link->dc->link_srv->edp_replay_residency(link, residency, is_start, mode); +} + +bool mod_power_replay_set_power_opt_and_coasting_vtotal(struct mod_power *mod_power, + const struct dc_stream_state *stream, unsigned int active_replay_events, uint32_t coasting_vtotal, + bool is_ultra_sleep_mode, uint16_t frame_skip_number) +{ + struct core_power *core_power = NULL; + struct dc_link *link = NULL; + unsigned int power_opt = 0; + + if (!stream) + return false; + + if (mod_power == NULL) + return false; + + core_power = MOD_POWER_TO_CORE(mod_power); + + if (core_power->num_entities == 0) + return false; + + link = dc_stream_get_link(stream); + + if (!link || !link->replay_settings.replay_feature_enabled) + return false; + + power_opt = mod_power_replay_setup_power_opt(link, active_replay_events, is_ultra_sleep_mode); + + return link->dc->link_srv->edp_set_replay_power_opt_and_coasting_vtotal(link, &power_opt, coasting_vtotal, frame_skip_number); +} + +void mod_power_replay_notify_mode_change(struct mod_power *mod_power, + struct dc *dc, + struct dc_link *link, + const struct dc_stream_state *stream, + unsigned int stream_index) +{ + struct core_power *core_power = NULL; + int active_replay_events = 0; + + if (!mod_power || !dc || !link || !stream) + return; + + core_power = MOD_POWER_TO_CORE(mod_power); + active_replay_events = core_power->map[stream_index].replay_events; + + link->replay_settings.replay_smu_opt_enable = + (link->replay_settings.config.replay_smu_opt_supported && + mod_power_only_edp(dc->current_state, stream)); + + if (active_replay_events & replay_event_os_request_force_ffu) { + link->replay_settings.config.os_request_force_ffu = true; + } + + if (dc_is_embedded_signal(stream->signal)) + dc->link_srv->dp_setup_replay(link, stream); +} + +void init_replay_config(struct dc_link *link, struct replay_config *pr_config) +{ + link->replay_settings.config = *pr_config; +} + +void set_replay_frame_skip_number(struct dc_link *link, + enum replay_coasting_vtotal_type type, + uint32_t coasting_vtotal_refresh_rate_uhz, + uint32_t flicker_free_refresh_rate_uhz, + bool is_defer) +{ + uint32_t *frame_skip_number_array = NULL; + uint32_t frame_skip_number = 0; + + if (link == NULL) + return; + + if (link->replay_settings.config.frame_skip_supported == false) + return; + + if (flicker_free_refresh_rate_uhz == 0 || coasting_vtotal_refresh_rate_uhz == 0) + return; + + if (is_defer) + frame_skip_number_array = link->replay_settings.defer_frame_skip_number_table; + else + frame_skip_number_array = link->replay_settings.frame_skip_number_table; + + if (frame_skip_number_array == NULL) + return; + + frame_skip_number = (coasting_vtotal_refresh_rate_uhz + 500000) / flicker_free_refresh_rate_uhz; + + if (frame_skip_number >= 1) + frame_skip_number_array[type] = frame_skip_number - 1; + else + frame_skip_number_array[type] = 0; +} + +void set_replay_defer_update_coasting_vtotal(struct dc_link *link, + enum replay_coasting_vtotal_type type, + uint32_t vtotal) +{ + link->replay_settings.defer_update_coasting_vtotal_table[type] = vtotal; +} + +void update_replay_coasting_vtotal_from_defer(struct dc_link *link, + enum replay_coasting_vtotal_type type) +{ + link->replay_settings.coasting_vtotal_table[type] = + link->replay_settings.defer_update_coasting_vtotal_table[type]; + link->replay_settings.frame_skip_number_table[type] = + link->replay_settings.defer_frame_skip_number_table[type]; +} + +void set_replay_coasting_vtotal(struct dc_link *link, + enum replay_coasting_vtotal_type type, + uint32_t vtotal) +{ + link->replay_settings.coasting_vtotal_table[type] = vtotal; +} + +void set_replay_low_rr_full_screen_video_src_vtotal(struct dc_link *link, uint16_t vtotal) +{ + link->replay_settings.low_rr_full_screen_video_pseudo_vtotal = vtotal; +} + +void calculate_replay_link_off_frame_count(struct dc_link *link, + uint16_t vtotal, uint16_t htotal) +{ + uint32_t max_link_off_frame_count = 0; + uint16_t max_deviation_line = 0, pixel_deviation_per_line = 0; + + if (!link || link->replay_settings.config.replay_version != DC_FREESYNC_REPLAY) + return; + + max_deviation_line = link->dpcd_caps.pr_info.max_deviation_line; + pixel_deviation_per_line = link->dpcd_caps.pr_info.pixel_deviation_per_line; + + if (htotal != 0 && vtotal != 0 && pixel_deviation_per_line != 0) + max_link_off_frame_count = htotal * max_deviation_line / (pixel_deviation_per_line * vtotal); + else + ASSERT(0); + + link->replay_settings.link_off_frame_count = max_link_off_frame_count; +} + +void reset_replay_dsync_error_count(struct dc_link *link) +{ + link->replay_settings.replay_desync_error_fail_count = 0; +} diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c index cc3a265700a5..a1318409e4b5 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c @@ -435,10 +435,12 @@ int smu_v15_0_fini_smc_tables(struct smu_context *smu) smu_table->watermarks_table = NULL; smu_table->metrics_time = 0; + kfree(smu_dpm->dpm_policies); kfree(smu_dpm->dpm_context); kfree(smu_dpm->golden_dpm_context); kfree(smu_dpm->dpm_current_power_state); kfree(smu_dpm->dpm_request_power_state); + smu_dpm->dpm_policies = NULL; smu_dpm->dpm_context = NULL; smu_dpm->golden_dpm_context = NULL; smu_dpm->dpm_context_size = 0; diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c index cb6498c30834..c22e53e84207 100644 --- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c +++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c @@ -139,7 +139,8 @@ static int amdgpu_ras_get_ras_safe_fb_addr_ranges(struct ras_core_context *ras_c struct amdgpu_mem_partition_info *mem_ranges; uint32_t i = 0; - if (cmd->input_size != sizeof(*input_data)) + if ((cmd->input_size != sizeof(*input_data)) || + (cmd->output_buf_size < sizeof(*ranges))) return RAS_CMD__ERROR_INVALID_INPUT_DATA; mem_ranges = adev->gmc.mem_partitions; @@ -207,7 +208,8 @@ static int amdgpu_ras_translate_fb_address(struct ras_core_context *ras_core, (struct ras_cmd_translate_fb_address_rsp *)cmd->output_buff_raw; int ret = RAS_CMD__ERROR_GENERIC; - if (cmd->input_size != sizeof(struct ras_cmd_translate_fb_address_req)) + if ((cmd->input_size != sizeof(struct ras_cmd_translate_fb_address_req)) || + (cmd->output_buf_size < sizeof(*rsp_buff))) return RAS_CMD__ERROR_INVALID_INPUT_SIZE; if ((req_buff->src_addr_type >= RAS_FB_ADDR_UNKNOWN) || @@ -279,7 +281,7 @@ int amdgpu_ras_submit_cmd(struct ras_core_context *ras_core, struct ras_cmd_ctx cmd->cmd_res = res; - if (cmd->output_size > cmd->output_buf_size) { + if (!res && (cmd->output_size > cmd->output_buf_size)) { RAS_DEV_ERR(cmd_core->dev, "Output size 0x%x exceeds output buffer size 0x%x!\n", cmd->output_size, cmd->output_buf_size); diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c index ad8862d43263..a22d1aebbeb9 100644 --- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c +++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c @@ -291,6 +291,8 @@ static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block) con->uniras_enabled = false; if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14) || + amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || + amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || adev->debug_enable_ras_aca) con->uniras_enabled = true; else @@ -307,13 +309,17 @@ static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block) if (!ras_mgr->ras_core) { RAS_DEV_ERR(adev, "Failed to create ras core!\n"); ret = -EINVAL; - goto err; + goto err1; } ras_mgr->ras_core->dev = adev; amdgpu_ras_process_init(adev); - ras_core_sw_init(ras_mgr->ras_core); + ret = ras_core_sw_init(ras_mgr->ras_core); + if (ret) { + RAS_DEV_ERR(adev, "ras_core_sw_init failed! ret:%d\n", ret); + goto err2; + } amdgpu_ras_mgr_init_event_mgr(ras_mgr->ras_core); if (amdgpu_sriov_vf(adev)) { @@ -321,14 +327,22 @@ static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block) if (ret) { RAS_DEV_ERR(adev, "Virt ras sw_init failed! ret:%d\n", ret); - goto err; + goto err3; } } return 0; -err: +err3: + if (ras_mgr->ras_core) + ras_core_sw_fini(ras_mgr->ras_core); +err2: + amdgpu_ras_process_fini(adev); + if (ras_mgr->ras_core) + ras_core_destroy(ras_mgr->ras_core); +err1: kfree(ras_mgr); + con->ras_mgr = NULL; return ret; } @@ -496,8 +510,10 @@ uint64_t amdgpu_ras_mgr_gen_ras_event_seqno(struct amdgpu_device *adev, if ((seqno_type == RAS_SEQNO_TYPE_DE) || (seqno_type == RAS_SEQNO_TYPE_POISON_CONSUMPTION)) { ret = ras_core_put_seqno(ras_mgr->ras_core, seqno_type, seq_no); - if (ret) + if (ret) { RAS_DEV_WARN(adev, "There are too many ras interrupts!"); + return 0; + } } return seq_no; diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c index fb4d375e87b2..c2761f3d06a0 100644 --- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c +++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c @@ -192,8 +192,17 @@ static int amdgpu_virt_ras_get_cper_snapshot(struct ras_core_context *ras_core, return RAS_CMD__SUCCESS; } +static bool amdgpu_virt_ras_check_batch_cached(struct ras_cmd_batch_trace_record_rsp *rsp, + uint64_t batch_id) +{ + return rsp->real_batch_num && + rsp->real_batch_num <= RAS_CMD_MAX_BATCH_NUM && + batch_id >= rsp->start_batch_id && + (batch_id - rsp->start_batch_id) < rsp->real_batch_num; +} + static int amdgpu_virt_ras_get_batch_records(struct ras_core_context *ras_core, uint64_t batch_id, - struct ras_log_info **trace_arr, uint32_t arr_num, + struct ras_log_info *trace_arr, uint32_t arr_num, struct ras_cmd_batch_trace_record_rsp *rsp_cache) { struct ras_cmd_batch_trace_record_req req = { @@ -204,27 +213,34 @@ static int amdgpu_virt_ras_get_batch_records(struct ras_core_context *ras_core, struct batch_ras_trace_info *batch; int ret = 0; uint32_t i; + uint32_t idx; - if (!rsp->real_batch_num || (batch_id < rsp->start_batch_id) || - (batch_id >= (rsp->start_batch_id + rsp->real_batch_num))) { - + if (!amdgpu_virt_ras_check_batch_cached(rsp, batch_id)) { memset(rsp, 0, sizeof(*rsp)); ret = amdgpu_virt_ras_send_remote_cmd(ras_core, RAS_CMD__GET_BATCH_TRACE_RECORD, &req, sizeof(req), rsp, sizeof(*rsp)); if (ret) return -EPIPE; + + if (!amdgpu_virt_ras_check_batch_cached(rsp, batch_id)) { + memset(rsp, 0, sizeof(*rsp)); + return -EIO; + } } - batch = &rsp->batchs[batch_id - rsp->start_batch_id]; - if (batch_id != batch->batch_id) - return -ENODATA; - - for (i = 0; i < batch->trace_num; i++) { - if (i >= arr_num) - break; - trace_arr[i] = &rsp->records[batch->offset + i]; + idx = (uint32_t)(batch_id - rsp->start_batch_id); + batch = &rsp->batchs[idx]; + if (batch_id != batch->batch_id || + batch->trace_num > MAX_RECORD_PER_BATCH || + (uint32_t)batch->offset + batch->trace_num > RAS_CMD_MAX_TRACE_NUM) { + memset(rsp, 0, sizeof(*rsp)); + return -EIO; } + for (i = 0; i < batch->trace_num && i < arr_num; i++) + memcpy(&trace_arr[i], + &rsp->records[batch->offset + i], sizeof(*trace_arr)); + return i; } @@ -240,19 +256,22 @@ static int amdgpu_virt_ras_get_cper_records(struct ras_core_context *ras_core, (struct ras_cmd_cper_record_rsp *)cmd->output_buff_raw; struct ras_log_batch_overview *overview = &virt_ras->batch_mgr.batch_overview; struct ras_cmd_batch_trace_record_rsp *rsp_cache = &virt_ras->batch_mgr.batch_trace; - struct ras_log_info **trace; + struct ras_log_info *trace; + uint32_t trace_count = MAX_RECORD_PER_BATCH; uint32_t offset = 0, real_data_len = 0; uint64_t batch_id; uint8_t *out_buf; int ret = 0, i, count; - if (cmd->input_size != sizeof(struct ras_cmd_cper_record_req)) + if (cmd->input_size != sizeof(struct ras_cmd_cper_record_req) || + (cmd->output_buf_size < sizeof(*rsp))) return RAS_CMD__ERROR_INVALID_INPUT_SIZE; - if (!req->buf_size || !req->buf_ptr || !req->cper_num) + if (!req->buf_size || !req->buf_ptr || !req->cper_num || + req->buf_size > RAS_CMD_MAX_CPER_BUF_SZ) return RAS_CMD__ERROR_INVALID_INPUT_DATA; - trace = kzalloc_objs(*trace, MAX_RECORD_PER_BATCH); + trace = kzalloc_objs(*trace, trace_count); if (!trace) return RAS_CMD__ERROR_GENERIC; @@ -269,7 +288,7 @@ static int amdgpu_virt_ras_get_cper_records(struct ras_core_context *ras_core, if (batch_id >= overview->last_batch_id) break; count = amdgpu_virt_ras_get_batch_records(ras_core, batch_id, - trace, MAX_RECORD_PER_BATCH, + trace, trace_count, rsp_cache); if (count > 0) { ret = ras_cper_generate_cper(ras_core, trace, count, @@ -455,7 +474,7 @@ int amdgpu_virt_ras_handle_cmd(struct ras_core_context *ras_core, cmd->cmd_res = res; - if (cmd->output_size > cmd->output_buf_size) { + if (!res && (cmd->output_size > cmd->output_buf_size)) { RAS_DEV_ERR(ras_core->dev, "Output data size 0x%x exceeds buffer size 0x%x!\n", cmd->output_size, cmd->output_buf_size); diff --git a/drivers/gpu/drm/amd/ras/rascore/ras.h b/drivers/gpu/drm/amd/ras/rascore/ras.h index 6449d7b8627d..c059fcebaf00 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras.h +++ b/drivers/gpu/drm/amd/ras/rascore/ras.h @@ -329,7 +329,6 @@ struct ras_core_context { struct ras_gfx ras_gfx; struct ras_mp1 ras_mp1; struct ras_process ras_proc; - struct ras_cmd_mgr ras_cmd; struct ras_log_ring ras_log_ring; const struct ras_sys_func *sys_fn; diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c index 4f89810d85a1..abd004eee95e 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c +++ b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c @@ -28,20 +28,6 @@ #define RAS_CMD_MINOR_VERSION 0 #define RAS_CMD_VERSION (((RAS_CMD_MAJOR_VERSION) << 10) | (RAS_CMD_MINOR_VERSION)) -static int ras_cmd_add_device(struct ras_core_context *ras_core) -{ - INIT_LIST_HEAD(&ras_core->ras_cmd.head); - ras_core->ras_cmd.ras_core = ras_core; - ras_core->ras_cmd.dev_handle = (uintptr_t)ras_core ^ RAS_CMD_DEV_HANDLE_MAGIC; - return 0; -} - -static int ras_cmd_remove_device(struct ras_core_context *ras_core) -{ - memset(&ras_core->ras_cmd, 0, sizeof(ras_core->ras_cmd)); - return 0; -} - static int ras_get_block_ecc_info(struct ras_core_context *ras_core, struct ras_cmd_ctx *cmd, void *data) { @@ -52,7 +38,8 @@ static int ras_get_block_ecc_info(struct ras_core_context *ras_core, struct ras_ecc_count err_data; int ret; - if (cmd->input_size != sizeof(struct ras_cmd_block_ecc_info_req)) + if ((cmd->input_size != sizeof(struct ras_cmd_block_ecc_info_req)) || + (cmd->output_buf_size < sizeof(*output_data))) return RAS_CMD__ERROR_INVALID_INPUT_SIZE; memset(&err_data, 0, sizeof(err_data)); @@ -87,25 +74,31 @@ static int ras_cmd_get_group_bad_pages(struct ras_core_context *ras_core, struct eeprom_umc_record record; struct ras_cmd_bad_page_record *ras_cmd_record; uint32_t i = 0, bp_cnt = 0, group_cnt = 0; + int ret = RAS_CMD__SUCCESS; output_data->bp_in_group = 0; output_data->group_index = 0; + mutex_lock(&ras_core->ras_umc.umc_lock); bp_cnt = ras_umc_get_badpage_count(ras_core); if (bp_cnt) { output_data->group_index = group_index; group_cnt = bp_cnt / RAS_CMD_MAX_BAD_PAGES_PER_GROUP + ((bp_cnt % RAS_CMD_MAX_BAD_PAGES_PER_GROUP) ? 1 : 0); - if (group_index >= group_cnt) - return RAS_CMD__ERROR_INVALID_INPUT_DATA; + if (group_index >= group_cnt) { + ret = RAS_CMD__ERROR_INVALID_INPUT_DATA; + goto out; + } i = group_index * RAS_CMD_MAX_BAD_PAGES_PER_GROUP; for (; i < bp_cnt && output_data->bp_in_group < RAS_CMD_MAX_BAD_PAGES_PER_GROUP; i++) { - if (ras_umc_get_badpage_record(ras_core, i, &record)) - return RAS_CMD__ERROR_GENERIC; + if (ras_umc_get_badpage_record(ras_core, i, &record)) { + ret = RAS_CMD__ERROR_GENERIC; + goto out; + } ras_cmd_record = &output_data->records[i % RAS_CMD_MAX_BAD_PAGES_PER_GROUP]; @@ -115,7 +108,10 @@ static int ras_cmd_get_group_bad_pages(struct ras_core_context *ras_core, } } output_data->bp_total_cnt = bp_cnt; - return RAS_CMD__SUCCESS; + +out: + mutex_unlock(&ras_core->ras_umc.umc_lock); + return ret; } static int ras_cmd_get_bad_pages(struct ras_core_context *ras_core, @@ -127,7 +123,8 @@ static int ras_cmd_get_bad_pages(struct ras_core_context *ras_core, (struct ras_cmd_bad_pages_info_rsp *)cmd->output_buff_raw; int ret; - if (cmd->input_size != sizeof(struct ras_cmd_bad_pages_info_req)) + if ((cmd->input_size != sizeof(struct ras_cmd_bad_pages_info_req)) || + (cmd->output_buf_size < sizeof(*output_data))) return RAS_CMD__ERROR_INVALID_INPUT_SIZE; ret = ras_cmd_get_group_bad_pages(ras_core, input_data->group_index, output_data); @@ -182,7 +179,8 @@ static int ras_cmd_get_cper_snapshot(struct ras_core_context *ras_core, (struct ras_cmd_cper_snapshot_rsp *)cmd->output_buff_raw; struct ras_log_batch_overview overview; - if (cmd->input_size != sizeof(struct ras_cmd_cper_snapshot_req)) + if ((cmd->input_size != sizeof(struct ras_cmd_cper_snapshot_req)) || + (cmd->output_buf_size < sizeof(*output_data))) return RAS_CMD__ERROR_INVALID_INPUT_SIZE; ras_log_ring_get_batch_overview(ras_core, &overview); @@ -204,23 +202,32 @@ static int ras_cmd_get_cper_records(struct ras_core_context *ras_core, (struct ras_cmd_cper_record_req *)cmd->input_buff_raw; struct ras_cmd_cper_record_rsp *rsp = (struct ras_cmd_cper_record_rsp *)cmd->output_buff_raw; - struct ras_log_info *trace[MAX_RECORD_PER_BATCH] = {0}; + struct ras_log_info *trace = NULL; + uint32_t trace_count = MAX_RECORD_PER_BATCH; struct ras_log_batch_overview overview; uint32_t offset = 0, real_data_len = 0; uint64_t batch_id; - uint8_t *buffer; + uint8_t *buffer = NULL; int ret = 0, i, count; - if (cmd->input_size != sizeof(struct ras_cmd_cper_record_req)) + if ((cmd->input_size != sizeof(struct ras_cmd_cper_record_req)) || + (cmd->output_buf_size < sizeof(*rsp))) return RAS_CMD__ERROR_INVALID_INPUT_SIZE; - if (!req->buf_size || !req->buf_ptr || !req->cper_num) + if (!req->buf_size || !req->buf_ptr || !req->cper_num || + req->buf_size > RAS_CMD_MAX_CPER_BUF_SZ) return RAS_CMD__ERROR_INVALID_INPUT_DATA; buffer = kzalloc(req->buf_size, GFP_KERNEL); if (!buffer) return RAS_CMD__ERROR_GENERIC; + trace = kcalloc(trace_count, sizeof(*trace), GFP_KERNEL); + if (!trace) { + ret = RAS_CMD__ERROR_GENERIC; + goto out; + } + ras_log_ring_get_batch_overview(ras_core, &overview); for (i = 0; i < req->cper_num; i++) { batch_id = req->cper_start_id + i; @@ -228,7 +235,7 @@ static int ras_cmd_get_cper_records(struct ras_core_context *ras_core, break; count = ras_log_ring_get_batch_records(ras_core, batch_id, trace, - ARRAY_SIZE(trace)); + trace_count); if (count > 0) { ret = ras_cper_generate_cper(ras_core, trace, count, &buffer[offset], req->buf_size - offset, &real_data_len); @@ -241,8 +248,8 @@ static int ras_cmd_get_cper_records(struct ras_core_context *ras_core, if ((ret && (ret != -ENOMEM)) || copy_to_user(u64_to_user_ptr(req->buf_ptr), buffer, offset)) { - kfree(buffer); - return RAS_CMD__ERROR_GENERIC; + ret = RAS_CMD__ERROR_GENERIC; + goto out; } rsp->real_data_size = offset; @@ -251,10 +258,12 @@ static int ras_cmd_get_cper_records(struct ras_core_context *ras_core, rsp->version = 0; cmd->output_size = sizeof(struct ras_cmd_cper_record_rsp); + ret = RAS_CMD__SUCCESS; +out: + kfree(trace); kfree(buffer); - - return RAS_CMD__SUCCESS; + return ret; } static int ras_cmd_get_batch_trace_snapshot(struct ras_core_context *ras_core, @@ -265,7 +274,8 @@ static int ras_cmd_get_batch_trace_snapshot(struct ras_core_context *ras_core, struct ras_log_batch_overview overview; - if (cmd->input_size != sizeof(struct ras_cmd_batch_trace_snapshot_req)) + if ((cmd->input_size != sizeof(struct ras_cmd_batch_trace_snapshot_req)) || + (cmd->output_buf_size < sizeof(*rsp))) return RAS_CMD__ERROR_INVALID_INPUT_SIZE; ras_log_ring_get_batch_overview(ras_core, &overview); @@ -287,13 +297,15 @@ static int ras_cmd_get_batch_trace_records(struct ras_core_context *ras_core, struct ras_cmd_batch_trace_record_rsp *output_data = (struct ras_cmd_batch_trace_record_rsp *)cmd->output_buff_raw; struct ras_log_batch_overview overview; - struct ras_log_info *trace_arry[MAX_RECORD_PER_BATCH] = {0}; + struct ras_log_info *trace_arry = NULL; + uint32_t trace_count = MAX_RECORD_PER_BATCH; struct ras_log_info *record; int i, j, count = 0, offset = 0; uint64_t id; bool completed = false; - if (cmd->input_size != sizeof(struct ras_cmd_batch_trace_record_req)) + if ((cmd->input_size != sizeof(struct ras_cmd_batch_trace_record_req)) || + (cmd->output_buf_size < sizeof(*output_data))) return RAS_CMD__ERROR_INVALID_INPUT_SIZE; if ((!input_data->batch_num) || (input_data->batch_num > RAS_CMD_MAX_BATCH_NUM)) @@ -304,6 +316,10 @@ static int ras_cmd_get_batch_trace_records(struct ras_core_context *ras_core, (input_data->start_batch_id >= overview.last_batch_id)) return RAS_CMD__ERROR_INVALID_INPUT_SIZE; + trace_arry = kcalloc(trace_count, sizeof(*trace_arry), GFP_KERNEL); + if (!trace_arry) + return RAS_CMD__ERROR_GENERIC; + for (i = 0; i < input_data->batch_num; i++) { id = input_data->start_batch_id + i; if (id >= overview.last_batch_id) { @@ -312,17 +328,17 @@ static int ras_cmd_get_batch_trace_records(struct ras_core_context *ras_core, } count = ras_log_ring_get_batch_records(ras_core, - id, trace_arry, ARRAY_SIZE(trace_arry)); + id, trace_arry, trace_count); if (count > 0) { if ((offset + count) > RAS_CMD_MAX_TRACE_NUM) break; for (j = 0; j < count; j++) { record = &output_data->records[offset + j]; - record->seqno = trace_arry[j]->seqno; - record->timestamp = trace_arry[j]->timestamp; - record->event = trace_arry[j]->event; + record->seqno = trace_arry[j].seqno; + record->timestamp = trace_arry[j].timestamp; + record->event = trace_arry[j].event; memcpy(&record->aca_reg, - &trace_arry[j]->aca_reg, sizeof(trace_arry[j]->aca_reg)); + &trace_arry[j].aca_reg, sizeof(trace_arry[j].aca_reg)); } } else { count = 0; @@ -341,6 +357,8 @@ static int ras_cmd_get_batch_trace_records(struct ras_core_context *ras_core, cmd->output_size = sizeof(struct ras_cmd_batch_trace_record_rsp); + kfree(trace_arry); + return RAS_CMD__SUCCESS; } @@ -420,6 +438,10 @@ static int ras_cmd_inject_error(struct ras_core_context *ras_core, .value = req->method, }; + if ((cmd->input_size != sizeof(*req)) || + (cmd->output_buf_size < sizeof(*output_data))) + return RAS_CMD__ERROR_INVALID_INPUT_SIZE; + ret = ras_psp_trigger_error(ras_core, &block_info, req->instance_mask); if (!ret) { output_data->version = 0; @@ -466,12 +488,11 @@ int rascore_handle_cmd(struct ras_core_context *ras_core, int ras_cmd_init(struct ras_core_context *ras_core) { - return ras_cmd_add_device(ras_core); + return 0; } int ras_cmd_fini(struct ras_core_context *ras_core) { - ras_cmd_remove_device(ras_core); return 0; } @@ -520,8 +541,3 @@ int ras_cmd_translate_bank_to_soc_pa(struct ras_core_context *ras_core, return ras_umc_translate_soc_pa_and_bank(ras_core, soc_pa, &umc_bank, true); } - -uint64_t ras_cmd_get_dev_handle(struct ras_core_context *ras_core) -{ - return ras_core->ras_cmd.dev_handle; -} diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h index 7ea35a028987..c376e074e15d 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h +++ b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h @@ -29,8 +29,6 @@ #include "ras_log_ring.h" #include "ras_cper.h" -#define RAS_CMD_DEV_HANDLE_MAGIC 0xFEEDAD00UL - #define RAS_CMD_MAX_IN_SIZE 256 #define RAS_CMD_MAX_GPU_NUM 32 #define RAS_CMD_MAX_BAD_PAGES_PER_GROUP 32 @@ -107,12 +105,6 @@ enum ras_error_type { struct ras_core_context; struct ras_cmd_ctx; -struct ras_cmd_mgr { - struct list_head head; - struct ras_core_context *ras_core; - uint64_t dev_handle; -}; - struct ras_cmd_func_map { uint32_t cmd_id; int (*func)(struct ras_core_context *ras_core, @@ -405,6 +397,9 @@ struct batch_ras_trace_info { #define RAS_CMD_MAX_BATCH_NUM 300 #define RAS_CMD_MAX_TRACE_NUM 300 + +/* Upper bounds for RAS_CMD__GET_CPER_RECORD to limit kernel allocations and work. */ +#define RAS_CMD_MAX_CPER_BUF_SZ (2 * 1024U * 1024U) /* 2 MiB */ struct ras_cmd_batch_trace_record_rsp { uint32_t version; uint16_t real_batch_num; @@ -478,7 +473,6 @@ struct ras_cmd_blocks_ecc_rsp { int ras_cmd_init(struct ras_core_context *ras_core); int ras_cmd_fini(struct ras_core_context *ras_core); int rascore_handle_cmd(struct ras_core_context *ras_core, struct ras_cmd_ctx *cmd, void *data); -uint64_t ras_cmd_get_dev_handle(struct ras_core_context *ras_core); int ras_cmd_query_interface_info(struct ras_core_context *ras_core, struct ras_query_interface_info_rsp *rsp); int ras_cmd_translate_soc_pa_to_bank(struct ras_core_context *ras_core, diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_core.c b/drivers/gpu/drm/amd/ras/rascore/ras_core.c index 49b3aa7489ff..62d124a3eeac 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_core.c +++ b/drivers/gpu/drm/amd/ras/rascore/ras_core.c @@ -196,9 +196,9 @@ uint64_t ras_core_gen_seqno(struct ras_core_context *ras_core, { uint64_t seqno = 0; - if (ras_core->sys_fn && - ras_core->sys_fn->gen_seqno) - ras_core->sys_fn->gen_seqno(ras_core, type, &seqno); + if (ras_core->sys_fn && ras_core->sys_fn->gen_seqno && + ras_core->sys_fn->gen_seqno(ras_core, type, &seqno)) + return 0; return seqno; } diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_cper.c b/drivers/gpu/drm/amd/ras/rascore/ras_cper.c index 0fc7522b7ab6..6e93a13bbc4c 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_cper.c +++ b/drivers/gpu/drm/amd/ras/rascore/ras_cper.c @@ -175,14 +175,14 @@ static int fill_section_runtime(struct ras_core_context *ras_core, } static int cper_generate_runtime_record(struct ras_core_context *ras_core, - struct cper_section_hdr *hdr, struct ras_log_info **trace_arr, uint32_t arr_num, + struct cper_section_hdr *hdr, struct ras_log_info *trace_arr, uint32_t arr_num, enum ras_cper_severity sev) { struct cper_section_descriptor *descriptor; struct cper_section_runtime *runtime; int i; - fill_section_hdr(ras_core, hdr, RAS_CPER_TYPE_RUNTIME, sev, trace_arr[0]); + fill_section_hdr(ras_core, hdr, RAS_CPER_TYPE_RUNTIME, sev, &trace_arr[0]); hdr->record_length = RAS_HDR_LEN + ((RAS_SEC_DESC_LEN + RAS_NONSTD_SEC_LEN) * arr_num); hdr->sec_cnt = arr_num; for (i = 0; i < arr_num; i++) { @@ -194,21 +194,21 @@ static int cper_generate_runtime_record(struct ras_core_context *ras_core, fill_section_descriptor(ras_core, descriptor, sev, RUNTIME, RAS_NONSTD_SEC_OFFSET(hdr->sec_cnt, i), sizeof(struct cper_section_runtime)); - fill_section_runtime(ras_core, runtime, trace_arr[i], sev); + fill_section_runtime(ras_core, runtime, &trace_arr[i], sev); } return 0; } static int cper_generate_fatal_record(struct ras_core_context *ras_core, - uint8_t *buffer, struct ras_log_info **trace_arr, uint32_t arr_num) + uint8_t *buffer, struct ras_log_info *trace_arr, uint32_t arr_num) { struct ras_cper_fatal_record record = {0}; int i = 0; for (i = 0; i < arr_num; i++) { fill_section_hdr(ras_core, &record.hdr, RAS_CPER_TYPE_FATAL, - RAS_CPER_SEV_FATAL_UE, trace_arr[i]); + RAS_CPER_SEV_FATAL_UE, &trace_arr[i]); record.hdr.record_length = RAS_HDR_LEN + RAS_SEC_DESC_LEN + RAS_FATAL_SEC_LEN; record.hdr.sec_cnt = 1; @@ -216,7 +216,7 @@ static int cper_generate_fatal_record(struct ras_core_context *ras_core, CRASHDUMP, offsetof(struct ras_cper_fatal_record, fatal), sizeof(struct cper_section_fatal)); - fill_section_fatal(ras_core, &record.fatal, trace_arr[i]); + fill_section_fatal(ras_core, &record.fatal, &trace_arr[i]); memcpy(buffer + (i * record.hdr.record_length), &record, record.hdr.record_length); @@ -271,7 +271,7 @@ static enum ras_cper_type cper_ras_log_event_to_cper_type(enum ras_log_event eve } int ras_cper_generate_cper(struct ras_core_context *ras_core, - struct ras_log_info **trace_list, uint32_t count, + struct ras_log_info *trace_list, uint32_t count, uint8_t *buf, uint32_t buf_len, uint32_t *real_data_len) { uint8_t *buffer = buf; @@ -281,14 +281,14 @@ int ras_cper_generate_cper(struct ras_core_context *ras_core, /* All the batch traces share the same event */ record_size = cper_get_record_size( - cper_ras_log_event_to_cper_type(trace_list[0]->event), count); + cper_ras_log_event_to_cper_type(trace_list[0].event), count); if ((record_size + saved_size) > buf_size) return -ENOMEM; hdr = (struct cper_section_hdr *)(buffer + saved_size); - switch (trace_list[0]->event) { + switch (trace_list[0].event) { case RAS_LOG_EVENT_RMA: cper_generate_runtime_record(ras_core, hdr, trace_list, count, RAS_CPER_SEV_RMA); break; @@ -304,7 +304,7 @@ int ras_cper_generate_cper(struct ras_core_context *ras_core, cper_generate_fatal_record(ras_core, buffer + saved_size, trace_list, count); break; default: - RAS_DEV_WARN(ras_core->dev, "Unprocessed trace event: %d\n", trace_list[0]->event); + RAS_DEV_WARN(ras_core->dev, "Unprocessed trace event: %d\n", trace_list[0].event); break; } diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_cper.h b/drivers/gpu/drm/amd/ras/rascore/ras_cper.h index 076c1883c1ce..e4e3615ecc2e 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_cper.h +++ b/drivers/gpu/drm/amd/ras/rascore/ras_cper.h @@ -299,6 +299,6 @@ struct ras_cper_fatal_record { struct ras_core_context; struct ras_log_info; int ras_cper_generate_cper(struct ras_core_context *ras_core, - struct ras_log_info **trace_list, uint32_t count, + struct ras_log_info *trace_list, uint32_t count, uint8_t *buf, uint32_t buf_len, uint32_t *real_data_len); #endif diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c index 65c1812a10fb..34b798f45af2 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c +++ b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c @@ -1163,6 +1163,13 @@ static int __check_ras_table_status(struct ras_core_context *ras_core) } control->ras_fri = RAS_OFFSET_TO_INDEX(control, hdr->first_rec_offset); + if (hdr->first_rec_offset < control->ras_record_offset || + control->ras_fri >= control->ras_max_record_count) { + RAS_DEV_ERR(ras_core->dev, + "RAS header invalid, ras_fri: %u, first_rec_offset:0x%x", + control->ras_fri, hdr->first_rec_offset); + return -EINVAL; + } return 0; } diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c index 29001e606d1b..f5fa80db91fb 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c +++ b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c @@ -270,6 +270,7 @@ int ras_fw_eeprom_read_idx(struct ras_core_context *ras_core, struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom; int i, ret, end_idx; u64 mca, ipid, ts; + u32 cu, mem_channel, mcumc_id; if (!ras_core->ras_umc.ip_func || !ras_core->ras_umc.ip_func->mca_ipid_parse) @@ -299,9 +300,10 @@ int ras_fw_eeprom_read_idx(struct ras_core_context *ras_core, record_umc[i - rec_idx].err_type = RAS_EEPROM_ERR_NON_RECOVERABLE; ras_core->ras_umc.ip_func->mca_ipid_parse(ras_core, ipid, - (uint32_t *)&(record_umc[i - rec_idx].cu), - (uint32_t *)&(record_umc[i - rec_idx].mem_channel), - (uint32_t *)&(record_umc[i - rec_idx].mcumc_id), NULL); + &cu, &mem_channel, &mcumc_id, NULL); + record_umc[i - rec_idx].cu = (u8)cu; + record_umc[i - rec_idx].mem_channel = (u8)mem_channel; + record_umc[i - rec_idx].mcumc_id = (u8)mcumc_id; /* update bad channel bitmap */ if ((record_umc[i - rec_idx].mem_channel < BITS_PER_TYPE(control->bad_channel_bitmap)) && diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.c b/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.c index f10e856f3c8c..c37343db39df 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.c +++ b/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.c @@ -265,8 +265,8 @@ void ras_log_ring_add_log_event(struct ras_core_context *ras_core, ras_log_ring_add_data(ras_core, log, batch_tag); } -static struct ras_log_info *ras_log_ring_lookup_data(struct ras_core_context *ras_core, - uint64_t idx) +static int ras_log_ring_lookup_data(struct ras_core_context *ras_core, + uint64_t idx, struct ras_log_info *log) { struct ras_log_ring *log_ring = &ras_core->ras_log_ring; unsigned long flags = 0; @@ -274,30 +274,27 @@ static struct ras_log_info *ras_log_ring_lookup_data(struct ras_core_context *ra spin_lock_irqsave(&log_ring->spin_lock, flags); data = radix_tree_lookup(&log_ring->ras_log_root, idx); + if (data) + memcpy(log, data, sizeof(*log)); spin_unlock_irqrestore(&log_ring->spin_lock, flags); - return (struct ras_log_info *)data; + return data ? 0 : -ENODATA; } int ras_log_ring_get_batch_records(struct ras_core_context *ras_core, uint64_t batch_id, - struct ras_log_info **log_arr, uint32_t arr_num) + struct ras_log_info *log_arr, uint32_t arr_num) { struct ras_log_ring *log_ring = &ras_core->ras_log_ring; uint32_t i, idx, count = 0; - void *data; - if ((batch_id >= log_ring->mono_upward_batch_id) || + if (!log_arr || !arr_num || (batch_id >= log_ring->mono_upward_batch_id) || (batch_id < log_ring->last_del_batch_id)) return -EINVAL; - for (i = 0; i < MAX_RECORD_PER_BATCH; i++) { + for (i = 0; i < MAX_RECORD_PER_BATCH && i < arr_num; i++) { idx = BATCH_IDX_TO_TREE_IDX(batch_id, i); - data = ras_log_ring_lookup_data(ras_core, idx); - if (data) { - log_arr[count++] = data; - if (count >= arr_num) - break; - } + if (!ras_log_ring_lookup_data(ras_core, idx, &log_arr[count])) + count++; } return count; diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.h b/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.h index 0ff6cc35678d..cb66beaa9f43 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.h +++ b/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.h @@ -86,7 +86,7 @@ void ras_log_ring_add_log_event(struct ras_core_context *ras_core, enum ras_log_event event, void *data, struct ras_log_batch_tag *tag); int ras_log_ring_get_batch_records(struct ras_core_context *ras_core, uint64_t batch_idx, - struct ras_log_info **log_arr, uint32_t arr_num); + struct ras_log_info *log_arr, uint32_t arr_num); int ras_log_ring_get_batch_overview(struct ras_core_context *ras_core, struct ras_log_batch_overview *overview); diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c index c7ae005ea705..91dd730de3ce 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c +++ b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c @@ -391,9 +391,8 @@ static int ras_umc_update_eeprom_ram_data(struct ras_core_context *ras_core, data->space_left--; } } else { - memcpy(&data->bps[data->count], bps, sizeof(*data->bps)); - data->count++; - data->space_left--; + RAS_DEV_ERR(ras_core->dev, "Failed to convert record to nps pages!"); + return -EINVAL; } return 0; diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c index 3142ef4da7f4..9196f85db9ce 100644 --- a/drivers/gpu/drm/radeon/evergreen_cs.c +++ b/drivers/gpu/drm/radeon/evergreen_cs.c @@ -312,8 +312,10 @@ static int evergreen_surface_check(struct radeon_cs_parser *p, case ARRAY_2D_TILED_THIN1: return evergreen_surface_check_2d(p, surf, prefix); default: - dev_warn(p->dev, "%s:%d %s invalid array mode %d\n", - __func__, __LINE__, prefix, surf->mode); + if (prefix) { + dev_warn(p->dev, "%s:%d %s invalid array mode %d\n", + __func__, __LINE__, prefix, surf->mode); + } return -EINVAL; } return -EINVAL;