drm/radeon: fix integer overflow in radeon_align_pitch()

radeon_align_pitch() has the same kind of overflow issue as the old
amdgpu helper: both the alignment round-up add and the final
'aligned * cpp' calculation can overflow signed int.

If that wraps, radeon_mode_dumb_create() can end up returning an
invalid pitch or creating a zero-sized dumb buffer.

Fix this by using check_add_overflow() for the alignment round-up and
check_mul_overflow() for the final pitch calculation, returning 0 on
overflow. Also reject zero pitch and size in
radeon_mode_dumb_create().

Found via AST-based call-graph analysis using sqry.

Fixes: ff72145bad ("drm: dumb scanout create/mmap for intel/radeon (v3)")
Signed-off-by: Werner Kasselman <werner@verivus.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
This commit is contained in:
Werner Kasselman 2026-04-15 22:13:52 +00:00 committed by Alex Deucher
parent 06e7f7aa7a
commit ce3b24eb3e

View File

@ -28,6 +28,7 @@
#include <linux/debugfs.h>
#include <linux/iosys-map.h>
#include <linux/overflow.h>
#include <linux/pci.h>
#include <drm/drm_device.h>
@ -812,6 +813,7 @@ int radeon_align_pitch(struct radeon_device *rdev, int width, int cpp, bool tile
int aligned = width;
int align_large = (ASIC_IS_AVIVO(rdev)) || tiled;
int pitch_mask = 0;
int pitch;
switch (cpp) {
case 1:
@ -826,9 +828,12 @@ int radeon_align_pitch(struct radeon_device *rdev, int width, int cpp, bool tile
break;
}
aligned += pitch_mask;
if (check_add_overflow(aligned, pitch_mask, &aligned))
return 0;
aligned &= ~pitch_mask;
return aligned * cpp;
if (check_mul_overflow(aligned, cpp, &pitch))
return 0;
return pitch;
}
int radeon_mode_dumb_create(struct drm_file *file_priv,
@ -842,8 +847,12 @@ int radeon_mode_dumb_create(struct drm_file *file_priv,
args->pitch = radeon_align_pitch(rdev, args->width,
DIV_ROUND_UP(args->bpp, 8), 0);
if (!args->pitch)
return -EINVAL;
args->size = (u64)args->pitch * args->height;
args->size = ALIGN(args->size, PAGE_SIZE);
if (!args->size)
return -EINVAL;
r = radeon_gem_object_create(rdev, args->size, 0,
RADEON_GEM_DOMAIN_VRAM, 0,