From 2111eb4273e7fda0815de85319653104f323c441 Mon Sep 17 00:00:00 2001 From: Danish Khateeb Date: Sat, 26 Sep 2026 10:24:14 -0500 Subject: [PATCH] tools/nolibc: check for overflow in malloc() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit malloc() adds the size of its header to the requested size without checking for overflow. For a size within the header size of SIZE_MAX, the sum wraps around to a few bytes, and malloc() returns a single page instead of NULL. calloc() checks its multiplication for overflow, but then passes the product to malloc(), so calloc(SIZE_MAX, 1) returns a single page too. So does realloc(ptr, SIZE_MAX). Such a size is usually a bug in the caller, for instance a length of -1 used as a size_t, and returning a page instead of NULL can turn it into a heap overflow. Fail with ENOMEM when adding the header overflows, as glibc does for sizes this large. Fixes: 0e0ff638400b ("tools/nolibc/stdlib: Implement `malloc()`, `calloc()`, `realloc()` and `free()`") Assisted-by: LLM Signed-off-by: Danish Khateeb Link: https://patch.msgid.link/20260926152414.251214-3-danishkhateeb03@gmail.com Signed-off-by: Thomas Weißschuh --- tools/include/nolibc/stdlib.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/include/nolibc/stdlib.h b/tools/include/nolibc/stdlib.h index dc554faff22a..cea8290ac0b9 100644 --- a/tools/include/nolibc/stdlib.h +++ b/tools/include/nolibc/stdlib.h @@ -130,7 +130,11 @@ void *malloc(size_t len) { struct nolibc_heap *heap; - len = sizeof(*heap) + len; + if (__builtin_expect(__builtin_add_overflow(len, sizeof(*heap), &len), 0)) { + SET_ERRNO(ENOMEM); + return NULL; + } + heap = mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); if (__builtin_expect(heap == MAP_FAILED, 0))