diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index aed121d729b0..6e555e31e622 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -38,6 +38,7 @@ struct iov_iter; /* in uio.h */ #define VM_DEFER_KMEMLEAK 0 #endif #define VM_SPARSE 0x00001000 /* sparse vm_area. not all pages are present. */ +#define VM_REQUIRE_HUGE_VMAP 0x00002000 /* huge page mapping or nothing */ /* bits [20..32] reserved for arch specific ioremap internals */ diff --git a/mm/execmem.c b/mm/execmem.c index 00dd6324cae0..77653b7f163d 100644 --- a/mm/execmem.c +++ b/mm/execmem.c @@ -51,7 +51,8 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size, } if (!p) { - pr_warn_ratelimited("unable to allocate memory\n"); + if (!(vm_flags & VM_REQUIRE_HUGE_VMAP)) + pr_warn_ratelimited("unable to allocate memory\n"); return NULL; } @@ -146,9 +147,10 @@ static void execmem_cache_clean(struct work_struct *work) struct vm_struct *vm = find_vm_area(area); size_t size = mas_range_len(&mas); - if (vm && get_vm_area_size(vm) == size && - IS_ALIGNED(size, PMD_SIZE) && - IS_ALIGNED(mas.index, PMD_SIZE)) { + if (vm && get_vm_area_size(vm) == size) { + VM_WARN_ON_ONCE(!IS_ALIGNED(mas.index, PMD_SIZE) || + !IS_ALIGNED(size, PMD_SIZE)); + /* * Preallocate to ensure mas_store does not fail * If there is no memory for the tree update, bail out, @@ -264,38 +266,41 @@ static void *__execmem_cache_alloc(struct execmem_range *range, size_t size) return execmem_cache_alloc_locked(range, size); } -static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size) +static void *execmem_vmalloc_rox(struct execmem_range *range, size_t size, + unsigned long vm_flags) { - unsigned long vm_flags = VM_ALLOW_HUGE_VMAP; - struct mutex *mutex = &execmem_cache.mutex; - struct vm_struct *vm; - size_t alloc_size; - int err = -ENOMEM; - void *p; - - alloc_size = round_up(size, PMD_SIZE); - p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags); - if (!p) { - alloc_size = size; - p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags); - } + void *p = execmem_vmalloc(range, size, PAGE_KERNEL, vm_flags); + int err; if (!p) return NULL; - vm = find_vm_area(p); - if (!vm) - goto err_free_mem; - /* fill memory with instructions that will trap */ - execmem_fill_trapping_insns(p, alloc_size); - + execmem_fill_trapping_insns(p, size); set_vm_flush_reset_perms(p); - - err = set_memory_rox((unsigned long)p, vm->nr_pages); + err = set_memory_rox((unsigned long)p, size >> PAGE_SHIFT); if (err) goto err_free_mem; + return p; + +err_free_mem: + vfree(p); + return NULL; +} + +static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size) +{ + unsigned long vm_flags = VM_REQUIRE_HUGE_VMAP; + size_t alloc_size = round_up(size, PMD_SIZE); + struct mutex *mutex = &execmem_cache.mutex; + int err; + void *p; + + p = execmem_vmalloc_rox(range, alloc_size, vm_flags); + if (!p) + return NULL; + /* * New memory blocks must be allocated and added to the cache * as an atomic operation, otherwise they may be consumed @@ -317,6 +322,11 @@ static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t si return NULL; } +static void *execmem_alloc_rox(struct execmem_range *range, size_t size) +{ + return execmem_vmalloc_rox(range, size, 0); +} + static void *execmem_cache_alloc(struct execmem_range *range, size_t size) { void *p; @@ -444,6 +454,11 @@ static void *execmem_cache_alloc(struct execmem_range *range, size_t size) return NULL; } +static void *execmem_alloc_rox(struct execmem_range *range, size_t size) +{ + return NULL; +} + static bool execmem_cache_free(void *ptr) { return false; @@ -453,17 +468,20 @@ static bool execmem_cache_free(void *ptr) void *execmem_alloc(enum execmem_type type, size_t size) { struct execmem_range *range = &execmem_info->ranges[type]; - bool use_cache = range->flags & EXECMEM_ROX_CACHE; + bool use_rox_cache = range->flags & EXECMEM_ROX_CACHE; unsigned long vm_flags = VM_FLUSH_RESET_PERMS; pgprot_t pgprot = range->pgprot; void *p = NULL; size = PAGE_ALIGN(size); - if (use_cache) + if (use_rox_cache) { p = execmem_cache_alloc(range, size); - else + if (!p) + p = execmem_alloc_rox(range, size); + } else { p = execmem_vmalloc(range, size, pgprot, vm_flags); + } return kasan_reset_tag(p); } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 395358b363fc..3845c7bd72b4 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -4021,6 +4021,12 @@ static gfp_t vmalloc_fix_flags(gfp_t flags) * %__GFP_SKIP_KASAN can be used to skip unpoisoning of mapped pages * (when prot=%PAGE_KERNEL). * + * %VM_ALLOW_HUGE_VMAP allocates huge pages when possible and falls back to + * base pages if huge page allocation fails. + * + * %VM_REQUIRE_HUGE_VMAP implies %VM_ALLOW_HUGE_VMAP and fails instead of + * silently falling back to base pages. + * * Can not be called from interrupt nor NMI contexts. * Return: the address of the area or %NULL on failure */ @@ -4046,6 +4052,10 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, return NULL; } + /* VM_REQUIRE_HUGE_VMAP implies VM_ALLOW_HUGE_VMAP */ + if (vm_flags & VM_REQUIRE_HUGE_VMAP) + vm_flags |= VM_ALLOW_HUGE_VMAP; + if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) { /* * Try huge pages. Only try for PAGE_KERNEL allocations, @@ -4062,6 +4072,9 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, align = max(original_align, 1UL << shift); } + if ((vm_flags & VM_REQUIRE_HUGE_VMAP) && shift == PAGE_SHIFT) + return NULL; + again: area = __get_vm_area_node(size, align, shift, VM_ALLOC | VM_UNINITIALIZED | vm_flags, start, end, node, @@ -4136,7 +4149,7 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, return area->addr; fail: - if (shift > PAGE_SHIFT) { + if (shift > PAGE_SHIFT && !(vm_flags & VM_REQUIRE_HUGE_VMAP)) { shift = PAGE_SHIFT; align = original_align; goto again;