diff --git a/mm/kmsan/core.c b/mm/kmsan/core.c index 90f427b95a21..3ad5f31d56b6 100644 --- a/mm/kmsan/core.c +++ b/mm/kmsan/core.c @@ -27,6 +27,7 @@ #include #include "../slab.h" +#include "../vmalloc.h" #include "kmsan.h" bool kmsan_enabled __read_mostly; @@ -240,11 +241,8 @@ struct page *kmsan_vmalloc_to_page_or_null(void *vaddr) { struct page *page; - if (!kmsan_internal_is_vmalloc_addr(vaddr) && - !kmsan_internal_is_module_addr(vaddr)) - return NULL; - page = vmalloc_to_page(vaddr); - if (pfn_valid(page_to_pfn(page))) + page = __vmalloc_to_page(vaddr); + if (page && pfn_valid(page_to_pfn(page))) return page; else return NULL; diff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c index 5f1b8053f9fa..24f71bc65896 100644 --- a/mm/kmsan/hooks.c +++ b/mm/kmsan/hooks.c @@ -20,6 +20,8 @@ #include #include +#include + #include "../internal.h" #include "../vmalloc.h" #include "../slab.h" @@ -142,6 +144,36 @@ void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end) flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end)); } +#define KMSAN_IOREMAP_META_ORDER 1 + +static void kmsan_iounmap_pages(unsigned long start, unsigned long end) +{ + unsigned long shadow_start = vmalloc_shadow(start), + shadow_end = vmalloc_shadow(end); + unsigned long origin_start = vmalloc_origin(start), + origin_end = vmalloc_origin(end); + unsigned long v_shadow, v_origin; + struct page *shadow, *origin; + int nr; + + nr = (end - start) / PAGE_SIZE; + v_shadow = shadow_start; + v_origin = origin_start; + for (int i = 0; i < nr; + i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) { + shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow); + origin = kmsan_vmalloc_to_page_or_null((void *)v_origin); + if (shadow) + __free_pages(shadow, KMSAN_IOREMAP_META_ORDER); + if (origin) + __free_pages(origin, KMSAN_IOREMAP_META_ORDER); + } + __vunmap_range_noflush(shadow_start, shadow_end); + __vunmap_range_noflush(origin_start, origin_end); + flush_tlb_kernel_range(shadow_start, shadow_end); + flush_tlb_kernel_range(origin_start, origin_end); +} + /* * This function creates new shadow/origin pages for the physical pages mapped * into the virtual memory. If those physical pages already had shadow/origin, @@ -219,28 +251,11 @@ int kmsan_ioremap_page_range(unsigned long start, unsigned long end, void kmsan_iounmap_page_range(unsigned long start, unsigned long end) { - unsigned long v_shadow, v_origin; - struct page *shadow, *origin; - int nr; - if (!kmsan_enabled || kmsan_in_runtime()) return; - nr = (end - start) / PAGE_SIZE; kmsan_enter_runtime(); - v_shadow = (unsigned long)vmalloc_shadow(start); - v_origin = (unsigned long)vmalloc_origin(start); - for (int i = 0; i < nr; - i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) { - shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow); - origin = kmsan_vmalloc_to_page_or_null((void *)v_origin); - __vunmap_range_noflush(v_shadow, vmalloc_shadow(end)); - __vunmap_range_noflush(v_origin, vmalloc_origin(end)); - if (shadow) - __free_pages(shadow, 1); - if (origin) - __free_pages(origin, 1); - } + kmsan_iounmap_pages(start, end); flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end)); flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end)); kmsan_leave_runtime(); diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 81cad5095889..64411d9140c2 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -817,9 +817,10 @@ EXPORT_SYMBOL_GPL(is_vmalloc_or_module_addr); /* * Walk a vmap address to the struct page it maps. Huge vmap mappings will * return the tail page that corresponds to the base page address, which - * matches small vmap mappings. + * matches small vmap mappings. Unlike vmalloc_to_page(), this also accepts + * addresses outside the vmalloc and module ranges, such as KMSAN metadata. */ -struct page *vmalloc_to_page(const void *vmalloc_addr) +__always_inline struct page *__vmalloc_to_page(const void *vmalloc_addr) { unsigned long addr = (unsigned long) vmalloc_addr; struct page *page = NULL; @@ -829,12 +830,6 @@ struct page *vmalloc_to_page(const void *vmalloc_addr) pmd_t *pmd; pte_t *ptep, pte; - /* - * XXX we might need to change this if we add VIRTUAL_BUG_ON for - * architectures that do not vmalloc module space - */ - VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); - if (pgd_none(*pgd)) return NULL; if (WARN_ON_ONCE(pgd_leaf(*pgd))) @@ -873,6 +868,16 @@ struct page *vmalloc_to_page(const void *vmalloc_addr) return page; } + +struct page *vmalloc_to_page(const void *vmalloc_addr) +{ + /* + * XXX we might need to change this if we add VIRTUAL_BUG_ON for + * architectures that do not vmalloc module space + */ + VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); + return __vmalloc_to_page(vmalloc_addr); +} EXPORT_SYMBOL(vmalloc_to_page); /* diff --git a/mm/vmalloc.h b/mm/vmalloc.h index 211869f36509..d8372a4f65c4 100644 --- a/mm/vmalloc.h +++ b/mm/vmalloc.h @@ -8,6 +8,7 @@ #include #ifdef CONFIG_MMU +struct page *__vmalloc_to_page(const void *vmalloc_addr); void __init vmalloc_init(void); int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages,