From be7dacb9c1d48280aeae0830a045f4046c6ef3e2 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Thu, 3 Sep 2026 12:28:27 +0300 Subject: [PATCH] set_memory: add number of pages parameter to set_direct_map APIs Patch series "arch, mm/execmem: resolve confusion about set_direct_map_valid_noflush()", v3. Recent discussion about implementation of execmem's ROX caches on arm64 revealed a confusion about how set_direct_map_valid_noflush() implemented on different architectures. On arm64 it sets or clears the PTE_VALID bit marking a PTE as present or not present. On other architectures it's a range version of set_direct_map_invalid_noflush() and set_direct_map_default_noflush() Unlike arm64::set_direct_map_valid_noflush(), set_direct_map_default_noflush() not only marks PTE as present, but also sets its default protection mode. Other than that, initial design of execmem ROX caches didn't rely on restoration of large mappings that's now available on x86, but completely removed the memory allocated for the ROX cache from the direct map to ensure that large mappings are not split. This precluded usage of VM_FLUSH_RESET_PERMS for the ROX cache allocations and required execmem to implement manipulation of the direct map alias. Current implementation of ROX caches does not remove the direct map alias but simply calls set_memory_rox() that updates the permissions in both vmalloc address space and the direct map and relies on collapse_large_pages() in x86 CPA to keep large mappings. This allow using VM_FLUSH_RESET_PERMS for execmem ROX cache allocations with small adjustments to set_direct_map APIs and vmalloc::reset_perms() behaviour: adding number of pages parameter to set_direct_map APIs and making resetting of the direct map permissions in vmalloc VMAP_HUGE friendly. Implement these adjustments, make execmem always use VM_FLUSH_RESET_PERMS and revert set_direct_map_valid_noflush() changes. This patch (of 6): When set_direct_map APIs were introduced by the commit d253ca0c3865 ("x86/mm/cpa: Add set_direct_map_*() functions") the single page parameter made sense because the initial callers (vmalloc and hibernation) had sets of unsorted struct pages that required changes of their mappings in the direct map. Since there is an increasing demand for direct map manipulation and it is also desirable to be able to update larger physically contiguous mappings, for example an entire large folio, extend set_direct_map APIs to receive number of pages parameter. As there is still only a handful of callers, change the existing functions directly and update all the call sites rather than adding wrappers for single page case. Link: https://lore.kernel.org/20260903-execmem-set-vm-perms-v0-2-v3-0-949b64a9f755@kernel.org Link: https://lore.kernel.org/20260903-execmem-set-vm-perms-v0-2-v3-1-949b64a9f755@kernel.org Link: https://lore.kernel.org/all/20260611130144.1385343-4-abarnas@google.com [1] Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Andrew Morton Acked-by: David Hildenbrand (Arm) Reviewed-by: Kevin Brodsky Cc: Albert Ou Cc: Alexander Gordeev Cc: Alexandre Ghiti Cc: Andy Lutomirski Cc: "Borislav Petkov (AMD)" Cc: Brendan Jackman Cc: Catalin Marinas Cc: Christian Borntraeger Cc: Dave Hansen Cc: Gerald Schaefer Cc: Heiko Carstens Cc: "H. Peter Anvin" Cc: Huacai Chen Cc: Ingo Molnar Cc: Len Brown Cc: Palmer Dabbelt Cc: Peter Zijlstra Cc: "Rafael J. Wysocki" Cc: Ryan Roberts Cc: Sven Schnelle Cc: "Uladzislau Rezki (Sony)" Cc: Vasily Gorbik Cc: WANG Xuerui Cc: Will Deacon Cc: Dev Jain --- arch/arm64/include/asm/set_memory.h | 4 ++-- arch/arm64/mm/pageattr.c | 8 ++++---- arch/loongarch/include/asm/set_memory.h | 4 ++-- arch/loongarch/mm/pageattr.c | 8 ++++---- arch/riscv/include/asm/set_memory.h | 4 ++-- arch/riscv/mm/pageattr.c | 8 ++++---- arch/s390/include/asm/set_memory.h | 4 ++-- arch/s390/mm/pageattr.c | 8 ++++---- arch/x86/include/asm/set_memory.h | 4 ++-- arch/x86/mm/pat/set_memory.c | 8 ++++---- include/linux/set_memory.h | 6 ++++-- kernel/power/snapshot.c | 4 ++-- mm/secretmem.c | 6 +++--- mm/vmalloc.c | 5 +++-- 14 files changed, 42 insertions(+), 39 deletions(-) diff --git a/arch/arm64/include/asm/set_memory.h b/arch/arm64/include/asm/set_memory.h index 90f61b17275e..b07fd4e026ea 100644 --- a/arch/arm64/include/asm/set_memory.h +++ b/arch/arm64/include/asm/set_memory.h @@ -11,8 +11,8 @@ bool can_set_direct_map(void); int set_memory_valid(unsigned long addr, int numpages, int enable); -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages); +int set_direct_map_default_noflush(struct page *page, unsigned int numpages); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index bbe98ac9ad8c..db8d60a84d14 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -251,7 +251,7 @@ int set_memory_valid(unsigned long addr, int numpages, int enable) __pgprot(PTE_PRESENT_VALID_KERNEL)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages) { pgprot_t clear_mask = __pgprot(PTE_PRESENT_VALID_KERNEL); pgprot_t set_mask = __pgprot(PTE_PRESENT_INVALID); @@ -260,10 +260,10 @@ int set_direct_map_invalid_noflush(struct page *page) return 0; return update_range_prot((unsigned long)page_address(page), - PAGE_SIZE, set_mask, clear_mask); + PAGE_SIZE * numpages, set_mask, clear_mask); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int numpages) { pgprot_t set_mask = __pgprot(PTE_PRESENT_VALID_KERNEL | PTE_WRITE); pgprot_t clear_mask = __pgprot(PTE_PRESENT_INVALID | PTE_RDONLY); @@ -272,7 +272,7 @@ int set_direct_map_default_noflush(struct page *page) return 0; return update_range_prot((unsigned long)page_address(page), - PAGE_SIZE, set_mask, clear_mask); + PAGE_SIZE * numpages, set_mask, clear_mask); } static int __set_memory_enc_dec(unsigned long addr, diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h index 55dfaefd02c8..563aab92896e 100644 --- a/arch/loongarch/include/asm/set_memory.h +++ b/arch/loongarch/include/asm/set_memory.h @@ -15,8 +15,8 @@ int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); bool kernel_page_present(struct page *page); -int set_direct_map_default_noflush(struct page *page); -int set_direct_map_invalid_noflush(struct page *page); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); #endif /* _ASM_LOONGARCH_SET_MEMORY_H */ diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c index 614ccc7afccb..43ad2a104f19 100644 --- a/arch/loongarch/mm/pageattr.c +++ b/arch/loongarch/mm/pageattr.c @@ -198,24 +198,24 @@ bool kernel_page_present(struct page *page) return pte_present(ptep_get(pte)); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { unsigned long addr = (unsigned long)page_address(page); if (addr < vm_map_base) return 0; - return __set_memory(addr, 1, PAGE_KERNEL, __pgprot(0)); + return __set_memory(addr, nr, PAGE_KERNEL, __pgprot(0)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { unsigned long addr = (unsigned long)page_address(page); if (addr < vm_map_base) return 0; - return __set_memory(addr, 1, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID)); + return __set_memory(addr, nr, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID)); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index ef59e1716a2c..db1d0ed82b69 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -40,8 +40,8 @@ static inline int set_kernel_memory(char *startp, char *endp, } #endif -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index 3f76db3d2769..20ef95b1d0c3 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -374,15 +374,15 @@ int set_memory_nx(unsigned long addr, int numpages) return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_address(page), 1, + return __set_memory((unsigned long)page_address(page), nr, __pgprot(0), __pgprot(_PAGE_PRESENT)); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_address(page), 1, + return __set_memory((unsigned long)page_address(page), nr, PAGE_KERNEL, __pgprot(_PAGE_EXEC)); } diff --git a/arch/s390/include/asm/set_memory.h b/arch/s390/include/asm/set_memory.h index 94092f4ae764..6b0aa9147ed8 100644 --- a/arch/s390/include/asm/set_memory.h +++ b/arch/s390/include/asm/set_memory.h @@ -60,8 +60,8 @@ __SET_MEMORY_FUNC(set_memory_rox, SET_MEMORY_RO | SET_MEMORY_X) __SET_MEMORY_FUNC(set_memory_rwnx, SET_MEMORY_RW | SET_MEMORY_NX) __SET_MEMORY_FUNC(set_memory_4k, SET_MEMORY_4K) -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 1e202e3d08e7..7549543d6241 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -382,14 +382,14 @@ int __set_memory(unsigned long addr, unsigned long numpages, unsigned long flags return rc; } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_INV); + return __set_memory((unsigned long)page_to_virt(page), nr, SET_MEMORY_INV); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_DEF); + return __set_memory((unsigned long)page_to_virt(page), nr, SET_MEMORY_DEF); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index 4362c26aa992..0c4235d159f4 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -86,8 +86,8 @@ int set_pages_wb(struct page *page, int numpages); int set_pages_ro(struct page *page, int numpages); int set_pages_rw(struct page *page, int numpages); -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 4652487b5572..d36a58ae94db 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -2685,14 +2685,14 @@ static int __set_pages_np(struct page *page, int numpages, unsigned int cpa_flag return __change_page_attr_set_clr(&cpa, 1); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_pages_np(page, 1, 0); + return __set_pages_np(page, nr, 0); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { - return __set_pages_p(page, 1, 0); + return __set_pages_p(page, nr, 0); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h index 3030d9245f5a..0b77f1d7d8b9 100644 --- a/include/linux/set_memory.h +++ b/include/linux/set_memory.h @@ -25,11 +25,13 @@ static inline int set_memory_rox(unsigned long addr, int numpages) #endif #ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP -static inline int set_direct_map_invalid_noflush(struct page *page) +static inline int set_direct_map_invalid_noflush(struct page *page, + unsigned int nr) { return 0; } -static inline int set_direct_map_default_noflush(struct page *page) +static inline int set_direct_map_default_noflush(struct page *page, + unsigned int nr) { return 0; } diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index b209712cb2c3..d5dba0e50b2e 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -88,7 +88,7 @@ static inline int hibernate_restore_unprotect_page(void *page_address) {return 0 static inline void hibernate_map_page(struct page *page) { if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) { - int ret = set_direct_map_default_noflush(page); + int ret = set_direct_map_default_noflush(page, 1); if (ret) pr_warn_once("Failed to remap page\n"); @@ -101,7 +101,7 @@ static inline void hibernate_unmap_page(struct page *page) { if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) { unsigned long addr = (unsigned long)page_address(page); - int ret = set_direct_map_invalid_noflush(page); + int ret = set_direct_map_invalid_noflush(page, 1); if (ret) pr_warn_once("Failed to remap page\n"); diff --git a/mm/secretmem.c b/mm/secretmem.c index 384f5cfc457f..6cbb8efc994a 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -139,7 +139,7 @@ static vm_fault_t secretmem_fault(struct vm_fault *vmf) goto out; } - err = set_direct_map_invalid_noflush(folio_page(folio, 0)); + err = set_direct_map_invalid_noflush(folio_page(folio, 0), 1); if (err) { secretmem_unaccount_folio(state, folio); folio_put(folio); @@ -156,7 +156,7 @@ static vm_fault_t secretmem_fault(struct vm_fault *vmf) * already happened when we marked the page invalid * which guarantees that this call won't fail */ - set_direct_map_default_noflush(folio_page(folio, 0)); + set_direct_map_default_noflush(folio_page(folio, 0), 1); folio_put(folio); if (err == -EEXIST) goto retry; @@ -228,7 +228,7 @@ static int secretmem_migrate_folio(struct address_space *mapping, static void secretmem_free_folio(struct folio *folio) { - set_direct_map_default_noflush(folio_page(folio, 0)); + set_direct_map_default_noflush(folio_page(folio, 0), 1); folio_zero_segment(folio, 0, folio_size(folio)); } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index bea9f76ed7e7..22566e0b6e38 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3358,14 +3358,15 @@ struct vm_struct *remove_vm_area(const void *addr) } static inline void set_area_direct_map(const struct vm_struct *area, - int (*set_direct_map)(struct page *page)) + int (*set_direct_map)(struct page *page, + unsigned int nr)) { unsigned long i; /* HUGE_VMALLOC passes small pages to set_direct_map */ for (i = 0; i < area->nr_pages; i++) if (page_address(area->pages[i])) - set_direct_map(area->pages[i]); + set_direct_map(area->pages[i], 1); } /*