diff --git a/arch/arm64/include/asm/set_memory.h b/arch/arm64/include/asm/set_memory.h index 90f61b17275e..b07fd4e026ea 100644 --- a/arch/arm64/include/asm/set_memory.h +++ b/arch/arm64/include/asm/set_memory.h @@ -11,8 +11,8 @@ bool can_set_direct_map(void); int set_memory_valid(unsigned long addr, int numpages, int enable); -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages); +int set_direct_map_default_noflush(struct page *page, unsigned int numpages); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index bbe98ac9ad8c..db8d60a84d14 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -251,7 +251,7 @@ int set_memory_valid(unsigned long addr, int numpages, int enable) __pgprot(PTE_PRESENT_VALID_KERNEL)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages) { pgprot_t clear_mask = __pgprot(PTE_PRESENT_VALID_KERNEL); pgprot_t set_mask = __pgprot(PTE_PRESENT_INVALID); @@ -260,10 +260,10 @@ int set_direct_map_invalid_noflush(struct page *page) return 0; return update_range_prot((unsigned long)page_address(page), - PAGE_SIZE, set_mask, clear_mask); + PAGE_SIZE * numpages, set_mask, clear_mask); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int numpages) { pgprot_t set_mask = __pgprot(PTE_PRESENT_VALID_KERNEL | PTE_WRITE); pgprot_t clear_mask = __pgprot(PTE_PRESENT_INVALID | PTE_RDONLY); @@ -272,7 +272,7 @@ int set_direct_map_default_noflush(struct page *page) return 0; return update_range_prot((unsigned long)page_address(page), - PAGE_SIZE, set_mask, clear_mask); + PAGE_SIZE * numpages, set_mask, clear_mask); } static int __set_memory_enc_dec(unsigned long addr, diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h index 55dfaefd02c8..563aab92896e 100644 --- a/arch/loongarch/include/asm/set_memory.h +++ b/arch/loongarch/include/asm/set_memory.h @@ -15,8 +15,8 @@ int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); bool kernel_page_present(struct page *page); -int set_direct_map_default_noflush(struct page *page); -int set_direct_map_invalid_noflush(struct page *page); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); #endif /* _ASM_LOONGARCH_SET_MEMORY_H */ diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c index 614ccc7afccb..43ad2a104f19 100644 --- a/arch/loongarch/mm/pageattr.c +++ b/arch/loongarch/mm/pageattr.c @@ -198,24 +198,24 @@ bool kernel_page_present(struct page *page) return pte_present(ptep_get(pte)); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { unsigned long addr = (unsigned long)page_address(page); if (addr < vm_map_base) return 0; - return __set_memory(addr, 1, PAGE_KERNEL, __pgprot(0)); + return __set_memory(addr, nr, PAGE_KERNEL, __pgprot(0)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { unsigned long addr = (unsigned long)page_address(page); if (addr < vm_map_base) return 0; - return __set_memory(addr, 1, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID)); + return __set_memory(addr, nr, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID)); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index ef59e1716a2c..db1d0ed82b69 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -40,8 +40,8 @@ static inline int set_kernel_memory(char *startp, char *endp, } #endif -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index 3f76db3d2769..20ef95b1d0c3 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -374,15 +374,15 @@ int set_memory_nx(unsigned long addr, int numpages) return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC)); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_address(page), 1, + return __set_memory((unsigned long)page_address(page), nr, __pgprot(0), __pgprot(_PAGE_PRESENT)); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_address(page), 1, + return __set_memory((unsigned long)page_address(page), nr, PAGE_KERNEL, __pgprot(_PAGE_EXEC)); } diff --git a/arch/s390/include/asm/set_memory.h b/arch/s390/include/asm/set_memory.h index 94092f4ae764..6b0aa9147ed8 100644 --- a/arch/s390/include/asm/set_memory.h +++ b/arch/s390/include/asm/set_memory.h @@ -60,8 +60,8 @@ __SET_MEMORY_FUNC(set_memory_rox, SET_MEMORY_RO | SET_MEMORY_X) __SET_MEMORY_FUNC(set_memory_rwnx, SET_MEMORY_RW | SET_MEMORY_NX) __SET_MEMORY_FUNC(set_memory_4k, SET_MEMORY_4K) -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 1e202e3d08e7..7549543d6241 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -382,14 +382,14 @@ int __set_memory(unsigned long addr, unsigned long numpages, unsigned long flags return rc; } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_INV); + return __set_memory((unsigned long)page_to_virt(page), nr, SET_MEMORY_INV); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { - return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_DEF); + return __set_memory((unsigned long)page_to_virt(page), nr, SET_MEMORY_DEF); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index 4362c26aa992..0c4235d159f4 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -86,8 +86,8 @@ int set_pages_wb(struct page *page, int numpages); int set_pages_ro(struct page *page, int numpages); int set_pages_rw(struct page *page, int numpages); -int set_direct_map_invalid_noflush(struct page *page); -int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr); +int set_direct_map_default_noflush(struct page *page, unsigned int nr); int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid); bool kernel_page_present(struct page *page); diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 4652487b5572..d36a58ae94db 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -2685,14 +2685,14 @@ static int __set_pages_np(struct page *page, int numpages, unsigned int cpa_flag return __change_page_attr_set_clr(&cpa, 1); } -int set_direct_map_invalid_noflush(struct page *page) +int set_direct_map_invalid_noflush(struct page *page, unsigned int nr) { - return __set_pages_np(page, 1, 0); + return __set_pages_np(page, nr, 0); } -int set_direct_map_default_noflush(struct page *page) +int set_direct_map_default_noflush(struct page *page, unsigned int nr) { - return __set_pages_p(page, 1, 0); + return __set_pages_p(page, nr, 0); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h index 3030d9245f5a..0b77f1d7d8b9 100644 --- a/include/linux/set_memory.h +++ b/include/linux/set_memory.h @@ -25,11 +25,13 @@ static inline int set_memory_rox(unsigned long addr, int numpages) #endif #ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP -static inline int set_direct_map_invalid_noflush(struct page *page) +static inline int set_direct_map_invalid_noflush(struct page *page, + unsigned int nr) { return 0; } -static inline int set_direct_map_default_noflush(struct page *page) +static inline int set_direct_map_default_noflush(struct page *page, + unsigned int nr) { return 0; } diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index b209712cb2c3..d5dba0e50b2e 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -88,7 +88,7 @@ static inline int hibernate_restore_unprotect_page(void *page_address) {return 0 static inline void hibernate_map_page(struct page *page) { if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) { - int ret = set_direct_map_default_noflush(page); + int ret = set_direct_map_default_noflush(page, 1); if (ret) pr_warn_once("Failed to remap page\n"); @@ -101,7 +101,7 @@ static inline void hibernate_unmap_page(struct page *page) { if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) { unsigned long addr = (unsigned long)page_address(page); - int ret = set_direct_map_invalid_noflush(page); + int ret = set_direct_map_invalid_noflush(page, 1); if (ret) pr_warn_once("Failed to remap page\n"); diff --git a/mm/secretmem.c b/mm/secretmem.c index 384f5cfc457f..6cbb8efc994a 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -139,7 +139,7 @@ static vm_fault_t secretmem_fault(struct vm_fault *vmf) goto out; } - err = set_direct_map_invalid_noflush(folio_page(folio, 0)); + err = set_direct_map_invalid_noflush(folio_page(folio, 0), 1); if (err) { secretmem_unaccount_folio(state, folio); folio_put(folio); @@ -156,7 +156,7 @@ static vm_fault_t secretmem_fault(struct vm_fault *vmf) * already happened when we marked the page invalid * which guarantees that this call won't fail */ - set_direct_map_default_noflush(folio_page(folio, 0)); + set_direct_map_default_noflush(folio_page(folio, 0), 1); folio_put(folio); if (err == -EEXIST) goto retry; @@ -228,7 +228,7 @@ static int secretmem_migrate_folio(struct address_space *mapping, static void secretmem_free_folio(struct folio *folio) { - set_direct_map_default_noflush(folio_page(folio, 0)); + set_direct_map_default_noflush(folio_page(folio, 0), 1); folio_zero_segment(folio, 0, folio_size(folio)); } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index bea9f76ed7e7..22566e0b6e38 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3358,14 +3358,15 @@ struct vm_struct *remove_vm_area(const void *addr) } static inline void set_area_direct_map(const struct vm_struct *area, - int (*set_direct_map)(struct page *page)) + int (*set_direct_map)(struct page *page, + unsigned int nr)) { unsigned long i; /* HUGE_VMALLOC passes small pages to set_direct_map */ for (i = 0; i < area->nr_pages; i++) if (page_address(area->pages[i])) - set_direct_map(area->pages[i]); + set_direct_map(area->pages[i], 1); } /*