From 13d0fdc0901614f0d6c9066193bca6fe63a70c6e Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Thu, 30 Apr 2026 06:38:59 +0100 Subject: [PATCH 01/24] arm64/mm: Replace BUG_ON() with VM_WARN_ON_ONCE() Avoid BUG_ON() while checking for inconsistent page table state conditions and instead replace them with VM_WARN_ON_ONCE(). Cc: Catalin Marinas Cc: Will Deacon Cc: Ryan Roberts Cc: David Hildenbrand Cc: linux-kernel@vger.kernel.org Cc: linux-arm-kernel@lists.infradead.org Acked-by: David Hildenbrand (Arm) Suggested-by: David Hildenbrand (Arm) Signed-off-by: Anshuman Khandual Signed-off-by: Will Deacon --- arch/arm64/include/asm/pgtable.h | 4 ++-- arch/arm64/mm/mmu.c | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 4dfa42b7d053..c9e4e00a9af2 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1007,7 +1007,7 @@ static inline pud_t *p4d_pgtable(p4d_t p4d) static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr) { - BUG_ON(!pgtable_l4_enabled()); + VM_WARN_ON_ONCE(!pgtable_l4_enabled()); return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t); } @@ -1130,7 +1130,7 @@ static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr) static inline phys_addr_t p4d_offset_phys(pgd_t *pgdp, unsigned long addr) { - BUG_ON(!pgtable_l5_enabled()); + VM_WARN_ON_ONCE(!pgtable_l5_enabled()); return pgd_page_paddr(READ_ONCE(*pgdp)) + p4d_index(addr) * sizeof(p4d_t); } diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index dd85e093ffdb..4c8959153ac4 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -273,8 +273,8 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, if (ret) return ret; - BUG_ON(pmd_val(old_pmd) != 0 && - pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); + VM_WARN_ON_ONCE(pmd_val(old_pmd) != 0 && + pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); } phys += next - addr; } while (pmdp++, addr = next, addr != end); @@ -394,8 +394,8 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, if (ret) goto out; - BUG_ON(pud_val(old_pud) != 0 && - pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); + VM_WARN_ON_ONCE(pud_val(old_pud) != 0 && + pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); } phys += next - addr; } while (pudp++, addr = next, addr != end); @@ -445,8 +445,8 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end, if (ret) goto out; - BUG_ON(p4d_val(old_p4d) != 0 && - p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp))); + VM_WARN_ON_ONCE(p4d_val(old_p4d) != 0 && + p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp))); phys += next - addr; } while (p4dp++, addr = next, addr != end); From 36ca7f4be8094d1441fc35e00dd581467badf298 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:52 +0200 Subject: [PATCH 02/24] arm64: mm: Remove bogus stop condition from map_mem() loop The memblock API guarantees that start is not greater than or equal to end, so there is no need to test it. And if it were, it is doubtful that breaking out of the loop would be a reasonable course of action here (rather than attempting to map the remaining regions) So let's drop this check. Reviewed-by: Ryan Roberts Reviewed-by: Kevin Brodsky Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 4c8959153ac4..96ab9e822bb1 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1173,8 +1173,6 @@ static void __init map_mem(pgd_t *pgdp) /* map all the memory banks */ for_each_mem_range(i, &start, &end) { - if (start >= end) - break; /* * The linear map must allow allocation tags reading/writing * if MTE is present. Otherwise, it has the same attributes as From 2e527667a3b9b17833ea82c81bb9da649eb3e50e Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:53 +0200 Subject: [PATCH 03/24] arm64: mm: Drop redundant pgd_t* argument from map_mem() __map_memblock() and map_mem() always operate on swapper_pg_dir, so there is no need to pass around a pgd_t pointer between them. Reviewed-by: Ryan Roberts Reviewed-by: Kevin Brodsky Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 96ab9e822bb1..6cc7f9a51634 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1035,11 +1035,11 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt, flush_tlb_kernel_range(virt, virt + size); } -static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, - phys_addr_t end, pgprot_t prot, int flags) +static void __init __map_memblock(phys_addr_t start, phys_addr_t end, + pgprot_t prot, int flags) { - early_create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, - prot, early_pgtable_alloc, flags); + early_create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), + end - start, prot, early_pgtable_alloc, flags); } void __init mark_linear_text_alias_ro(void) @@ -1087,13 +1087,13 @@ static phys_addr_t __init arm64_kfence_alloc_pool(void) return kfence_pool; } -static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) +static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool) { if (!kfence_pool) return; /* KFENCE pool needs page-level mapping. */ - __map_memblock(pgdp, kfence_pool, kfence_pool + KFENCE_POOL_SIZE, + __map_memblock(kfence_pool, kfence_pool + KFENCE_POOL_SIZE, pgprot_tagged(PAGE_KERNEL), NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE); @@ -1129,11 +1129,11 @@ bool arch_kfence_init_pool(void) #else /* CONFIG_KFENCE */ static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; } -static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) { } +static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool) { } #endif /* CONFIG_KFENCE */ -static void __init map_mem(pgd_t *pgdp) +static void __init map_mem(void) { static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); phys_addr_t kernel_start = __pa_symbol(_text); @@ -1178,7 +1178,7 @@ static void __init map_mem(pgd_t *pgdp) * if MTE is present. Otherwise, it has the same attributes as * PAGE_KERNEL. */ - __map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL), + __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } @@ -1192,10 +1192,9 @@ static void __init map_mem(pgd_t *pgdp) * Note that contiguous mappings cannot be remapped in this way, * so we should avoid them here. */ - __map_memblock(pgdp, kernel_start, kernel_end, - PAGE_KERNEL, NO_CONT_MAPPINGS); + __map_memblock(kernel_start, kernel_end, PAGE_KERNEL, NO_CONT_MAPPINGS); memblock_clear_nomap(kernel_start, kernel_end - kernel_start); - arm64_kfence_map_pool(early_kfence_pool, pgdp); + arm64_kfence_map_pool(early_kfence_pool); } void mark_rodata_ro(void) @@ -1417,7 +1416,7 @@ static void __init create_idmap(void) void __init paging_init(void) { - map_mem(swapper_pg_dir); + map_mem(); memblock_allow_resize(); From 8dd640d9233dd1fdd649920fe1f7fce78a8b40fa Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:54 +0200 Subject: [PATCH 04/24] arm64: mm: Check for pud_/pmd_set_huge() failures on kernel mappings Sashiko reports: | If pmd_set_huge() rejects an unsafe page table transition (such as | mapping a different physical address over an existing block mapping), | it returns 0 and leaves the page table entry unmodified. | | Because *pmdp remains unmodified, READ_ONCE(pmd_val(*pmdp)) will equal | pmd_val(old_pmd). The transition from old_pmd to old_pmd is evaluated | as safe by pgattr_change_is_safe(), so the BUG_ON never triggers. | | This allows invalid and unsafe mapping updates to be silently dropped | instead of panicking, leaving stale memory mappings active while the | caller assumes the update was successful. The same applies to pud_set_huge() in alloc_init_pud(). Given how it is generally preferred to limp on rather than blow up the system if an unexpected condition such as this one occurs, and the fact that there are no known cases where this disparity results in real problems, let's WARN on these failures rather than BUG, allowing the system to survive to the point where it can actually report them. Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 6cc7f9a51634..3b302a0e8f34 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -257,7 +257,7 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, /* try section mapping first */ if (((addr | next | phys) & ~PMD_MASK) == 0 && (flags & NO_BLOCK_MAPPINGS) == 0) { - pmd_set_huge(pmdp, phys, prot); + WARN_ON(!pmd_set_huge(pmdp, phys, prot)); /* * After the PMD entry has been populated once, we @@ -380,7 +380,7 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, if (pud_sect_supported() && ((addr | next | phys) & ~PUD_MASK) == 0 && (flags & NO_BLOCK_MAPPINGS) == 0) { - pud_set_huge(pudp, phys, prot); + WARN_ON(!pud_set_huge(pudp, phys, prot)); /* * After the PUD entry has been populated once, we From a64293e993f6ee6b5dcf107d3a7aa8c6ccca029c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:55 +0200 Subject: [PATCH 05/24] arm64: mm: Preserve existing table mappings when mapping DRAM Instead of blindly overwriting an existing table entry when mapping DRAM regions, take care not to replace a pre-existing table entry with a block entry. This permits the logic of mapping the kernel's linear alias to be simplified in a subsequent patch. Reviewed-by: Ryan Roberts Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 3b302a0e8f34..1db44adc8717 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -256,7 +256,8 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, /* try section mapping first */ if (((addr | next | phys) & ~PMD_MASK) == 0 && - (flags & NO_BLOCK_MAPPINGS) == 0) { + (flags & NO_BLOCK_MAPPINGS) == 0 && + !pmd_table(old_pmd)) { WARN_ON(!pmd_set_huge(pmdp, phys, prot)); /* @@ -379,7 +380,8 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, */ if (pud_sect_supported() && ((addr | next | phys) & ~PUD_MASK) == 0 && - (flags & NO_BLOCK_MAPPINGS) == 0) { + (flags & NO_BLOCK_MAPPINGS) == 0 && + !pud_table(old_pud)) { WARN_ON(!pud_set_huge(pudp, phys, prot)); /* From ecda73ae92cab57611037cec8d29dd6f2ca68fe2 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:56 +0200 Subject: [PATCH 06/24] arm64: mm: Preserve non-contiguous descriptors when mapping DRAM Instead of blindly overwriting existing live entries regardless of the value of their contiguous bit when mapping DRAM regions at contiguous-hint granularity, check whether the contiguous region in question contains any valid descriptors that have the contiguous bit cleared, and in that case, leave the contiguous bit unset on the entire region. This permits the logic of mapping the kernel's linear alias to be simplified in a subsequent patch. Note that this can only result in a misprogrammed contiguous bit (as per ARM ARM RNGLXZ) if the region in question already contains a mix of valid contiguous and valid non-contiguous descriptors, in which case it was already misprogrammed to begin with. Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/include/asm/pgtable.h | 4 ++++ arch/arm64/mm/mmu.c | 22 ++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index c9e4e00a9af2..491ba0a6492d 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -181,6 +181,10 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) * Returns true if the pte is valid and has the contiguous bit set. */ #define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte)) +/* + * Returns true if the pte is valid and has the contiguous bit cleared. + */ +#define pte_valid_noncont(pte) (pte_valid(pte) && !pte_cont(pte)) /* * Could the pte be present in the TLB? We must check mm_tlb_flush_pending * so that we don't erroneously return false for pages that have been diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 1db44adc8717..1f2ec06a8736 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -187,6 +187,14 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end, } while (ptep++, addr += PAGE_SIZE, addr != end); } +static bool pte_range_has_valid_noncont(pte_t *ptep) +{ + for (int i = 0; i < CONT_PTES; i++) + if (pte_valid_noncont(__ptep_get(&ptep[i]))) + return true; + return false; +} + static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, @@ -224,7 +232,8 @@ static int alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, /* use a contiguous mapping if the range is suitably aligned */ if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && - (flags & NO_CONT_MAPPINGS) == 0) + (flags & NO_CONT_MAPPINGS) == 0 && + !pte_range_has_valid_noncont(ptep)) __prot = __pgprot(pgprot_val(prot) | PTE_CONT); init_pte(ptep, addr, next, phys, __prot); @@ -283,6 +292,14 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, return 0; } +static bool pmd_range_has_valid_noncont(pmd_t *pmdp) +{ + for (int i = 0; i < CONT_PMDS; i++) + if (pte_valid_noncont(pmd_pte(READ_ONCE(pmdp[i])))) + return true; + return false; +} + static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, @@ -324,7 +341,8 @@ static int alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, /* use a contiguous mapping if the range is suitably aligned */ if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && - (flags & NO_CONT_MAPPINGS) == 0) + (flags & NO_CONT_MAPPINGS) == 0 && + !pmd_range_has_valid_noncont(pmdp)) __prot = __pgprot(pgprot_val(prot) | PTE_CONT); ret = init_pmd(pmdp, addr, next, phys, __prot, pgtable_alloc, flags); From 05c5c31e9d8d3930d3852018de154bcdb9eeb817 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:57 +0200 Subject: [PATCH 07/24] arm64: mm: Permit contiguous descriptors to be manipulated Currently, pgattr_change_is_safe() is overly pedantic when it comes to descriptors with the contiguous hint attribute set, as it rejects assignments even if the old and the new value are the same. In fact, as per ARM ARM RJQQTC, manipulating descriptors with the contiguous bit set is safe as long as the bit itself does not change value, in the sense that no TLB conflict aborts or other exceptions may be raised as a result. Inconsistent permission attributes within the contiguous region may result in any of the alternatives to be taken to apply to the entire region, which might be a programming error, but it does not constitute an unsafe manipulation in terms of what pgattr_change_is_safe() is intended to detect. So drop the special PTE_CONT check, but still omit PTE_CONT from 'mask' so that modifying the bit is still regarded as unsafe. Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 1f2ec06a8736..da81444015dc 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -134,10 +134,6 @@ bool pgattr_change_is_safe(pteval_t old, pteval_t new) if (pte_pfn(__pte(old)) != pte_pfn(__pte(new))) return false; - /* live contiguous mappings may not be manipulated at all */ - if ((old | new) & PTE_CONT) - return false; - /* Transitioning from Non-Global to Global is unsafe */ if (old & ~new & PTE_NG) return false; From dfd73e574d389f3df4e6e39157b22e71a978db34 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:58 +0200 Subject: [PATCH 08/24] arm64: kfence: Avoid NOMAP tricks when mapping the early pool Now that the map_mem() routines respect existing page mappings and contiguous granule sized blocks with the contiguous bit cleared, there is no longer a reason to play tricks with the memblock NOMAP attribute. Instead, the kfence pool can be allocated and mapped with page granularity first, and this granularity will be respected when the rest of DRAM is mapped later, even if block and contiguous mappings are allowed for the remainder of those mappings. Add the NO_EXEC_MAPPINGS flag to ensure that hierarchical XN attributes are set on the intermediate page tables that are allocated when mapping the pool. Signed-off-by: Ard Biesheuvel Reviewed-by: Kevin Brodsky Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index da81444015dc..661ff3c1fe10 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1083,36 +1083,24 @@ static int __init parse_kfence_early_init(char *arg) } early_param("kfence.sample_interval", parse_kfence_early_init); -static phys_addr_t __init arm64_kfence_alloc_pool(void) +static void __init arm64_kfence_map_pool(void) { phys_addr_t kfence_pool; if (!kfence_early_init) - return 0; + return; kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE); if (!kfence_pool) { pr_err("failed to allocate kfence pool\n"); kfence_early_init = false; - return 0; - } - - /* Temporarily mark as NOMAP. */ - memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE); - - return kfence_pool; -} - -static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool) -{ - if (!kfence_pool) return; + } /* KFENCE pool needs page-level mapping. */ __map_memblock(kfence_pool, kfence_pool + KFENCE_POOL_SIZE, pgprot_tagged(PAGE_KERNEL), - NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); - memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE); + NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS | NO_EXEC_MAPPINGS); __kfence_pool = phys_to_virt(kfence_pool); } @@ -1144,8 +1132,7 @@ bool arch_kfence_init_pool(void) } #else /* CONFIG_KFENCE */ -static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; } -static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool) { } +static inline void arm64_kfence_map_pool(void) { } #endif /* CONFIG_KFENCE */ @@ -1155,7 +1142,6 @@ static void __init map_mem(void) phys_addr_t kernel_start = __pa_symbol(_text); phys_addr_t kernel_end = __pa_symbol(__init_begin); phys_addr_t start, end; - phys_addr_t early_kfence_pool; int flags = NO_EXEC_MAPPINGS; u64 i; @@ -1172,7 +1158,7 @@ static void __init map_mem(void) BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end) && pgd_index(_PAGE_OFFSET(VA_BITS_MIN)) != PTRS_PER_PGD - 1); - early_kfence_pool = arm64_kfence_alloc_pool(); + arm64_kfence_map_pool(); linear_map_requires_bbml2 = !force_pte_mapping() && can_set_direct_map(); @@ -1210,7 +1196,6 @@ static void __init map_mem(void) */ __map_memblock(kernel_start, kernel_end, PAGE_KERNEL, NO_CONT_MAPPINGS); memblock_clear_nomap(kernel_start, kernel_end - kernel_start); - arm64_kfence_map_pool(early_kfence_pool); } void mark_rodata_ro(void) From 28becb2c1d741259b2f12737e0a3828fe59700d9 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:01:59 +0200 Subject: [PATCH 09/24] arm64: mm: Permit contiguous attribute for preliminary mappings There are a few cases where we omit the contiguous hint for mappings that start out as read-write and are remapped read-only later, on the basis that manipulating live descriptors with the PTE_CONT attribute set is unsafe. When support for the contiguous hint was added to the code, the ARM ARM was ambiguous about this, and so we erred on the side of caution. In the meantime, this has been clarified [0], and regions that will be remapped in their entirety, retaining the contiguous bit on all entries, can use the contiguous hint both in the initial mapping as well as the one that replaces it. Note that this requires that the logic that may be called to remap overlapping regions respects existing valid descriptors that have the contiguous bit cleared. So omit the NO_CONT_MAPPINGS flag in places where it is unneeded. [0] RJQQTC For a TLB lookup in a contiguous region mapped by translation table entries that have consistent values for the Contiguous bit, but have the OA, attributes, or permissions misprogrammed, that TLB lookup is permitted to produce an OA, access permissions, and memory attributes that are consistent with any one of the programmed translation table values. Reviewed-by: Kevin Brodsky Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 661ff3c1fe10..ff9e74045308 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1016,8 +1016,7 @@ void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, &phys, virt); return; } - early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, - NO_CONT_MAPPINGS); + early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0); } void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, @@ -1044,8 +1043,7 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt, return; } - early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, - NO_CONT_MAPPINGS); + early_create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, 0); /* flush the TLBs after updating live kernel mappings */ flush_tlb_kernel_range(virt, virt + size); @@ -1191,10 +1189,8 @@ static void __init map_mem(void) * alternative patching has completed). This makes the contents * of the region accessible to subsystems such as hibernate, * but protects it from inadvertent modification or execution. - * Note that contiguous mappings cannot be remapped in this way, - * so we should avoid them here. */ - __map_memblock(kernel_start, kernel_end, PAGE_KERNEL, NO_CONT_MAPPINGS); + __map_memblock(kernel_start, kernel_end, PAGE_KERNEL, 0); memblock_clear_nomap(kernel_start, kernel_end - kernel_start); } From 382a03e12ebad387fad616da78b99720ea3ee683 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:00 +0200 Subject: [PATCH 10/24] arm64: Move fixmap and kasan page tables to end of kernel image Move the fixmap and kasan page tables out of the BSS section, and place them at the end of the image, right before the init_pg_dir section where some of the other statically allocated page tables live. These page tables are currently the only data objects in vmlinux that are meant to be accessed via the kernel image's linear alias, and so placing them together allows the remainder of the data/bss section to be remapped read-only or unmapped entirely. Reviewed-by: Kevin Brodsky Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/include/asm/mmu.h | 2 ++ arch/arm64/kernel/vmlinux.lds.S | 8 +++++++- arch/arm64/mm/fixmap.c | 6 +++--- arch/arm64/mm/kasan_init.c | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 5e1211c540ab..fb95754f2876 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -13,6 +13,8 @@ #ifndef __ASSEMBLER__ +#define __pgtbl_bss __section(".pgdir.bss") __aligned(PAGE_SIZE) + #include #include diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index e1ac876200a3..2b0ebfb30c63 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -349,9 +349,15 @@ SECTIONS _edata = .; /* start of zero-init region */ - BSS_SECTION(SBSS_ALIGN, 0, 0) + BSS_SECTION(SBSS_ALIGN, 0, PAGE_SIZE) __pi___bss_start = __bss_start; + /* fixmap BSS starts here - preceding data/BSS is omitted from the linear map */ + .pgdir.bss (NOLOAD) : ALIGN(PAGE_SIZE) { + *(.pgdir.bss) + } + ASSERT(ADDR(.pgdir.bss) == __bss_stop, ".pgdir.bss must follow BSS") + . = ALIGN(PAGE_SIZE); __pi_init_pg_dir = .; . += INIT_DIR_SIZE; diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index c5c5425791da..1a3bbd67dd76 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -31,9 +31,9 @@ static_assert(NR_BM_PMD_TABLES == 1); #define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) -static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss; -static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; -static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; +static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __pgtbl_bss; +static pmd_t bm_pmd[PTRS_PER_PMD] __pgtbl_bss __maybe_unused; +static pud_t bm_pud[PTRS_PER_PUD] __pgtbl_bss __maybe_unused; static inline pte_t *fixmap_pte(unsigned long addr) { diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index abeb81bf6ebd..dbf22cae82ee 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -214,7 +214,7 @@ asmlinkage void __init kasan_early_init(void) * shadow pud_t[]/p4d_t[], which could end up getting corrupted * when the linear region is mapped. */ - static pte_t tbl[PTRS_PER_PTE] __page_aligned_bss; + static pte_t tbl[PTRS_PER_PTE] __pgtbl_bss; pgd_t *pgdp = pgd_offset_k(KASAN_SHADOW_START); set_pgd(pgdp, __pgd(__pa_symbol(tbl) | PGD_TYPE_TABLE)); From d672a4b72c9542911244b0bc5d5c14c77251159c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:01 +0200 Subject: [PATCH 11/24] arm64: mm: Don't abuse memblock NOMAP to check for overlaps Now that the linear region mapping routines respect existing table mappings and contiguous block and page mappings, it is no longer needed to fiddle with the memblock tables to set and clear the NOMAP attribute in order to omit text and rodata when creating the linear map. Instead, map the kernel text and rodata alias first with the desired initial attributes and granularity, so that the loop iterating over the memblocks will not remap it in a manner that prevents it from being remapped with updated attributes later. Signed-off-by: Ard Biesheuvel Reviewed-by: Kevin Brodsky Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index ff9e74045308..6b6b2dac41f5 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1164,12 +1164,17 @@ static void __init map_mem(void) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; /* - * Take care not to create a writable alias for the - * read-only text and rodata sections of the kernel image. - * So temporarily mark them as NOMAP to skip mappings in - * the following for-loop + * Map the linear alias of the [_text, __init_begin) interval first + * so that its write permissions can be removed later without the need + * to split any block mappings created by the loop below. + * + * Write permissions are needed for alternatives patching, and will be + * removed later by mark_linear_text_alias_ro() above. This makes the + * contents of the region accessible to subsystems such as hibernate, + * but protects it from inadvertent modification or execution. */ - memblock_mark_nomap(kernel_start, kernel_end - kernel_start); + __map_memblock(kernel_start, kernel_end, pgprot_tagged(PAGE_KERNEL), + flags); /* map all the memory banks */ for_each_mem_range(i, &start, &end) { @@ -1181,17 +1186,6 @@ static void __init map_mem(void) __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } - - /* - * Map the linear alias of the [_text, __init_begin) interval - * as non-executable now, and remove the write permission in - * mark_linear_text_alias_ro() below (which will be called after - * alternative patching has completed). This makes the contents - * of the region accessible to subsystems such as hibernate, - * but protects it from inadvertent modification or execution. - */ - __map_memblock(kernel_start, kernel_end, PAGE_KERNEL, 0); - memblock_clear_nomap(kernel_start, kernel_end - kernel_start); } void mark_rodata_ro(void) From c0693153fb17069f536548396c5a7946692b8948 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:02 +0200 Subject: [PATCH 12/24] powerpc/code-patching: Avoid r/w mapping of the zero page The only remaining use of map_patch_area() is mapping the zero page, and immediately unmapping it again so that the intermediate page table levels are all guaranteed to be populated. The use of the zero page here is completely arbitrary, and not harmful per se, but currently, it creates a writable mapping, and does so in a manner that requires that the empty_zero_page[] symbol is not const-qualified. Given that this is about to change, and that map_patch_area() now never maps anything other than the zero page, let's simplify the code and - remove the helpers and call [un]map_kernel_page() directly - take the PA of empty_zero_page directly - create a read-only temporary mapping. This allows empty_zero_page[] to be repainted as const u8[] in a subsequent patch, without making substantial changes to this code patching logic. Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Christophe Leroy (CS GROUP) Link: https://lore.kernel.org/all/20260520085423.485402-1-ardb@kernel.org/ Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/powerpc/lib/code-patching.c | 52 ++------------------------------ 1 file changed, 2 insertions(+), 50 deletions(-) diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c index f84e0337cc02..44ff9f684bef 100644 --- a/arch/powerpc/lib/code-patching.c +++ b/arch/powerpc/lib/code-patching.c @@ -60,9 +60,6 @@ struct patch_context { static DEFINE_PER_CPU(struct patch_context, cpu_patching_context); -static int map_patch_area(void *addr, unsigned long text_poke_addr); -static void unmap_patch_area(unsigned long addr); - static bool mm_patch_enabled(void) { return IS_ENABLED(CONFIG_SMP) && radix_enabled(); @@ -117,11 +114,11 @@ static int text_area_cpu_up(unsigned int cpu) // Map/unmap the area to ensure all page tables are pre-allocated addr = (unsigned long)area->addr; - err = map_patch_area(empty_zero_page, addr); + err = map_kernel_page(addr, __pa_symbol(empty_zero_page), PAGE_KERNEL_RO); if (err) return err; - unmap_patch_area(addr); + unmap_kernel_page(addr); this_cpu_write(cpu_patching_context.area, area); this_cpu_write(cpu_patching_context.addr, addr); @@ -233,51 +230,6 @@ static unsigned long get_patch_pfn(void *addr) return __pa_symbol(addr) >> PAGE_SHIFT; } -/* - * This can be called for kernel text or a module. - */ -static int map_patch_area(void *addr, unsigned long text_poke_addr) -{ - unsigned long pfn = get_patch_pfn(addr); - - return map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL); -} - -static void unmap_patch_area(unsigned long addr) -{ - pte_t *ptep; - pmd_t *pmdp; - pud_t *pudp; - p4d_t *p4dp; - pgd_t *pgdp; - - pgdp = pgd_offset_k(addr); - if (WARN_ON(pgd_none(*pgdp))) - return; - - p4dp = p4d_offset(pgdp, addr); - if (WARN_ON(p4d_none(*p4dp))) - return; - - pudp = pud_offset(p4dp, addr); - if (WARN_ON(pud_none(*pudp))) - return; - - pmdp = pmd_offset(pudp, addr); - if (WARN_ON(pmd_none(*pmdp))) - return; - - ptep = pte_offset_kernel(pmdp, addr); - if (WARN_ON(pte_none(*ptep))) - return; - - /* - * In hash, pte_clear flushes the tlb, in radix, we have to - */ - pte_clear(&init_mm, addr, ptep); - flush_tlb_kernel_range(addr, addr + PAGE_SIZE); -} - static int __do_patch_mem_mm(void *addr, unsigned long val, bool is_dword) { int err; From 99bad3e992e26addd2a7d779cfbe42c048abd808 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:03 +0200 Subject: [PATCH 13/24] sh: Drop cache flush of the zero page at boot SuperH performs cache maintenance on the zero page during boot, presumably because before commit 6215d9f4470f ("arch, mm: consolidate empty_zero_page") the zero page did double duty as a boot params region, and was cleared separately, as it was not part of BSS. The memset() in question was dropped by that commit, but the __flush_wback_region() call remained. As empty_zero_page[] has been moved to BSS, it can be treated as any other BSS memory, and so the cache flush can be dropped. Cc: Yoshinori Sato Cc: Rich Felker Cc: John Paul Adrian Glaubitz Cc: Mike Rapoport Cc: Geert Uytterhoeven Signed-off-by: Ard Biesheuvel Reviewed-by: Geert Uytterhoeven Acked-by: Mike Rapoport (Microsoft) Signed-off-by: Will Deacon --- arch/sh/mm/init.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 4e40d5e96be9..110308bdef01 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -331,9 +331,6 @@ void __init mem_init(void) /* Set this up early, so we can take care of the zero page */ cpu_cache_init(); - /* clear the zero-page */ - __flush_wback_region(empty_zero_page, PAGE_SIZE); - vsyscall_init(); pr_info("virtual kernel memory layout:\n" From 0aae825f1ed7ce3eedfadc54684aa86bbbe188b0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:04 +0200 Subject: [PATCH 14/24] mm: Make empty_zero_page[] const The empty zero page is used to back any kernel or user space mapping that is supposed to remain cleared, and so the page itself is never supposed to be modified. So mark it as const, which moves it into .rodata rather than .bss: on most architectures, this ensures that both the kernel's mapping of it and any aliases that are accessible via the kernel direct (linear) map are mapped read-only, and cannot be used (inadvertently or maliciously) to corrupt the contents of the zero page. Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Kevin Brodsky Acked-by: David Hildenbrand (Arm) Reviewed-by: Jann Horn Reviewed-by: Feng Tang Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- include/linux/pgtable.h | 2 +- mm/mm_init.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index cdd68ed3ae1a..67aa23814010 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1993,7 +1993,7 @@ static inline unsigned long zero_pfn(unsigned long addr) return zero_page_pfn; } -extern uint8_t empty_zero_page[PAGE_SIZE]; +extern const uint8_t empty_zero_page[PAGE_SIZE]; extern struct page *__zero_page; static inline struct page *_zero_page(unsigned long addr) diff --git a/mm/mm_init.c b/mm/mm_init.c index f9f8e1af921c..46cf001238c5 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -57,7 +57,7 @@ unsigned long zero_page_pfn __ro_after_init; EXPORT_SYMBOL(zero_page_pfn); #ifndef __HAVE_COLOR_ZERO_PAGE -uint8_t empty_zero_page[PAGE_SIZE] __page_aligned_bss; +const uint8_t empty_zero_page[PAGE_SIZE] __aligned(PAGE_SIZE); EXPORT_SYMBOL(empty_zero_page); struct page *__zero_page __ro_after_init; From f2ba877402e5f74b27d9dbc2c8d059e7e9daf500 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:05 +0200 Subject: [PATCH 15/24] arm64: mm: Map the kernel data/bss read-only in the linear map On systems where the bootloader adheres to the original arm64 boot protocol, the placement of the kernel in the physical address space is highly predictable, and this makes the placement of its linear alias in the kernel virtual address space equally predictable, given the lack of randomization of the linear map. The linear aliases of the kernel text and rodata regions are already mapped read-only, but the kernel data and bss are mapped read-write in this region. This is not needed, so map them read-only as well. Note that the statically allocated kernel page tables do need to be modifiable via the linear map, so leave these mapped read-write. Reviewed-by: Kevin Brodsky Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 6b6b2dac41f5..7f7693a5dfff 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1138,7 +1138,9 @@ static void __init map_mem(void) { static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); phys_addr_t kernel_start = __pa_symbol(_text); - phys_addr_t kernel_end = __pa_symbol(__init_begin); + phys_addr_t init_begin = __pa_symbol(__init_begin); + phys_addr_t init_end = __pa_symbol(__init_end); + phys_addr_t kernel_end = __pa_symbol(__bss_stop); phys_addr_t start, end; int flags = NO_EXEC_MAPPINGS; u64 i; @@ -1173,7 +1175,11 @@ static void __init map_mem(void) * contents of the region accessible to subsystems such as hibernate, * but protects it from inadvertent modification or execution. */ - __map_memblock(kernel_start, kernel_end, pgprot_tagged(PAGE_KERNEL), + __map_memblock(kernel_start, init_begin, pgprot_tagged(PAGE_KERNEL), + flags); + + /* Map the kernel data/bss so it can be remapped later */ + __map_memblock(init_end, kernel_end, pgprot_tagged(PAGE_KERNEL), flags); /* map all the memory banks */ @@ -1186,6 +1192,11 @@ static void __init map_mem(void) __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } + + /* Map the kernel data/bss read-only in the linear map */ + __map_memblock(init_end, kernel_end, PAGE_KERNEL_RO, flags); + flush_tlb_kernel_range((unsigned long)lm_alias(__init_end), + (unsigned long)lm_alias(__bss_stop)); } void mark_rodata_ro(void) From 63e0b6a5b6934d6a919d1c65ea185303200a1874 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:06 +0200 Subject: [PATCH 16/24] arm64: mm: Unmap kernel data/bss entirely from the linear map The linear aliases of the kernel text and rodata are also mapped read-only in the linear map. Given that the contents of these regions are mostly identical to the version in the loadable image, mapping them read-only and leaving their contents visible is a reasonable hardening measure. Data and bss, however, are now also mapped read-only but the contents of these regions are more likely to contain data that we'd rather not leak. So let's unmap these entirely in the linear map when the kernel is running normally. When going into hibernation or waking up from it, these regions need to be mapped, so map the region initially, and toggle the valid bit so map/unmap the region as needed. Doing so is required because pages covering the kernel image are marked as PageReserved, and therefore disregarded for snapshotting by the hibernate logic unless they are mapped. Signed-off-by: Ard Biesheuvel Reviewed-by: Kevin Brodsky Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 45 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 7f7693a5dfff..be51f6cac86f 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1056,6 +1057,29 @@ static void __init __map_memblock(phys_addr_t start, phys_addr_t end, end - start, prot, early_pgtable_alloc, flags); } +static void mark_linear_data_alias_valid(bool valid) +{ + set_memory_valid((unsigned long)lm_alias(__init_end), + (unsigned long)(__bss_stop - __init_end) / PAGE_SIZE, + valid); +} + +static int arm64_hibernate_pm_notify(struct notifier_block *nb, + unsigned long mode, void *unused) +{ + switch (mode) { + default: + break; + case PM_POST_HIBERNATION: + mark_linear_data_alias_valid(false); + break; + case PM_HIBERNATION_PREPARE: + mark_linear_data_alias_valid(true); + break; + } + return 0; +} + void __init mark_linear_text_alias_ro(void) { /* @@ -1064,6 +1088,21 @@ void __init mark_linear_text_alias_ro(void) update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text), (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); + + /* + * Register a PM notifier to remap the linear alias of data/bss as + * valid read-only before hibernation. This is needed because the + * snapshot logic disregards PageReserved pages (such as the ones + * covering the kernel image) unless they are mapped in the linear + * map. + */ + if (IS_ENABLED(CONFIG_HIBERNATION)) { + static struct notifier_block nb = { + .notifier_call = arm64_hibernate_pm_notify + }; + + register_pm_notifier(&nb); + } } #ifdef CONFIG_KFENCE @@ -1193,10 +1232,8 @@ static void __init map_mem(void) flags); } - /* Map the kernel data/bss read-only in the linear map */ - __map_memblock(init_end, kernel_end, PAGE_KERNEL_RO, flags); - flush_tlb_kernel_range((unsigned long)lm_alias(__init_end), - (unsigned long)lm_alias(__bss_stop)); + /* Map the kernel data/bss as invalid in the linear map */ + mark_linear_data_alias_valid(false); } void mark_rodata_ro(void) From 9c401fa7398fc2362b57e7d8b2ea1ab440d8b39f Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 4 Jun 2026 17:11:53 +0200 Subject: [PATCH 17/24] arm64: Rename page table BSS section to .bss..pgtbl Rename the .pgdir.bss section to .bss..pgtbl so that the compiler will notice the leading ".bss" and mark it as NOBITS by default (rather than PROGBITS, which would take up space in Image binary, forcing all of the preceding BSS to be emitted into the image as well). This supersedes the NOLOAD linker directive, which achieves the same thing, and can be therefore be dropped. Also, rename .pgdir to .pgtbl to be more generic, as page tables of various levels will reside here. Signed-off-by: Ard Biesheuvel Tested-by: Mark Brown Signed-off-by: Will Deacon --- arch/arm64/include/asm/linkage.h | 2 ++ arch/arm64/include/asm/mmu.h | 2 -- arch/arm64/kernel/vmlinux.lds.S | 8 ++++---- arch/arm64/mm/fixmap.c | 6 +++--- arch/arm64/mm/kasan_init.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h index 40bd17add539..8637f667667c 100644 --- a/arch/arm64/include/asm/linkage.h +++ b/arch/arm64/include/asm/linkage.h @@ -43,4 +43,6 @@ SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ bti c ; +#define __bss_pgtbl __section(".bss..pgtbl") __aligned(PAGE_SIZE) + #endif diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index fb95754f2876..5e1211c540ab 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -13,8 +13,6 @@ #ifndef __ASSEMBLER__ -#define __pgtbl_bss __section(".pgdir.bss") __aligned(PAGE_SIZE) - #include #include diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 2b0ebfb30c63..d3ed59abab38 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -352,11 +352,11 @@ SECTIONS BSS_SECTION(SBSS_ALIGN, 0, PAGE_SIZE) __pi___bss_start = __bss_start; - /* fixmap BSS starts here - preceding data/BSS is omitted from the linear map */ - .pgdir.bss (NOLOAD) : ALIGN(PAGE_SIZE) { - *(.pgdir.bss) + /* page table BSS starts here - preceding data/BSS is omitted from the linear map */ + .pgtbl : ALIGN(PAGE_SIZE) { + *(.bss..pgtbl) } - ASSERT(ADDR(.pgdir.bss) == __bss_stop, ".pgdir.bss must follow BSS") + ASSERT(ADDR(.pgtbl) == __bss_stop, ".pgtbl must follow BSS") . = ALIGN(PAGE_SIZE); __pi_init_pg_dir = .; diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index 1a3bbd67dd76..f66a0016dd02 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -31,9 +31,9 @@ static_assert(NR_BM_PMD_TABLES == 1); #define BM_PTE_TABLE_IDX(addr) __BM_TABLE_IDX(addr, PMD_SHIFT) -static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __pgtbl_bss; -static pmd_t bm_pmd[PTRS_PER_PMD] __pgtbl_bss __maybe_unused; -static pud_t bm_pud[PTRS_PER_PUD] __pgtbl_bss __maybe_unused; +static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __bss_pgtbl; +static pmd_t bm_pmd[PTRS_PER_PMD] __bss_pgtbl __maybe_unused; +static pud_t bm_pud[PTRS_PER_PUD] __bss_pgtbl __maybe_unused; static inline pte_t *fixmap_pte(unsigned long addr) { diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index dbf22cae82ee..3fcad956fdf7 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -214,7 +214,7 @@ asmlinkage void __init kasan_early_init(void) * shadow pud_t[]/p4d_t[], which could end up getting corrupted * when the linear region is mapped. */ - static pte_t tbl[PTRS_PER_PTE] __pgtbl_bss; + static pte_t tbl[PTRS_PER_PTE] __bss_pgtbl; pgd_t *pgdp = pgd_offset_k(KASAN_SHADOW_START); set_pgd(pgdp, __pgd(__pa_symbol(tbl) | PGD_TYPE_TABLE)); From 9f7f685758c6988a6076d4a494d20e99337b79ed Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 4 Jun 2026 17:11:54 +0200 Subject: [PATCH 18/24] kasan: Move generic KASAN page tables out of BSS too Make sure that all KASAN page tables are emitted into the .pgtbl section (provided that the arch has one - otherwise, fall back to page aligned BSS) This is needed because BSS itself is no longer accessible via the linear map on arm64. Cc: Andrey Ryabinin Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Dmitry Vyukov Cc: Vincenzo Frascino Cc: kasan-dev@googlegroups.com Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- include/linux/linkage.h | 4 ++++ mm/kasan/init.c | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/include/linux/linkage.h b/include/linux/linkage.h index b11660b706c5..53fe1f48fd28 100644 --- a/include/linux/linkage.h +++ b/include/linux/linkage.h @@ -39,6 +39,10 @@ #define __page_aligned_data __section(".data..page_aligned") __aligned(PAGE_SIZE) #define __page_aligned_bss __section(".bss..page_aligned") __aligned(PAGE_SIZE) +#ifndef __bss_pgtbl +#define __bss_pgtbl __page_aligned_bss +#endif + /* * For assembly routines. * diff --git a/mm/kasan/init.c b/mm/kasan/init.c index 9c880f607c6a..66a883887987 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -26,10 +26,10 @@ * - Latter it reused it as zero shadow to cover large ranges of memory * that allowed to access, but not handled by kasan (vmalloc/vmemmap ...). */ -unsigned char kasan_early_shadow_page[PAGE_SIZE] __page_aligned_bss; +unsigned char kasan_early_shadow_page[PAGE_SIZE] __bss_pgtbl; #if CONFIG_PGTABLE_LEVELS > 4 -p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __page_aligned_bss; +p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __bss_pgtbl; static inline bool kasan_p4d_table(pgd_t pgd) { return pgd_page(pgd) == virt_to_page(lm_alias(kasan_early_shadow_p4d)); @@ -41,7 +41,7 @@ static inline bool kasan_p4d_table(pgd_t pgd) } #endif #if CONFIG_PGTABLE_LEVELS > 3 -pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __page_aligned_bss; +pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __bss_pgtbl; static inline bool kasan_pud_table(p4d_t p4d) { return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud)); @@ -53,7 +53,7 @@ static inline bool kasan_pud_table(p4d_t p4d) } #endif #if CONFIG_PGTABLE_LEVELS > 2 -pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __page_aligned_bss; +pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __bss_pgtbl; static inline bool kasan_pmd_table(pud_t pud) { return pud_page(pud) == virt_to_page(lm_alias(kasan_early_shadow_pmd)); @@ -65,7 +65,7 @@ static inline bool kasan_pmd_table(pud_t pud) } #endif pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS] - __page_aligned_bss; + __bss_pgtbl; static inline bool kasan_pte_table(pmd_t pmd) { From 568def8e87fc666682ec5a74713983ac32335213 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 4 Jun 2026 17:11:55 +0200 Subject: [PATCH 19/24] arm64: Avoid double evaluation of __ptep_get() Sashiko warns that the new pte_valid_noncont() macro is used in a manner where the argument (which performs a READ_ONCE() of the descriptor) is evaluated twice. Drop the macro that we just added, and move the check into the newly added users. Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/include/asm/pgtable.h | 4 ---- arch/arm64/mm/mmu.c | 14 ++++++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 491ba0a6492d..c9e4e00a9af2 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -181,10 +181,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) * Returns true if the pte is valid and has the contiguous bit set. */ #define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte)) -/* - * Returns true if the pte is valid and has the contiguous bit cleared. - */ -#define pte_valid_noncont(pte) (pte_valid(pte) && !pte_cont(pte)) /* * Could the pte be present in the TLB? We must check mm_tlb_flush_pending * so that we don't erroneously return false for pages that have been diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index be51f6cac86f..d68e691c093a 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -186,9 +186,12 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end, static bool pte_range_has_valid_noncont(pte_t *ptep) { - for (int i = 0; i < CONT_PTES; i++) - if (pte_valid_noncont(__ptep_get(&ptep[i]))) + for (int i = 0; i < CONT_PTES; i++) { + pte_t pte = __ptep_get(&ptep[i]); + + if (pte_valid(pte) && !pte_cont(pte)) return true; + } return false; } @@ -291,9 +294,12 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end, static bool pmd_range_has_valid_noncont(pmd_t *pmdp) { - for (int i = 0; i < CONT_PMDS; i++) - if (pte_valid_noncont(pmd_pte(READ_ONCE(pmdp[i])))) + for (int i = 0; i < CONT_PMDS; i++) { + pte_t pte = pmd_pte(READ_ONCE(pmdp[i])); + + if (pte_valid(pte) && !pte_cont(pte)) return true; + } return false; } From 2986a625740599fe6e7635b0586fed2a95bcd1f7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 4 Jun 2026 17:11:56 +0200 Subject: [PATCH 20/24] KVM: arm64: Omit tag sync on stage-2 mappings of the zero page Commit f620d66af316 ("arm64: mte: Do not flag the zero page as PG_mte_tagged") removed the PG_mte_tagged flag from the zero page, but missed a KVM code path that may set this flag on the zero page when it is used in a stage-2 CoW mapping of anonymous memory. So disregard the zero page explicitly in sanitise_mte_tags(). Fixes: f620d66af316 ("arm64: mte: Do not flag the zero page as PG_mte_tagged") Cc: stable@vger.kernel.org # 5.10.x Suggested-by: Catalin Marinas Signed-off-by: Ard Biesheuvel Reviewed-by: Catalin Marinas Signed-off-by: Will Deacon --- arch/arm64/kvm/mmu.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index d089c107d9b7..445d6cf035c9 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1479,6 +1479,11 @@ static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn, if (!kvm_has_mte(kvm)) return; + if (is_zero_pfn(pfn)) { + WARN_ON_ONCE(nr_pages != 1); + return; + } + if (folio_test_hugetlb(folio)) { /* Hugetlb has MTE flags set on head page only */ if (folio_try_hugetlb_mte_tagging(folio)) { From 53205d56212cbff880a77497e25a0e44036d490a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 4 Jun 2026 17:11:57 +0200 Subject: [PATCH 21/24] arm64: mm: Defer remap of linear alias of data/bss Marking the linear alias of data/bss invalid involves calling set_memory_valid(), which calls split_kernel_leaf_mapping() under the hood. On BBML2_NOABORT capable systems, this may result in the need to allocate page tables at a time when the generic memory allocation APIs are not yet available, resulting in a splat like WARNING: arch/arm64/mm/mmu.c:821 at split_kernel_leaf_mapping+0x15c/0x170, CPU#0: swapper/0 Modules linked in: CPU: 0 UID: 0 PID: 0 Comm: swapper Not tainted 7.1.0-rc6 #1 PREEMPT(undef) pstate: a04000c9 (NzCv daIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : split_kernel_leaf_mapping+0x15c/0x170 lr : update_range_prot+0x40/0x128 sp : ffffc99ad3863c80 ... Call trace: split_kernel_leaf_mapping+0x15c/0x170 (P) update_range_prot+0x40/0x128 set_memory_valid+0x94/0xe0 mark_linear_data_alias_valid+0x54/0x68 map_mem+0x1fc/0x240 paging_init+0x48/0x210 setup_arch+0x274/0x338 start_kernel+0x98/0x538 __primary_switched+0x88/0x98 as reported by CKI automated testing. So defer the boot-time call to mark_linear_data_alias_valid() to a later time when page allocations can be made normally. Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index d68e691c093a..3134f1c1097c 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1095,6 +1095,9 @@ void __init mark_linear_text_alias_ro(void) (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); + /* Map the kernel data/bss as invalid in the linear map */ + mark_linear_data_alias_valid(false); + /* * Register a PM notifier to remap the linear alias of data/bss as * valid read-only before hibernation. This is needed because the @@ -1237,9 +1240,6 @@ static void __init map_mem(void) __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } - - /* Map the kernel data/bss as invalid in the linear map */ - mark_linear_data_alias_valid(false); } void mark_rodata_ro(void) From c34d4d48c4a1427c4af888b02d0ca21fde3e299f Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 20 May 2026 07:34:17 +0100 Subject: [PATCH 22/24] arm64/mm: Rename ptdesc_t ptdesc_t sounds very similar to the core MM struct ptdesc which is actually the memory descriptor for page table allocations. Hence rename this typedef element as ptval_t instead for better clarity and separation. Cc: Catalin Marinas Cc: Will Deacon Cc: David Hildenbrand Cc: Mike Rapoport Cc: linux-efi@vger.kernel.org Cc: linux-kernel@vger.kernel.org Cc: linux-arm-kernel@lists.infradead.org Acked-by: Mike Rapoport (Microsoft) Acked-by: David Hildenbrand (Arm) Suggested-by: David Hildenbrand (Arm) Signed-off-by: Anshuman Khandual Signed-off-by: Will Deacon --- arch/arm64/include/asm/io.h | 2 +- arch/arm64/include/asm/pgtable-types.h | 14 +++++++------- arch/arm64/include/asm/ptdump.h | 8 ++++---- arch/arm64/include/asm/tlbflush.h | 4 ++-- arch/arm64/kernel/efi.c | 4 ++-- arch/arm64/kernel/pi/map_kernel.c | 2 +- arch/arm64/kernel/pi/map_range.c | 4 ++-- arch/arm64/kernel/pi/pi.h | 2 +- arch/arm64/mm/mmap.c | 4 ++-- arch/arm64/mm/pageattr.c | 2 +- arch/arm64/mm/ptdump.c | 2 +- 11 files changed, 24 insertions(+), 24 deletions(-) diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 8cbd1e96fd50..21c8e400107c 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -270,7 +270,7 @@ static inline void __iomem *ioremap_prot(phys_addr_t phys, size_t size, pgprot_t user_prot) { pgprot_t prot; - ptdesc_t user_prot_val = pgprot_val(user_prot); + ptval_t user_prot_val = pgprot_val(user_prot); if (WARN_ON_ONCE(!(user_prot_val & PTE_USER))) return NULL; diff --git a/arch/arm64/include/asm/pgtable-types.h b/arch/arm64/include/asm/pgtable-types.h index 265e8301d7ba..2f2f5527930f 100644 --- a/arch/arm64/include/asm/pgtable-types.h +++ b/arch/arm64/include/asm/pgtable-types.h @@ -17,13 +17,13 @@ * Generic page table descriptor format from which * all level specific descriptors can be derived. */ -typedef u64 ptdesc_t; +typedef u64 ptval_t; -typedef ptdesc_t pteval_t; -typedef ptdesc_t pmdval_t; -typedef ptdesc_t pudval_t; -typedef ptdesc_t p4dval_t; -typedef ptdesc_t pgdval_t; +typedef ptval_t pteval_t; +typedef ptval_t pmdval_t; +typedef ptval_t pudval_t; +typedef ptval_t p4dval_t; +typedef ptval_t pgdval_t; /* * These are used to make use of C type-checking.. @@ -54,7 +54,7 @@ typedef struct { pgdval_t pgd; } pgd_t; #define pgd_val(x) ((x).pgd) #define __pgd(x) ((pgd_t) { (x) } ) -typedef struct { ptdesc_t pgprot; } pgprot_t; +typedef struct { ptval_t pgprot; } pgprot_t; #define pgprot_val(x) ((x).pgprot) #define __pgprot(x) ((pgprot_t) { (x) } ) diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h index baff24004459..5b374a6ab34a 100644 --- a/arch/arm64/include/asm/ptdump.h +++ b/arch/arm64/include/asm/ptdump.h @@ -26,8 +26,8 @@ struct ptdump_info { }; struct ptdump_prot_bits { - ptdesc_t mask; - ptdesc_t val; + ptval_t mask; + ptval_t val; const char *set; const char *clear; }; @@ -36,7 +36,7 @@ struct ptdump_pg_level { const struct ptdump_prot_bits *bits; char name[4]; int num; - ptdesc_t mask; + ptval_t mask; }; /* @@ -53,7 +53,7 @@ struct ptdump_pg_state { const struct mm_struct *mm; unsigned long start_address; int level; - ptdesc_t current_prot; + ptval_t current_prot; bool check_wx; unsigned long wx_pages; unsigned long uxn_pages; diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index c0bf5b398041..d52ac8c17190 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -725,9 +725,9 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b sme_dvmsync_add_pending(batch, mm); } -static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval) +static inline bool __pte_flags_need_flush(ptval_t oldval, ptval_t newval) { - ptdesc_t diff = oldval ^ newval; + ptval_t diff = oldval ^ newval; /* invalid to valid transition requires no flush */ if (!(oldval & PTE_VALID)) diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index a81cb4aa4738..30cd7f804398 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -31,7 +31,7 @@ static bool region_is_misaligned(const efi_memory_desc_t *md) * executable, everything else can be mapped with the XN bits * set. Also take the new (optional) RO/XP bits into account. */ -static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md) +static __init ptval_t create_mapping_protection(efi_memory_desc_t *md) { u64 attr = md->attribute; u32 type = md->type; @@ -85,7 +85,7 @@ static __init ptdesc_t create_mapping_protection(efi_memory_desc_t *md) int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) { - ptdesc_t prot_val = create_mapping_protection(md); + ptval_t prot_val = create_mapping_protection(md); bool page_mappings_only = (md->type == EFI_RUNTIME_SERVICES_CODE || md->type == EFI_RUNTIME_SERVICES_DATA); diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index a852264958c3..fb44cbdd2f29 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -165,7 +165,7 @@ static void noinline __section(".idmap.text") set_ttbr0_for_lpa2(phys_addr_t ttb static void __init remap_idmap_for_lpa2(void) { /* clear the bits that change meaning once LPA2 is turned on */ - ptdesc_t mask = PTE_SHARED; + ptval_t mask = PTE_SHARED; /* * We have to clear bits [9:8] in all block or page descriptors in the diff --git a/arch/arm64/kernel/pi/map_range.c b/arch/arm64/kernel/pi/map_range.c index de52cd85c691..761b14893f74 100644 --- a/arch/arm64/kernel/pi/map_range.c +++ b/arch/arm64/kernel/pi/map_range.c @@ -31,7 +31,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, u64 va_offset) { u64 cmask = (level == 3) ? CONT_PTE_SIZE - 1 : U64_MAX; - ptdesc_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK; + ptval_t protval = pgprot_val(prot) & ~PTE_TYPE_MASK; int lshift = (3 - level) * PTDESC_TABLE_SHIFT; u64 lmask = (PAGE_SIZE << lshift) - 1; @@ -88,7 +88,7 @@ void __init map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, } } -asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptdesc_t clrmask) +asmlinkage phys_addr_t __init create_init_idmap(pgd_t *pg_dir, ptval_t clrmask) { phys_addr_t ptep = (phys_addr_t)pg_dir + PAGE_SIZE; /* MMU is off */ pgprot_t text_prot = PAGE_KERNEL_ROX; diff --git a/arch/arm64/kernel/pi/pi.h b/arch/arm64/kernel/pi/pi.h index aec3172d4003..5dfd8484d200 100644 --- a/arch/arm64/kernel/pi/pi.h +++ b/arch/arm64/kernel/pi/pi.h @@ -35,4 +35,4 @@ void map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, asmlinkage void early_map_kernel(u64 boot_status, phys_addr_t fdt); -asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptdesc_t clrmask); +asmlinkage phys_addr_t create_init_idmap(pgd_t *pgd, ptval_t clrmask); diff --git a/arch/arm64/mm/mmap.c b/arch/arm64/mm/mmap.c index 92b2f5097a96..32e0771d6477 100644 --- a/arch/arm64/mm/mmap.c +++ b/arch/arm64/mm/mmap.c @@ -34,7 +34,7 @@ static pgprot_t protection_map[16] __ro_after_init = { [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED_EXEC }; -static ptdesc_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO; +static ptval_t gcs_page_prot __ro_after_init = _PAGE_GCS_RO; /* * You really shouldn't be using read() or write() on /dev/mem. This might go @@ -87,7 +87,7 @@ arch_initcall(adjust_protection_map); pgprot_t vm_get_page_prot(vm_flags_t vm_flags) { - ptdesc_t prot; + ptval_t prot; /* Short circuit GCS to avoid bloating the table. */ if (system_supports_gcs() && (vm_flags & VM_SHADOW_STACK)) { diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index ce035e1b4eaf..bbe98ac9ad8c 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -21,7 +21,7 @@ struct page_change_data { pgprot_t clear_mask; }; -static ptdesc_t set_pageattr_masks(ptdesc_t val, struct mm_walk *walk) +static ptval_t set_pageattr_masks(ptval_t val, struct mm_walk *walk) { struct page_change_data *masks = walk->private; diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index ab9899ca1e5f..1c20144700d7 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -194,7 +194,7 @@ void note_page(struct ptdump_state *pt_st, unsigned long addr, int level, struct ptdump_pg_state *st = container_of(pt_st, struct ptdump_pg_state, ptdump); struct ptdump_pg_level *pg_level = st->pg_level; static const char units[] = "KMGTPE"; - ptdesc_t prot = 0; + ptval_t prot = 0; /* check if the current level has been folded dynamically */ if (st->mm && ((level == 1 && mm_p4d_folded(st->mm)) || From 81479b6888f8ff7099103c08efab35f4817976d5 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 10 Jun 2026 11:34:39 +0100 Subject: [PATCH 23/24] Revert "arm64: mm: Defer remap of linear alias of data/bss" This reverts commit 53205d56212cbff880a77497e25a0e44036d490a. Unmapping the kernel '.bss' appears to break KVM initialisation on some devices, breaking the boot on popular platforms such as RaspberryPi3 and 4. Revert this change for now so that we can revisit it in future. Reported-by: Mark Brown Reported-by: Marek Szyprowski Link: https://lore.kernel.org/all/aicVyebkEMs6w6UV@sirena.co.uk Link: https://lore.kernel.org/r/a1b27e97-182c-485d-a448-56c19c5de2c2@samsung.com Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 3134f1c1097c..d68e691c093a 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1095,9 +1095,6 @@ void __init mark_linear_text_alias_ro(void) (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); - /* Map the kernel data/bss as invalid in the linear map */ - mark_linear_data_alias_valid(false); - /* * Register a PM notifier to remap the linear alias of data/bss as * valid read-only before hibernation. This is needed because the @@ -1240,6 +1237,9 @@ static void __init map_mem(void) __map_memblock(start, end, pgprot_tagged(PAGE_KERNEL), flags); } + + /* Map the kernel data/bss as invalid in the linear map */ + mark_linear_data_alias_valid(false); } void mark_rodata_ro(void) From f102131c842d1e1d95bc7b818ab93944195da7e9 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 10 Jun 2026 11:40:23 +0100 Subject: [PATCH 24/24] Revert "arm64: mm: Unmap kernel data/bss entirely from the linear map" This reverts commit 63e0b6a5b6934d6a919d1c65ea185303200a1874. Unmapping the kernel '.bss' appears to break KVM initialisation on some devices, breaking the boot on popular platforms such as RaspberryPi3 and 4. Revert this change for now so that we can revisit it in future. Reported-by: Mark Brown Reported-by: Marek Szyprowski Link: https://lore.kernel.org/all/aicVyebkEMs6w6UV@sirena.co.uk Link: https://lore.kernel.org/r/a1b27e97-182c-485d-a448-56c19c5de2c2@samsung.com Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 45 ++++----------------------------------------- 1 file changed, 4 insertions(+), 41 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index d68e691c093a..354bedf0e638 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -1063,29 +1062,6 @@ static void __init __map_memblock(phys_addr_t start, phys_addr_t end, end - start, prot, early_pgtable_alloc, flags); } -static void mark_linear_data_alias_valid(bool valid) -{ - set_memory_valid((unsigned long)lm_alias(__init_end), - (unsigned long)(__bss_stop - __init_end) / PAGE_SIZE, - valid); -} - -static int arm64_hibernate_pm_notify(struct notifier_block *nb, - unsigned long mode, void *unused) -{ - switch (mode) { - default: - break; - case PM_POST_HIBERNATION: - mark_linear_data_alias_valid(false); - break; - case PM_HIBERNATION_PREPARE: - mark_linear_data_alias_valid(true); - break; - } - return 0; -} - void __init mark_linear_text_alias_ro(void) { /* @@ -1094,21 +1070,6 @@ void __init mark_linear_text_alias_ro(void) update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text), (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); - - /* - * Register a PM notifier to remap the linear alias of data/bss as - * valid read-only before hibernation. This is needed because the - * snapshot logic disregards PageReserved pages (such as the ones - * covering the kernel image) unless they are mapped in the linear - * map. - */ - if (IS_ENABLED(CONFIG_HIBERNATION)) { - static struct notifier_block nb = { - .notifier_call = arm64_hibernate_pm_notify - }; - - register_pm_notifier(&nb); - } } #ifdef CONFIG_KFENCE @@ -1238,8 +1199,10 @@ static void __init map_mem(void) flags); } - /* Map the kernel data/bss as invalid in the linear map */ - mark_linear_data_alias_valid(false); + /* Map the kernel data/bss read-only in the linear map */ + __map_memblock(init_end, kernel_end, PAGE_KERNEL_RO, flags); + flush_tlb_kernel_range((unsigned long)lm_alias(__init_end), + (unsigned long)lm_alias(__bss_stop)); } void mark_rodata_ro(void)