From 233b5554e07b3902be9b0dd25bbdcc718824b022 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 23 Jul 2026 14:03:21 +0100 Subject: [PATCH 1/5] arm64: mm: Treat all devices as dma-coherent when CLIDR_EL1.LoC == 0 On systems where CLIDR_EL1.LoC == 0, no cache maintenance is required when cleaning or invalidating to the Point of Coherency and therefore all DMA agents can be treated as coherent. Extend arch_setup_dma_ops() to take CLIDR_EL1.LoC into account when setting the DMA ops for a device, emitting a warning message if the firmware advertises a non-coherent device on a fully coherent system. Cc: Steffen Eiden Cc: Andreas Grapentin Cc: Mark Rutland Cc: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm64/mm/dma-mapping.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 994b7b36e2b9..7f6f2f40dba2 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -42,6 +42,11 @@ void arch_setup_dma_ops(struct device *dev, bool coherent) { int cls = cache_line_size_of_cpu(); + if (!coherent && !CLIDR_LOC(read_sysreg(clidr_el1))) { + dev_warn(dev, "CLIDR_EL1.LoC == 0, treating as coherent\n"); + coherent = true; + } + WARN_TAINT(!coherent && cls > ARCH_DMA_MINALIGN, TAINT_CPU_OUT_OF_SPEC, "%s %s: ARCH_DMA_MINALIGN smaller than CTR_EL0.CWG (%d < %d)", From 3b56ebafecc43367b811361459c09c9ef9bfc167 Mon Sep 17 00:00:00 2001 From: Yeoreum Yun Date: Wed, 22 Jul 2026 16:30:28 +0100 Subject: [PATCH 2/5] arm64: pgtable: convert pte_present() from macro to static inline pte_present() is used as the basis for both pmd_present() and pud_present(). It is currently implemented as a macro composed of pte_val() and pte_present_invalid(). When pte_present() or its higher-level variants are used directly with ptep_get() or pXdp_get(), for example: pte_present(ptep_get(pte)); pmd_present(pmdp_get(pmd)); pud_present(pudp_get(pud)); the macro expansion causes the compiler to evaluate the argument twice, resulting in redundant loads. For example, pte_present() expands to: !pte_val(READ_ONCE(*pte) || pte_present_invalid(READ_ONCE(*pte)) A typical example is pud_free_pmd_page(), where the expansion of pmd_present() generates: ... /* pmd_present() (x20 = pmdp) */ 1b88: f9400288 ldr x8, [x20] // read pmdp. 1b8c: f9000fa8 str x8, [x29, #0x18] 1b90: 3707fec8 tbnz w8, #0x0, 0x1b68 1b94: f9400288 ldr x8, [x20] // redundant read of pmdp. 1b98: 8a170109 and x9, x8, x23 1b9c: f9000fa8 str x8, [x29, #0x18] 1ba0: f120013f cmp x9, #0x800 1ba4: 54fffe20 b.eq 0x1b68 1ba8: 17fffff4 b 0x1b78 ... Convert pte_present() to static inline function so that prevent the generation of redundant code and move pte_valid() and pte_present_invalid() further up so the inline function can use them. After this change, the generated code becomes: ... /* pmd_present() (x20 = pmdp) */ 1a30: f9400288 ldr x8, [x20] 1a34: 8a170109 and x9, x8, x23 1a38: f9000fa8 str x8, [x29, #0x18] 1a3c: f120013f cmp x9, #0x800 1a40: 54fffe80 b.eq 0x1a10 1a44: 3607fee8 tbz w8, #0x0, 0x1a20 1a48: 17fffff2 b 0x1a10 ... This eliminates the redundant load and also reduces code size at call sites using this pattern. For example, pud_free_pmd_page() shrinks from 7,500 bytes to 7,148 bytes, a reduction of approximately 4.7%. Signed-off-by: Yeoreum Yun Signed-off-by: Will Deacon --- arch/arm64/include/asm/pgtable.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 27689c62bd25..25001694ae8e 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -140,10 +140,17 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) #define pte_none(pte) (!pte_val(pte)) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) +#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) +#define pte_present_invalid(pte) \ + ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID) + /* * The following only work if pte_present(). Undefined behaviour otherwise. */ -#define pte_present(pte) (pte_valid(pte) || pte_present_invalid(pte)) +static __always_inline bool pte_present(pte_t pte) +{ + return pte_valid(pte) || pte_present_invalid(pte); +} #define pte_young(pte) (!!(pte_val(pte) & PTE_AF)) #define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) #define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) @@ -168,9 +175,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) -#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) -#define pte_present_invalid(pte) \ - ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID) /* * Execute-only user mappings do not have the PTE_USER bit set. All valid * kernel mappings have the PTE_UXN bit set. From 7ace06a01efaba1ec585630aa4c98ac39b38452a Mon Sep 17 00:00:00 2001 From: liulhong617 Date: Wed, 13 May 2026 09:02:55 +0800 Subject: [PATCH 3/5] arm64: mm: fix accidental linear mapping of no-map reserved memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When reserved-memory regions with the "no-map" property are not page-aligned, the kernel may accidentally map them into the linear mapping, contradicting the no-map semantics. The root cause is a mismatch between /proc/iomem's address boundaries and the actual page table mapping boundaries: 1. /proc/iomem derives its ranges from memblock via memblock_region_reserved_base_pfn/memblock_region_reserved_end_pfn, which perform PFN rounding so the displayed boundaries are page-aligned. This gives the impression that the no-map region occupies whole pages. 2. However, memblock_mark_nomap() splits memblock.memory regions at exact byte boundaries (memblock_isolate_range preserves raw DT base/size with no alignment). When for_each_mem_range iterates the non-NOMAP regions adjacent to a no-map region, it returns start/end values that are NOT page-aligned — they are the precise byte boundaries from the memblock split. 3. These sub-page-aligned values are passed to __create_pgd_mapping_locked(), which does: phys &= PAGE_MASK; addr = virt & PAGE_MASK; end = PAGE_ALIGN(virt + size); The downward rounding of phys via PAGE_MASK extends the mapped range backward into the adjacent no-map region, effectively including no-map memory in the linear mapping. For example, with 64K pages, reserved_region@A2000000 (base=0xA2000000, size=0x8000, no-map) causes for_each_mem_range to return start=0xA2008000 for the next mappable region. After phys &= PAGE_MASK, the actual mapping starts at 0xA2000000 — the entire no-map region is incorrectly mapped. Fix this by rounding the mappable range inward to PAGE_SIZE boundaries before passing it to __map_memblock: start is rounded UP and end is rounded DOWN. This ensures the mapped area never overlaps with adjacent no-map regions. The cost is at most one page of unmapped gap at each boundary, which is preferable to violating no-map semantics. Signed-off-by: liulhong617 Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index a25d8beacc83..f51566316ba4 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1190,6 +1190,20 @@ static void __init map_mem(void) /* map all the memory banks */ for_each_mem_range(i, &start, &end) { + /* + * for_each_mem_range may return sub-page-aligned boundaries + * after memblock_mark_nomap() splits regions at byte precision. + * __create_pgd_mapping_locked aligns phys down to PAGE_MASK, + * which could accidentally map no-map memory on the boundary. + * Round the mappable range inward: start UP, end DOWN, so + * that the mapped area never overlaps with adjacent no-map + * regions. The cost is at most one page of unmapped gap at + * each boundary. + */ + start = PAGE_ALIGN(start); + end = end & PAGE_MASK; + if (start >= end) + continue; /* * The linear map must allow allocation tags reading/writing * if MTE is present. Otherwise, it has the same attributes as From fb0f3ef5601a67877cee38257e872c21db9d7f77 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 3 Aug 2026 18:33:47 +0200 Subject: [PATCH 4/5] arm64: mm: Unmap kernel data/bss entirely from the linear map The linear aliases of the kernel text and rodata are also mapped read-only in the linear map. Given that the contents of these regions are mostly identical to the version in the loadable image, mapping them read-only and leaving their contents visible is a reasonable hardening measure. Data and bss, however, are now also mapped read-only but the contents of these regions are more likely to contain data that we'd rather not leak. So let's unmap these entirely in the linear map when the kernel is running normally. When going into hibernation or waking up from it, these regions need to be mapped, so map the region initially, and toggle the valid bit so map/unmap the region as needed. Doing so is required because pages covering the kernel image are marked as PageReserved, and therefore disregarded for snapshotting by the hibernate logic unless they are mapped. Cc: Ryan Roberts Cc: Anshuman Khandual Cc: Kevin Brodsky Cc: Liz Prucka Cc: Seth Jenkins Cc: Kees Cook Cc: David Hildenbrand Cc: Jann Horn Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 46 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index f51566316ba4..abe53881553f 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1062,6 +1063,29 @@ static void __init __map_memblock(phys_addr_t start, phys_addr_t end, end - start, prot, early_pgtable_alloc, flags); } +static void mark_linear_data_alias_valid(bool valid) +{ + set_memory_valid((unsigned long)lm_alias(__init_end), + (unsigned long)(__bss_stop - __init_end) / PAGE_SIZE, + valid); +} + +static int arm64_hibernate_pm_notify(struct notifier_block *nb, + unsigned long mode, void *unused) +{ + switch (mode) { + default: + break; + case PM_POST_HIBERNATION: + mark_linear_data_alias_valid(false); + break; + case PM_HIBERNATION_PREPARE: + mark_linear_data_alias_valid(true); + break; + } + return 0; +} + void __init mark_linear_text_alias_ro(void) { /* @@ -1070,6 +1094,21 @@ void __init mark_linear_text_alias_ro(void) update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text), (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); + + /* + * Register a PM notifier to remap the linear alias of data/bss as + * valid read/write before hibernation. This is needed because the + * snapshot logic disregards PageReserved pages (such as the ones + * covering the kernel image) unless they are mapped in the linear + * map. + */ + if (IS_ENABLED(CONFIG_HIBERNATION) && rodata_enabled) { + static struct notifier_block nb = { + .notifier_call = arm64_hibernate_pm_notify + }; + + register_pm_notifier(&nb); + } } #ifdef CONFIG_KFENCE @@ -1231,11 +1270,8 @@ void mark_rodata_ro(void) (unsigned long)_stext - (unsigned long)_text, PAGE_KERNEL_RO); - /* Map the kernel data/bss read-only in the linear map */ - update_mapping_prot(__pa_symbol(__init_end), - (unsigned long)lm_alias(__init_end), - (unsigned long)__bss_stop - (unsigned long)__init_end, - PAGE_KERNEL_RO); + /* Map the kernel data/bss as invalid in the linear map */ + mark_linear_data_alias_valid(false); } static void __init declare_vma(struct vm_struct *vma, From e98a9d0146372b046d863164025a66ab4488b972 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 11 Aug 2026 15:04:29 +0100 Subject: [PATCH 5/5] arm64/efi: Avoid voluntary preemption with efi_mm installed Gus reports a bad kernel memory access when using software PAN (CONFIG_ARM64_SW_TTBR0_PAN=y) on a machine with support for EFI runtime services: Unable to handle kernel access to user memory outside uaccess routines at virtual address 00000000f322ff30 Mem abort info: ESR = 0x0000000096000004 FSC = 0x04: level 0 translation fault Internal error: Oops: 0000000096000004 [#1] SMP Workqueue: efi_rts_wq efi_call_rts pstate: 80400005 (Nzcv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : efi_call_rts+0xd8/0x288 Call trace: efi_call_rts+0xd8/0x288 (P) process_one_work+0x178/0x4f8 worker_thread+0x194/0x328 This is because the fpsimd context management code called from __efi_fpsimd_begin() can preempt voluntarily, returning later to the EFI code with an incorrect value for TTBR0_EL1 thanks to the deferred mm switching used by the software PAN implementation. Since EFI runtime services cannot preempt voluntarily and because the fpsimd switching code does not rely on the TTBR0_EL1 mappings, simply reorder the fpsimd switch so that it occurs before we change the page-table. Cc: Ard Biesheuvel Reported-by: Gus Bourg Tested-by: Gus Bourg Fixes: a5baf582f4c0 ("arm64/efi: Call EFI runtime services without disabling preemption") Link: https://lore.kernel.org/all/20260806000144.3388823-1-gus@bourg.net/ Reviewed-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/kernel/efi.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 30cd7f804398..0ec90fd1754e 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -184,6 +184,8 @@ void arch_efi_call_virt_setup(void) efi_virtmap_load(); } + __efi_fpsimd_begin(); + /* * Enable access to the valid TTBR0_EL1 and invoke the errata * workaround directly since there is no return from exception when @@ -191,8 +193,6 @@ void arch_efi_call_virt_setup(void) */ uaccess_ttbr0_enable(); post_ttbr_update_workaround(); - - __efi_fpsimd_begin(); } void arch_efi_call_virt_teardown(void)