diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 27689c62bd25..25001694ae8e 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -140,10 +140,17 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) #define pte_none(pte) (!pte_val(pte)) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) +#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) +#define pte_present_invalid(pte) \ + ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID) + /* * The following only work if pte_present(). Undefined behaviour otherwise. */ -#define pte_present(pte) (pte_valid(pte) || pte_present_invalid(pte)) +static __always_inline bool pte_present(pte_t pte) +{ + return pte_valid(pte) || pte_present_invalid(pte); +} #define pte_young(pte) (!!(pte_val(pte) & PTE_AF)) #define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) #define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) @@ -168,9 +175,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) -#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) -#define pte_present_invalid(pte) \ - ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID) /* * Execute-only user mappings do not have the PTE_USER bit set. All valid * kernel mappings have the PTE_UXN bit set. diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 30cd7f804398..0ec90fd1754e 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -184,6 +184,8 @@ void arch_efi_call_virt_setup(void) efi_virtmap_load(); } + __efi_fpsimd_begin(); + /* * Enable access to the valid TTBR0_EL1 and invoke the errata * workaround directly since there is no return from exception when @@ -191,8 +193,6 @@ void arch_efi_call_virt_setup(void) */ uaccess_ttbr0_enable(); post_ttbr_update_workaround(); - - __efi_fpsimd_begin(); } void arch_efi_call_virt_teardown(void) diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 994b7b36e2b9..7f6f2f40dba2 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -42,6 +42,11 @@ void arch_setup_dma_ops(struct device *dev, bool coherent) { int cls = cache_line_size_of_cpu(); + if (!coherent && !CLIDR_LOC(read_sysreg(clidr_el1))) { + dev_warn(dev, "CLIDR_EL1.LoC == 0, treating as coherent\n"); + coherent = true; + } + WARN_TAINT(!coherent && cls > ARCH_DMA_MINALIGN, TAINT_CPU_OUT_OF_SPEC, "%s %s: ARCH_DMA_MINALIGN smaller than CTR_EL0.CWG (%d < %d)", diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index fad71d6bff40..ccdc2ba1cb95 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1062,6 +1063,29 @@ static void __init __map_memblock(phys_addr_t start, phys_addr_t end, end - start, prot, early_pgtable_alloc, flags); } +static void mark_linear_data_alias_valid(bool valid) +{ + set_memory_valid((unsigned long)lm_alias(__init_end), + (unsigned long)(__bss_stop - __init_end) / PAGE_SIZE, + valid); +} + +static int arm64_hibernate_pm_notify(struct notifier_block *nb, + unsigned long mode, void *unused) +{ + switch (mode) { + default: + break; + case PM_POST_HIBERNATION: + mark_linear_data_alias_valid(false); + break; + case PM_HIBERNATION_PREPARE: + mark_linear_data_alias_valid(true); + break; + } + return 0; +} + void __init mark_linear_text_alias_ro(void) { /* @@ -1070,6 +1094,21 @@ void __init mark_linear_text_alias_ro(void) update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text), (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); + + /* + * Register a PM notifier to remap the linear alias of data/bss as + * valid read/write before hibernation. This is needed because the + * snapshot logic disregards PageReserved pages (such as the ones + * covering the kernel image) unless they are mapped in the linear + * map. + */ + if (IS_ENABLED(CONFIG_HIBERNATION) && rodata_enabled) { + static struct notifier_block nb = { + .notifier_call = arm64_hibernate_pm_notify + }; + + register_pm_notifier(&nb); + } } #ifdef CONFIG_KFENCE @@ -1190,6 +1229,20 @@ static void __init map_mem(void) /* map all the memory banks */ for_each_mem_range(i, &start, &end) { + /* + * for_each_mem_range may return sub-page-aligned boundaries + * after memblock_mark_nomap() splits regions at byte precision. + * __create_pgd_mapping_locked aligns phys down to PAGE_MASK, + * which could accidentally map no-map memory on the boundary. + * Round the mappable range inward: start UP, end DOWN, so + * that the mapped area never overlaps with adjacent no-map + * regions. The cost is at most one page of unmapped gap at + * each boundary. + */ + start = PAGE_ALIGN(start); + end = end & PAGE_MASK; + if (start >= end) + continue; /* * The linear map must allow allocation tags reading/writing * if MTE is present. Otherwise, it has the same attributes as @@ -1217,11 +1270,8 @@ void mark_rodata_ro(void) (unsigned long)_stext - (unsigned long)_text, PAGE_KERNEL_RO); - /* Map the kernel data/bss read-only in the linear map */ - update_mapping_prot(__pa_symbol(__init_end), - (unsigned long)lm_alias(__init_end), - (unsigned long)__bss_stop - (unsigned long)__init_end, - PAGE_KERNEL_RO); + /* Map the kernel data/bss as invalid in the linear map */ + mark_linear_data_alias_valid(false); } static void __init declare_vma(struct vm_struct *vma,