From 73c4b5d728265a6b019c3aba6d7ea2329ca4785a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 3 Nov 2025 14:39:26 +0100 Subject: [PATCH 1/3] s390: Add Dat-Enhancement facility 1 to architecture level set Add the Dat-Enhancement facility 1 to the list of facilities which are required to start the kernel. The facility provides the CSPG and IDTE instructions. In particular the CSPG instruction can be used to replace a valid page table entry with a different page table entry, which also differs in the page frame real address. Without the CSPG instruction it is possible to use the CSP instruction to change valid page table entries, however it only allows to change the lower or higher 32 bits of such entries, which means it cannot be used to change the page frame real address of valid page table entries. Given that there is code around (e.g. HugeTLB vmemmap optimization) which requires to change valid page table entries of the kernel mapping, without the detour over an invalid page table entry, make the CSPG instruction unconditionally available. The Dat-Enhancement facility 1 is available since z990, which is older than the currently supported minimum architecture (z10). Therefore adding this to the architecture level set shouldn't cause any problems. Reviewed-by: Alexander Gordeev Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/tools/gen_facilities.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c index d5c68ade71ab..2d28a569f793 100644 --- a/arch/s390/tools/gen_facilities.c +++ b/arch/s390/tools/gen_facilities.c @@ -29,6 +29,7 @@ static struct facility_def facility_defs[] = { .bits = (int[]){ 0, /* N3 instructions */ 1, /* z/Arch mode installed */ + 3, /* dat-enhancement 1 */ 18, /* long displacement facility */ 21, /* extended-immediate facility */ 25, /* store clock fast */ From 220d8e10d69a30db7d9cbd0d3483a739768ed16c Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 3 Nov 2025 14:39:27 +0100 Subject: [PATCH 2/3] s390/mm: Remove cpu_has_idte() Remove cpu_has_idte(). The IDTE instruction is part of the Dat-Enhancement facility 1, which is always available. Therefore remove the helper and now superfluous code. Acked-by: Alexander Gordeev Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/cpufeature.h | 1 - arch/s390/include/asm/tlbflush.h | 7 ++----- arch/s390/mm/gmap.c | 15 ++++----------- arch/s390/mm/pageattr.c | 4 +--- arch/s390/mm/pgtable.c | 14 ++------------ 5 files changed, 9 insertions(+), 32 deletions(-) diff --git a/arch/s390/include/asm/cpufeature.h b/arch/s390/include/asm/cpufeature.h index 6c6a99660e78..d6fb999c8c6d 100644 --- a/arch/s390/include/asm/cpufeature.h +++ b/arch/s390/include/asm/cpufeature.h @@ -27,7 +27,6 @@ int cpu_have_feature(unsigned int nr); #define cpu_has_edat1() test_facility(8) #define cpu_has_edat2() test_facility(78) #define cpu_has_gs() test_facility(133) -#define cpu_has_idte() test_facility(3) #define cpu_has_nx() test_facility(130) #define cpu_has_rdp() test_facility(194) #define cpu_has_seq_insn() test_facility(85) diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h index 75491baa2197..c7f04f9e99b3 100644 --- a/arch/s390/include/asm/tlbflush.h +++ b/arch/s390/include/asm/tlbflush.h @@ -54,7 +54,7 @@ static inline void __tlb_flush_mm(struct mm_struct *mm) cpumask_copy(mm_cpumask(mm), &mm->context.cpu_attach_mask); barrier(); gmap_asce = READ_ONCE(mm->context.gmap_asce); - if (cpu_has_idte() && gmap_asce != -1UL) { + if (gmap_asce != -1UL) { if (gmap_asce) __tlb_flush_idte(gmap_asce); __tlb_flush_idte(mm->context.asce); @@ -68,10 +68,7 @@ static inline void __tlb_flush_mm(struct mm_struct *mm) static inline void __tlb_flush_kernel(void) { - if (cpu_has_idte()) - __tlb_flush_idte(init_mm.context.asce); - else - __tlb_flush_global(); + __tlb_flush_idte(init_mm.context.asce); } static inline void __tlb_flush_mm_lazy(struct mm_struct * mm) diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 8ff6bba107e8..f9174be33023 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -138,10 +138,7 @@ EXPORT_SYMBOL_GPL(gmap_create); static void gmap_flush_tlb(struct gmap *gmap) { - if (cpu_has_idte()) - __tlb_flush_idte(gmap->asce); - else - __tlb_flush_global(); + __tlb_flush_idte(gmap->asce); } static void gmap_radix_tree_free(struct radix_tree_root *root) @@ -1988,10 +1985,8 @@ static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *pmdp, pmd_t new, if (machine_has_tlb_guest()) __pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce, IDTE_GLOBAL); - else if (cpu_has_idte()) - __pmdp_idte(gaddr, (pmd_t *)pmdp, 0, 0, IDTE_GLOBAL); else - __pmdp_csp(pmdp); + __pmdp_idte(gaddr, (pmd_t *)pmdp, 0, 0, IDTE_GLOBAL); set_pmd(pmdp, new); } @@ -2066,7 +2061,7 @@ void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr) if (machine_has_tlb_guest()) __pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE, gmap->asce, IDTE_LOCAL); - else if (cpu_has_idte()) + else __pmdp_idte(gaddr, pmdp, 0, 0, IDTE_LOCAL); *pmdp = __pmd(_SEGMENT_ENTRY_EMPTY); } @@ -2099,10 +2094,8 @@ void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr) if (machine_has_tlb_guest()) __pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE, gmap->asce, IDTE_GLOBAL); - else if (cpu_has_idte()) - __pmdp_idte(gaddr, pmdp, 0, 0, IDTE_GLOBAL); else - __pmdp_csp(pmdp); + __pmdp_idte(gaddr, pmdp, 0, 0, IDTE_GLOBAL); *pmdp = __pmd(_SEGMENT_ENTRY_EMPTY); } spin_unlock(&gmap->guest_table_lock); diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 348e759840e7..3042647c9dbf 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -78,10 +78,8 @@ static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr, } table = (unsigned long *)((unsigned long)old & mask); crdte(*old, new, table, dtt, addr, get_lowcore()->kernel_asce.val); - } else if (cpu_has_idte()) { - cspg(old, *old, new); } else { - csp((unsigned int *)old + 1, *old, new); + cspg(old, *old, new); } } diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 0fde20bbc50b..36305a357cb8 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -360,14 +360,10 @@ static inline void pmdp_idte_global(struct mm_struct *mm, mm->context.asce, IDTE_GLOBAL); if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) gmap_pmdp_idte_global(mm, addr); - } else if (cpu_has_idte()) { + } else { __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) gmap_pmdp_idte_global(mm, addr); - } else { - __pmdp_csp(pmdp); - if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) - gmap_pmdp_csp(mm, addr); } } @@ -487,14 +483,8 @@ static inline void pudp_idte_global(struct mm_struct *mm, if (machine_has_tlb_guest()) __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, mm->context.asce, IDTE_GLOBAL); - else if (cpu_has_idte()) - __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL); else - /* - * Invalid bit position is the same for pmd and pud, so we can - * reuse _pmd_csp() here - */ - __pmdp_csp((pmd_t *) pudp); + __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL); } static inline pud_t pudp_flush_direct(struct mm_struct *mm, From 68807a894f0c1cb294a568d8e53c1a77ceaf0c6f Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 3 Nov 2025 14:39:28 +0100 Subject: [PATCH 3/3] s390/mm: Replace the CSP instruction with CSPG The CSPG instruction is part of the Dat-Enhancement facility 1, which is always available. Given that it can be used everywhere where also the CSP instruction can be used, replace CSP with CSPG everywhere. This allows to remove the csp() inline assembly. Also remove the unused gmap_pmdp_csp() function. Acked-by: Alexander Gordeev Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/include/asm/pgtable.h | 19 +++---------------- arch/s390/include/asm/tlbflush.h | 4 ++-- arch/s390/mm/gmap.c | 13 +------------ 3 files changed, 6 insertions(+), 30 deletions(-) diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index b7100c6a4054..bb915cbffc7b 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -648,18 +648,6 @@ static inline int mm_uses_skeys(struct mm_struct *mm) return 0; } -static inline void csp(unsigned int *ptr, unsigned int old, unsigned int new) -{ - union register_pair r1 = { .even = old, .odd = new, }; - unsigned long address = (unsigned long)ptr | 1; - - asm volatile( - " csp %[r1],%[address]" - : [r1] "+&d" (r1.pair), "+m" (*ptr) - : [address] "d" (address) - : "cc"); -} - /** * cspg() - Compare and Swap and Purge (CSPG) * @ptr: Pointer to the value to be exchanged @@ -1402,7 +1390,6 @@ int set_pgste_bits(struct mm_struct *mm, unsigned long addr, int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep); int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc, unsigned long *oldpte, unsigned long *oldpgste); -void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr); void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr); void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr); void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr); @@ -1692,10 +1679,10 @@ static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot) #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLB_PAGE */ -static inline void __pmdp_csp(pmd_t *pmdp) +static inline void __pmdp_cspg(pmd_t *pmdp) { - csp((unsigned int *)pmdp + 1, pmd_val(*pmdp), - pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID); + cspg((unsigned long *)pmdp, pmd_val(*pmdp), + pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID); } #define IDTE_GLOBAL 0 diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h index c7f04f9e99b3..be8ca0d854d3 100644 --- a/arch/s390/include/asm/tlbflush.h +++ b/arch/s390/include/asm/tlbflush.h @@ -35,9 +35,9 @@ static inline void __tlb_flush_idte(unsigned long asce) */ static inline void __tlb_flush_global(void) { - unsigned int dummy = 0; + unsigned long dummy = 0; - csp(&dummy, 0, 0); + cspg(&dummy, 0, 0); } /* diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index f9174be33023..603d9e5febb5 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -2007,7 +2007,7 @@ static void gmap_pmdp_clear(struct mm_struct *mm, unsigned long vmaddr, _SEGMENT_ENTRY_GMAP_UC | _SEGMENT_ENTRY)); if (purge) - __pmdp_csp(pmdp); + __pmdp_cspg(pmdp); set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY)); } spin_unlock(&gmap->guest_table_lock); @@ -2027,17 +2027,6 @@ void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr) } EXPORT_SYMBOL_GPL(gmap_pmdp_invalidate); -/** - * gmap_pmdp_csp - csp all affected guest pmd entries - * @mm: pointer to the process mm_struct - * @vmaddr: virtual address in the process address space - */ -void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr) -{ - gmap_pmdp_clear(mm, vmaddr, 1); -} -EXPORT_SYMBOL_GPL(gmap_pmdp_csp); - /** * gmap_pmdp_idte_local - invalidate and clear a guest pmd entry * @mm: pointer to the process mm_struct