From 2dfbc0f077fe0c88edf620627691a64d8d9d84ca Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Wed, 2 Sep 2026 23:20:17 +0200 Subject: [PATCH 001/102] s390/ebcdic: Add character classes for the ASCII subset of EBCDIC Add _ctype-like array for EBCDIC characters. Mark only those that also exist in ASCII, loosely based on invariant subset of EBCDIC [1], but with cent, broken bar, not sign, and plus-minus sign, left out. [1] https://en.wikipedia.org/wiki/EBCDIC#Code_page_layout [hca@linux.ibm.com: Reformatted tabled] Signed-off-by: Ilya Leoshkevich Reviewed-by: Heiko Carstens Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ebcdic.h | 6 +++++ arch/s390/kernel/ebcdic.c | 40 +++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/arch/s390/include/asm/ebcdic.h b/arch/s390/include/asm/ebcdic.h index 7164cb658435..78d2e37e985d 100644 --- a/arch/s390/include/asm/ebcdic.h +++ b/arch/s390/include/asm/ebcdic.h @@ -10,6 +10,7 @@ #ifndef _EBCDIC_H #define _EBCDIC_H +#include #include extern __u8 _ascebc_500[256]; /* ASCII -> EBCDIC 500 conversion table */ @@ -18,6 +19,11 @@ extern __u8 _ascebc[256]; /* ASCII -> EBCDIC conversion table */ extern __u8 _ebcasc[256]; /* EBCDIC -> ASCII conversion table */ extern __u8 _ebc_tolower[256]; /* EBCDIC -> lowercase */ extern __u8 _ebc_toupper[256]; /* EBCDIC -> uppercase */ +extern const __u8 _ebctype_asc[256]; /* EBCDIC -> character classes */ + +#define __ismask_ebc_asc(x) (_ebctype_asc[(int)(unsigned char)(x)]) +#define isprint_ebc_asc(c) ((__ismask_ebc_asc(c)&(_P|_U|_L|_D|_SP)) != 0) +#define isspace_ebc_asc(c) ((__ismask_ebc_asc(c)&(_S)) != 0) static inline void codepage_convert(const __u8 *codepage, volatile char *addr, unsigned long nr) diff --git a/arch/s390/kernel/ebcdic.c b/arch/s390/kernel/ebcdic.c index 0e51fa537262..970aace8719d 100644 --- a/arch/s390/kernel/ebcdic.c +++ b/arch/s390/kernel/ebcdic.c @@ -392,10 +392,48 @@ __u8 _ebc_toupper[256] = 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF }; +/* + * ASCII subset of EBCDIC -> character classes + */ +const __u8 _ebctype_asc[256] = { + _C, _C, _C, _C, _C, _C|_S, _C, _C, /* 0x00-0x07 */ + _C, _C, _C, _C|_S, _C|_S, _C|_S, _C, _C, /* 0x08-0x0f */ + _C, _C, _C, _C, _C, _C|_S, _C, _C, /* 0x10-0x17 */ + _C, _C, _C, _C, _C, _C, _C, _C, /* 0x18-0x1f */ + _C, _C, _C, _C, _C, _C|_S, _C, _C, /* 0x20-0x27 */ + _C, _C, _C, _C, _C, _C, _C, _C, /* 0x28-0x2f */ + 0, 0, _C, _C, _C, _C, _C, _C, /* 0x30-0x37 */ + _C, _C, _C, _C, _C, _C, 0, _C, /* 0x38-0x3f */ + _S|_SP, 0, 0, 0, 0, 0, 0, 0, /* 0x40-0x47 */ + 0, 0, 0, _P, _P, _P, _P, _P, /* 0x48-0x4f */ + _P, 0, 0, 0, 0, 0, 0, 0, /* 0x50-0x57 */ + 0, 0, _P, _P, _P, _P, _P, 0, /* 0x58-0x5f */ + _P, _P, 0, 0, 0, 0, 0, 0, /* 0x60-0x67 */ + 0, 0, 0, _P, _P, _P, _P, _P, /* 0x68-0x6f */ + 0, 0, 0, 0, 0, 0, 0, 0, /* 0x70-0x77 */ + 0, _P, _P, _P, _P, _P, _P, _P, /* 0x78-0x7f */ + 0, _L|_X, _L|_X, _L|_X, _L|_X, _L|_X, _L|_X, _L, /* 0x80-0x87 */ + _L, _L, 0, 0, 0, 0, 0, 0, /* 0x88-0x8f */ + 0, _L, _L, _L, _L, _L, _L, _L, /* 0x90-0x97 */ + _L, _L, 0, 0, 0, 0, 0, 0, /* 0x98-0x9f */ + 0, _P, _L, _L, _L, _L, _L, _L, /* 0xa0-0xa7 */ + _L, _L, 0, 0, 0, 0, 0, 0, /* 0xa8-0xaf */ + _P, 0, 0, 0, 0, 0, 0, 0, /* 0xb0-0xb7 */ + 0, 0, _P, _P, 0, 0, 0, 0, /* 0xb8-0xbf */ + _P, _U|_X, _U|_X, _U|_X, _U|_X, _U|_X, _U|_X, _U, /* 0xc0-0xc7 */ + _U, _U, 0, 0, 0, 0, 0, 0, /* 0xc8-0xcf */ + _P, _U, _U, _U, _U, _U, _U, _U, /* 0xd0-0xd7 */ + _U, _U, 0, 0, 0, 0, 0, 0, /* 0xd8-0xdf */ + _P, 0, _U, _U, _U, _U, _U, _U, /* 0xe0-0xe7 */ + _U, _U, 0, 0, 0, 0, 0, 0, /* 0xe8-0xef */ + _D, _D, _D, _D, _D, _D, _D, _D, /* 0xf0-0xf7 */ + _D, _D, 0, 0, 0, 0, 0, _C, /* 0xf8-0xff */ +}; + EXPORT_SYMBOL(_ascebc_500); EXPORT_SYMBOL(_ebcasc_500); EXPORT_SYMBOL(_ascebc); EXPORT_SYMBOL(_ebcasc); EXPORT_SYMBOL(_ebc_tolower); EXPORT_SYMBOL(_ebc_toupper); - +EXPORT_SYMBOL(_ebctype_asc); From 8410516674b52a51ce2c15c7e2ea7dee2a8b6489 Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Wed, 2 Sep 2026 23:20:18 +0200 Subject: [PATCH 002/102] s390: Warn if kernel command line contains non-printable EBCDIC characters Users may accidentally add multi-byte UTF-8 characters to zipl.conf parmline, for example, by copying snippets containing non-breaking spaces (\xC2\xA0) from web pages. The kernel will then interpret the entire command line as EBCDIC, making it unusable. Distinguish this situation from the legitimate EBCDIC conversion by looking for non-printable characters and issue a warning. Accept the following tradeoffs: * Do not make this a hard failure, there may be a very small number of users who put characters with diacritics on their EBCDIC-encoded command lines. * Do not use heuristics with arbitrary thresholds, these may fail intermittently on, e.g., punctuation-heavy command lines, and having to tune thresholds in subsequent patches is not desirable. * Use the ASCII subset of EBCDIC to determine whether characters are printable: the vast majority of command lines will contain characters only from this subset. Signed-off-by: Ilya Leoshkevich Reviewed-by: Heiko Carstens Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/ipl_parm.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index c1b43e5e688a..637cb7b6343f 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -172,12 +172,23 @@ static inline int has_ebcdic_char(const char *str) return 0; } +static inline bool has_nonprintable_ebcdic_char(const char *str) +{ + for (; *str; str++) + if (!isprint_ebc_asc(*str) && !isspace_ebc_asc(*str)) + return true; + return false; +} + void setup_boot_command_line(void) { parmarea.command_line[COMMAND_LINE_SIZE - 1] = 0; /* convert arch command line to ascii if necessary */ - if (has_ebcdic_char(parmarea.command_line)) + if (has_ebcdic_char(parmarea.command_line)) { + if (has_nonprintable_ebcdic_char(parmarea.command_line)) + boot_warn("Kernel command line was treated as EBCDIC, but contains non-printable characters\n"); EBCASC(parmarea.command_line, COMMAND_LINE_SIZE); + } /* copy arch command line */ strscpy(early_command_line, strim(parmarea.command_line)); From 6db0f16f27e338e3b8b71be30609aa5f7823c8be Mon Sep 17 00:00:00 2001 From: Jan Polensky Date: Tue, 8 Sep 2026 15:58:43 +0200 Subject: [PATCH 003/102] s390/rwlock: Add contention tracepoints to rwlock slowpath Instrument arch_read_lock_wait() and arch_write_lock_wait() with trace_contention_begin() and trace_contention_end(). These tracepoints are used by lock contention analysis tools such as perf lock contention to identify contended rwlocks and measure wait times. The generic implementation in kernel/locking/qrwlock.c already emits the same events from its read and write slowpaths. For arch_read_lock_wait(), the in_interrupt() fast-path is intentionally left outside the tracepoint scope. That path spins without entering the wait queue, so it does not represent queue-based contention and its duration is not comparable to the normal slowpath. This matches the behaviour of queued_read_lock_slowpath() in qrwlock.c. The include for is already present from commit ffa796cc1f45 ("s390/spinlock: Add contention tracepoints to lock slowpath"). Signed-off-by: Jan Polensky Reviewed-by: Heiko Carstens Signed-off-by: Heiko Carstens --- arch/s390/lib/spinlock.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index dbabca35c008..c84991a1be20 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -318,6 +318,7 @@ void arch_read_lock_wait(arch_rwlock_t *rw) /* Remove this reader again to allow recursive read locking */ __atomic_add_const(-1, &rw->cnts); + trace_contention_begin(rw, LCB_F_SPIN | LCB_F_READ); /* Put the reader into the wait queue */ arch_spin_lock(&rw->wait); /* Now add this reader to the count value again */ @@ -326,6 +327,7 @@ void arch_read_lock_wait(arch_rwlock_t *rw) while (READ_ONCE(rw->cnts) & 0x10000) barrier(); arch_spin_unlock(&rw->wait); + trace_contention_end(rw, 0); } EXPORT_SYMBOL(arch_read_lock_wait); @@ -333,6 +335,7 @@ void arch_write_lock_wait(arch_rwlock_t *rw) { int old; + trace_contention_begin(rw, LCB_F_SPIN | LCB_F_WRITE); /* Add this CPU to the write waiters */ __atomic_add(0x20000, &rw->cnts); @@ -349,6 +352,7 @@ void arch_write_lock_wait(arch_rwlock_t *rw) } arch_spin_unlock(&rw->wait); + trace_contention_end(rw, 0); } EXPORT_SYMBOL(arch_write_lock_wait); From 9c5eebc9de1fe6fcc2adad8a6f5fbb506d1eb9d0 Mon Sep 17 00:00:00 2001 From: Euan Bourke Date: Fri, 11 Sep 2026 11:13:09 +0200 Subject: [PATCH 004/102] s390/hiperdispatch: Use system_dfl_wq to reschedule hd_capacity_work In hd_enable_hiperdispatch(), hd_capacity_work is scheduled on system_dfl_wq, however the work function reschedules itself with schedule_delayed_work(), which uses system_percpu_wq. Hiperdispatch does not benefit from using the percpu workqueue. Use queue_delayed_work() with system_dfl_wq for consistency. Changing the workqueue from system_percpu_wq to system_dfl_wq causes no functional changes. Signed-off-by: Euan Bourke Tested-by: Mete Durlu Reviewed-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/kernel/hiperdispatch.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/hiperdispatch.c b/arch/s390/kernel/hiperdispatch.c index 217206522266..8494823559b1 100644 --- a/arch/s390/kernel/hiperdispatch.c +++ b/arch/s390/kernel/hiperdispatch.c @@ -265,7 +265,7 @@ static void hd_capacity_work_fn(struct work_struct *work) } trace_s390_hd_work_fn(steal_percentage, hd_entitled_cores, hd_high_capacity_cores); mutex_unlock(&smp_cpu_state_mutex); - schedule_delayed_work(&hd_capacity_work, HD_DELAY_INTERVAL); + queue_delayed_work(system_dfl_wq, &hd_capacity_work, HD_DELAY_INTERVAL); } static int hiperdispatch_ctl_handler(const struct ctl_table *ctl, int write, From 1206e6e850c43ec9306a2ec0be04532fd4cf32cb Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 17 Sep 2026 14:12:40 -0700 Subject: [PATCH 005/102] s390/keyboard: Allocate key_maps and func_table with ARRAY_SIZE() In preparation for converting the kmalloc family of allocators to the type-aware kmalloc_obj family, we need to make sure that the returned type from the allocation matches the type of the variable being assigned. (The kmalloc family returns "void *", which can be implicitly cast to any pointer type.) These allocate as many pointers as ebc_key_maps and ebc_func_table hold, but the sizes were taken from the whole arrays, which would make the allocation types pointers to those arrays rather than the "u_short **" and "char **" being assigned. Allocate ARRAY_SIZE-many entries instead. The resulting allocation sizes are the same. Build tested ARCH=s390 allmodconfig with GCC s390x-linux-gnu 16.1.0: drivers/s390/char/keyboard.o Assisted-by: LLM coccinelle Signed-off-by: Kees Cook Signed-off-by: Heiko Carstens --- drivers/s390/char/keyboard.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/s390/char/keyboard.c b/drivers/s390/char/keyboard.c index ecd9ace2b3aa..63bcb52d6e87 100644 --- a/drivers/s390/char/keyboard.c +++ b/drivers/s390/char/keyboard.c @@ -81,7 +81,8 @@ kbd_alloc(void) { kbd = kzalloc_obj(struct kbd_data); if (!kbd) goto out; - kbd->key_maps = kzalloc(sizeof(ebc_key_maps), GFP_KERNEL); + kbd->key_maps = kcalloc(ARRAY_SIZE(ebc_key_maps), + sizeof(*kbd->key_maps), GFP_KERNEL); if (!kbd->key_maps) goto out_kbd; for (i = 0; i < ARRAY_SIZE(ebc_key_maps); i++) { @@ -93,7 +94,8 @@ kbd_alloc(void) { goto out_maps; } } - kbd->func_table = kzalloc(sizeof(ebc_func_table), GFP_KERNEL); + kbd->func_table = kcalloc(ARRAY_SIZE(ebc_func_table), + sizeof(*kbd->func_table), GFP_KERNEL); if (!kbd->func_table) goto out_maps; for (i = 0; i < ARRAY_SIZE(ebc_func_table); i++) { From c25f69edca8c77a197c8d55a2f2ea18ca6976c4c Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:08 +0300 Subject: [PATCH 006/102] s390/chsc: Use kzalloc() for CUBs cub_alloc() allocates the channel measurement unit blocks and their extended counterparts. This memory can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. Replace use of get_zeroed_page() with kzalloc() and free_page() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/chsc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index 9689f722c863..60a4a7e6086c 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -931,12 +931,12 @@ static int cub_alloc(struct channel_subsystem *css) int i; for (i = 0; i < CSS_NUM_CUB_PAGES; i++) { - css->cub[i] = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + css->cub[i] = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!css->cub[i]) return -ENOMEM; } for (i = 0; i < CSS_NUM_ECUB_PAGES; i++) { - css->ecub[i] = (void *)get_zeroed_page(GFP_KERNEL); + css->ecub[i] = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!css->ecub[i]) return -ENOMEM; } @@ -949,11 +949,11 @@ static void cub_free(struct channel_subsystem *css) int i; for (i = 0; i < CSS_NUM_CUB_PAGES; i++) { - free_page((unsigned long)css->cub[i]); + kfree(css->cub[i]); css->cub[i] = NULL; } for (i = 0; i < CSS_NUM_ECUB_PAGES; i++) { - free_page((unsigned long)css->ecub[i]); + kfree(css->ecub[i]); css->ecub[i] = NULL; } } From b5555485912a3a81e4038d9be5718c267e82807c Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:09 +0300 Subject: [PATCH 007/102] s390/chsc: Use kzalloc() for the SEI work area chsc_init() allocates the work area for store event information data. This memory can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. Replace use of get_zeroed_page() with kzalloc() and free_page() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/chsc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index 60a4a7e6086c..c3186c0a372b 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -1142,7 +1142,7 @@ int __init chsc_init(void) { int ret; - sei_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sei_page = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); chsc_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); if (!sei_page || !chsc_page) { ret = -ENOMEM; @@ -1154,7 +1154,7 @@ int __init chsc_init(void) return ret; out_err: free_page((unsigned long)chsc_page); - free_page((unsigned long)sei_page); + kfree(sei_page); return ret; } @@ -1162,7 +1162,7 @@ void __init chsc_init_cleanup(void) { crw_unregister_handler(CRW_RSC_CSS); free_page((unsigned long)chsc_page); - free_page((unsigned long)sei_page); + kfree(sei_page); } int __chsc_enable_facility(struct chsc_sda_area *sda_area, int operation_code) From a68e07158f50926e8c6b2bdac0204cbce9605abf Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:10 +0300 Subject: [PATCH 008/102] s390/chsc_sch: Use kzalloc() for CHSC requests The CHSC ioctl handlers allocate request and response areas for CHSC commands issued on behalf of userspace. This memory can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. Replace use of get_zeroed_page() with kzalloc() and free_page() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/chsc_sch.c | 44 ++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c index b6cb8bb8bcc4..c8d6871d1841 100644 --- a/drivers/s390/cio/chsc_sch.c +++ b/drivers/s390/cio/chsc_sch.c @@ -292,7 +292,7 @@ static int chsc_ioctl_start(void __user *user_area) if (!css_general_characteristics.dynio) /* It makes no sense to try. */ return -EOPNOTSUPP; - chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL); + chsc_area = kzalloc(PAGE_SIZE, GFP_DMA | GFP_KERNEL); if (!chsc_area) return -ENOMEM; request = kzalloc_obj(*request); @@ -321,7 +321,7 @@ static int chsc_ioctl_start(void __user *user_area) snprintf(dbf, sizeof(dbf), "ret:%d", ret); CHSC_LOG(0, dbf); kfree(request); - free_page((unsigned long)chsc_area); + kfree(chsc_area); return ret; } @@ -340,7 +340,7 @@ static int chsc_ioctl_on_close_set(void __user *user_area) ret = -ENOMEM; goto out_unlock; } - on_close_chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL); + on_close_chsc_area = kzalloc(PAGE_SIZE, GFP_DMA | GFP_KERNEL); if (!on_close_chsc_area) { ret = -ENOMEM; goto out_free_request; @@ -353,7 +353,7 @@ static int chsc_ioctl_on_close_set(void __user *user_area) goto out_unlock; out_free_chsc: - free_page((unsigned long)on_close_chsc_area); + kfree(on_close_chsc_area); on_close_chsc_area = NULL; out_free_request: kfree(on_close_request); @@ -375,7 +375,7 @@ static int chsc_ioctl_on_close_remove(void) ret = -ENOENT; goto out_unlock; } - free_page((unsigned long)on_close_chsc_area); + kfree(on_close_chsc_area); on_close_chsc_area = NULL; kfree(on_close_request); on_close_request = NULL; @@ -392,7 +392,7 @@ static int chsc_ioctl_start_sync(void __user *user_area) struct chsc_sync_area *chsc_area; int ret, ccode; - chsc_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + chsc_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!chsc_area) return -ENOMEM; if (copy_from_user(chsc_area, user_area, PAGE_SIZE)) { @@ -414,7 +414,7 @@ static int chsc_ioctl_start_sync(void __user *user_area) else ret = 0; out_free: - free_page((unsigned long)chsc_area); + kfree(chsc_area); return ret; } @@ -438,7 +438,7 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *scpcd_area; - scpcd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + scpcd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!scpcd_area) return -ENOMEM; cd = kzalloc_obj(*cd); @@ -476,7 +476,7 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd) ret = 0; out_free: kfree(cd); - free_page((unsigned long)scpcd_area); + kfree(scpcd_area); return ret; } @@ -500,7 +500,7 @@ static int chsc_ioctl_info_cu(void __user *user_cd) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *scucd_area; - scucd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + scucd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!scucd_area) return -ENOMEM; cd = kzalloc_obj(*cd); @@ -538,7 +538,7 @@ static int chsc_ioctl_info_cu(void __user *user_cd) ret = 0; out_free: kfree(cd); - free_page((unsigned long)scucd_area); + kfree(scucd_area); return ret; } @@ -563,7 +563,7 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *sscud_area; - sscud_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sscud_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sscud_area) return -ENOMEM; cud = kzalloc_obj(*cud); @@ -602,7 +602,7 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud) ret = 0; out_free: kfree(cud); - free_page((unsigned long)sscud_area); + kfree(sscud_area); return ret; } @@ -625,7 +625,7 @@ static int chsc_ioctl_conf_info(void __user *user_ci) u8 data[PAGE_SIZE - 20]; } __attribute__ ((packed)) *sci_area; - sci_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sci_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sci_area) return -ENOMEM; ci = kzalloc_obj(*ci); @@ -662,7 +662,7 @@ static int chsc_ioctl_conf_info(void __user *user_ci) ret = 0; out_free: kfree(ci); - free_page((unsigned long)sci_area); + kfree(sci_area); return ret; } @@ -696,7 +696,7 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl) u32 res; } __attribute__ ((packed)) *cssids_parm; - sccl_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sccl_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sccl_area) return -ENOMEM; ccl = kzalloc_obj(*ccl); @@ -745,7 +745,7 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl) ret = 0; out_free: kfree(ccl); - free_page((unsigned long)sccl_area); + kfree(sccl_area); return ret; } @@ -756,7 +756,7 @@ static int chsc_ioctl_chpd(void __user *user_chpd) int ret; chpd = kzalloc_obj(*chpd); - scpd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + scpd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!scpd_area || !chpd) { ret = -ENOMEM; goto out_free; @@ -775,7 +775,7 @@ static int chsc_ioctl_chpd(void __user *user_chpd) ret = -EFAULT; out_free: kfree(chpd); - free_page((unsigned long)scpd_area); + kfree(scpd_area); return ret; } @@ -796,7 +796,7 @@ static int chsc_ioctl_dcal(void __user *user_dcal) u8 data[PAGE_SIZE - 36]; } __attribute__ ((packed)) *sdcal_area; - sdcal_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + sdcal_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sdcal_area) return -ENOMEM; dcal = kzalloc_obj(*dcal); @@ -834,7 +834,7 @@ static int chsc_ioctl_dcal(void __user *user_dcal) ret = 0; out_free: kfree(dcal); - free_page((unsigned long)sdcal_area); + kfree(sdcal_area); return ret; } @@ -904,7 +904,7 @@ static int chsc_release(struct inode *inode, struct file *filp) } snprintf(dbf, sizeof(dbf), "relret:%d", ret); CHSC_LOG(0, dbf); - free_page((unsigned long)on_close_chsc_area); + kfree(on_close_chsc_area); on_close_chsc_area = NULL; kfree(on_close_request); on_close_request = NULL; From 4effd91250d211d67f7a4e2f3199994cee681d87 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:11 +0300 Subject: [PATCH 009/102] s390/chsc_sch: Use __free(kfree) for synchronous CHSC requests Use __free(kfree) for local buffers in the synchronous CHSC ioctl handlers and replace their cleanup labels with early returns. Keep the asynchronous and on-close handlers unchanged. Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/chsc_sch.c | 272 ++++++++++++------------------------ 1 file changed, 92 insertions(+), 180 deletions(-) diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c index c8d6871d1841..225aaac7dd30 100644 --- a/drivers/s390/cio/chsc_sch.c +++ b/drivers/s390/cio/chsc_sch.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -389,39 +390,29 @@ static int chsc_ioctl_on_close_remove(void) static int chsc_ioctl_start_sync(void __user *user_area) { - struct chsc_sync_area *chsc_area; - int ret, ccode; + struct chsc_sync_area *chsc_area __free(kfree) = NULL; + int ccode; chsc_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!chsc_area) return -ENOMEM; - if (copy_from_user(chsc_area, user_area, PAGE_SIZE)) { - ret = -EFAULT; - goto out_free; - } - if (chsc_area->header.code & 0x4000) { - ret = -EINVAL; - goto out_free; - } + if (copy_from_user(chsc_area, user_area, PAGE_SIZE)) + return -EFAULT; + if (chsc_area->header.code & 0x4000) + return -EINVAL; chsc_log_command(chsc_area); ccode = chsc(chsc_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (copy_to_user(user_area, chsc_area, PAGE_SIZE)) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(chsc_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_info_channel_path(void __user *user_cd) { - struct chsc_chp_cd *cd; - int ret, ccode; + struct chsc_chp_cd *cd __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 : 2; @@ -436,20 +427,16 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd) u32 : 32; struct chsc_header response; u8 data[PAGE_SIZE - 20]; - } __attribute__ ((packed)) *scpcd_area; + } __attribute__ ((packed)) *scpcd_area __free(kfree) = NULL; scpcd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!scpcd_area) return -ENOMEM; cd = kzalloc_obj(*cd); - if (!cd) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(cd, user_cd, sizeof(*cd))) { - ret = -EFAULT; - goto out_free; - } + if (!cd) + return -ENOMEM; + if (copy_from_user(cd, user_cd, sizeof(*cd))) + return -EFAULT; scpcd_area->request.length = 0x0010; scpcd_area->request.code = 0x0028; scpcd_area->m = cd->m; @@ -459,31 +446,23 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd) scpcd_area->last_chpid = cd->chpid.id; ccode = chsc(scpcd_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (scpcd_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "scpcd: response code=%x\n", scpcd_area->response.code); - goto out_free; + return -EIO; } memcpy(&cd->cpcb, &scpcd_area->response, scpcd_area->response.length); if (copy_to_user(user_cd, cd, sizeof(*cd))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(cd); - kfree(scpcd_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_info_cu(void __user *user_cd) { - struct chsc_cu_cd *cd; - int ret, ccode; + struct chsc_cu_cd *cd __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 : 2; @@ -498,20 +477,16 @@ static int chsc_ioctl_info_cu(void __user *user_cd) u32 : 32; struct chsc_header response; u8 data[PAGE_SIZE - 20]; - } __attribute__ ((packed)) *scucd_area; + } __attribute__ ((packed)) *scucd_area __free(kfree) = NULL; scucd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!scucd_area) return -ENOMEM; cd = kzalloc_obj(*cd); - if (!cd) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(cd, user_cd, sizeof(*cd))) { - ret = -EFAULT; - goto out_free; - } + if (!cd) + return -ENOMEM; + if (copy_from_user(cd, user_cd, sizeof(*cd))) + return -EFAULT; scucd_area->request.length = 0x0010; scucd_area->request.code = 0x0026; scucd_area->m = cd->m; @@ -521,31 +496,23 @@ static int chsc_ioctl_info_cu(void __user *user_cd) scucd_area->last_cun = cd->cun; ccode = chsc(scucd_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (scucd_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "scucd: response code=%x\n", scucd_area->response.code); - goto out_free; + return -EIO; } memcpy(&cd->cucb, &scucd_area->response, scucd_area->response.length); if (copy_to_user(user_cd, cd, sizeof(*cd))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(cd); - kfree(scucd_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_info_sch_cu(void __user *user_cud) { - struct chsc_sch_cud *cud; - int ret, ccode; + struct chsc_sch_cud *cud __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 : 2; @@ -561,20 +528,16 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud) u32 : 32; struct chsc_header response; u8 data[PAGE_SIZE - 20]; - } __attribute__ ((packed)) *sscud_area; + } __attribute__ ((packed)) *sscud_area __free(kfree) = NULL; sscud_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sscud_area) return -ENOMEM; cud = kzalloc_obj(*cud); - if (!cud) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(cud, user_cud, sizeof(*cud))) { - ret = -EFAULT; - goto out_free; - } + if (!cud) + return -ENOMEM; + if (copy_from_user(cud, user_cud, sizeof(*cud))) + return -EFAULT; sscud_area->request.length = 0x0010; sscud_area->request.code = 0x0006; sscud_area->m = cud->schid.m; @@ -585,31 +548,23 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud) sscud_area->last_sch = cud->schid.sch_no; ccode = chsc(sscud_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (sscud_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "sscud: response code=%x\n", sscud_area->response.code); - goto out_free; + return -EIO; } memcpy(&cud->scub, &sscud_area->response, sscud_area->response.length); if (copy_to_user(user_cud, cud, sizeof(*cud))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(cud); - kfree(sscud_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_conf_info(void __user *user_ci) { - struct chsc_conf_info *ci; - int ret, ccode; + struct chsc_conf_info *ci __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 : 2; @@ -623,20 +578,16 @@ static int chsc_ioctl_conf_info(void __user *user_ci) u64 : 64; struct chsc_header response; u8 data[PAGE_SIZE - 20]; - } __attribute__ ((packed)) *sci_area; + } __attribute__ ((packed)) *sci_area __free(kfree) = NULL; sci_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sci_area) return -ENOMEM; ci = kzalloc_obj(*ci); - if (!ci) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(ci, user_ci, sizeof(*ci))) { - ret = -EFAULT; - goto out_free; - } + if (!ci) + return -ENOMEM; + if (copy_from_user(ci, user_ci, sizeof(*ci))) + return -EFAULT; sci_area->request.length = 0x0010; sci_area->request.code = 0x0012; sci_area->m = ci->id.m; @@ -645,31 +596,23 @@ static int chsc_ioctl_conf_info(void __user *user_ci) sci_area->ssid = ci->id.ssid; ccode = chsc(sci_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (sci_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "sci: response code=%x\n", sci_area->response.code); - goto out_free; + return -EIO; } memcpy(&ci->scid, &sci_area->response, sci_area->response.length); if (copy_to_user(user_ci, ci, sizeof(*ci))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(ci); - kfree(sci_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_conf_comp_list(void __user *user_ccl) { - struct chsc_comp_list *ccl; - int ret, ccode; + struct chsc_comp_list *ccl __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 ctype : 8; @@ -681,7 +624,7 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl) u64 : 64; struct chsc_header response; u8 data[PAGE_SIZE - 36]; - } __attribute__ ((packed)) *sccl_area; + } __attribute__ ((packed)) *sccl_area __free(kfree) = NULL; struct { u32 m : 1; u32 : 31; @@ -700,14 +643,10 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl) if (!sccl_area) return -ENOMEM; ccl = kzalloc_obj(*ccl); - if (!ccl) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(ccl, user_ccl, sizeof(*ccl))) { - ret = -EFAULT; - goto out_free; - } + if (!ccl) + return -ENOMEM; + if (copy_from_user(ccl, user_ccl, sizeof(*ccl))) + return -EFAULT; sccl_area->request.length = 0x0020; sccl_area->request.code = 0x0030; sccl_area->fmt = ccl->req.fmt; @@ -728,61 +667,46 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl) break; } ccode = chsc(sccl_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (sccl_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "sccl: response code=%x\n", sccl_area->response.code); - goto out_free; + return -EIO; } memcpy(&ccl->sccl, &sccl_area->response, sccl_area->response.length); if (copy_to_user(user_ccl, ccl, sizeof(*ccl))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(ccl); - kfree(sccl_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_chpd(void __user *user_chpd) { - struct chsc_scpd *scpd_area; - struct chsc_cpd_info *chpd; + struct chsc_scpd *scpd_area __free(kfree) = NULL; + struct chsc_cpd_info *chpd __free(kfree) = NULL; int ret; chpd = kzalloc_obj(*chpd); scpd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); - if (!scpd_area || !chpd) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(chpd, user_chpd, sizeof(*chpd))) { - ret = -EFAULT; - goto out_free; - } + if (!scpd_area || !chpd) + return -ENOMEM; + if (copy_from_user(chpd, user_chpd, sizeof(*chpd))) + return -EFAULT; ret = chsc_determine_channel_path_desc(chpd->chpid, chpd->fmt, chpd->rfmt, chpd->c, chpd->m, scpd_area); if (ret) - goto out_free; + return ret; memcpy(&chpd->chpdb, &scpd_area->response, scpd_area->response.length); if (copy_to_user(user_chpd, chpd, sizeof(*chpd))) - ret = -EFAULT; -out_free: - kfree(chpd); - kfree(scpd_area); - return ret; + return -EFAULT; + return 0; } static int chsc_ioctl_dcal(void __user *user_dcal) { - struct chsc_dcal *dcal; - int ret, ccode; + struct chsc_dcal *dcal __free(kfree) = NULL; + int ccode; struct { struct chsc_header request; u32 atype : 8; @@ -794,20 +718,16 @@ static int chsc_ioctl_dcal(void __user *user_dcal) u32 res1[2]; struct chsc_header response; u8 data[PAGE_SIZE - 36]; - } __attribute__ ((packed)) *sdcal_area; + } __attribute__ ((packed)) *sdcal_area __free(kfree) = NULL; sdcal_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sdcal_area) return -ENOMEM; dcal = kzalloc_obj(*dcal); - if (!dcal) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(dcal, user_dcal, sizeof(*dcal))) { - ret = -EFAULT; - goto out_free; - } + if (!dcal) + return -ENOMEM; + if (copy_from_user(dcal, user_dcal, sizeof(*dcal))) + return -EFAULT; sdcal_area->request.length = 0x0020; sdcal_area->request.code = 0x0034; sdcal_area->atype = dcal->req.atype; @@ -816,26 +736,18 @@ static int chsc_ioctl_dcal(void __user *user_dcal) sizeof(sdcal_area->list_parm)); ccode = chsc(sdcal_area); - if (ccode != 0) { - ret = -EIO; - goto out_free; - } + if (ccode != 0) + return -EIO; if (sdcal_area->response.code != 0x0001) { - ret = -EIO; CHSC_MSG(0, "sdcal: response code=%x\n", sdcal_area->response.code); - goto out_free; + return -EIO; } memcpy(&dcal->sdcal, &sdcal_area->response, sdcal_area->response.length); if (copy_to_user(user_dcal, dcal, sizeof(*dcal))) - ret = -EFAULT; - else - ret = 0; -out_free: - kfree(dcal); - kfree(sdcal_area); - return ret; + return -EFAULT; + return 0; } static long chsc_ioctl(struct file *filp, unsigned int cmd, From a59ccfb427c60d25ca1e5039e3d4e22f65acc51e Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:12 +0300 Subject: [PATCH 010/102] s390/cio: Use kzalloc() for CHSC work areas chsc_init() allocates the work area for CHSC commands and qdio_allocate() allocates one for CHSC calls during qdio_establish(). This memory can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. While on it, change qdio_irq.chsc_page to void * to get rid of the casts. Replace use of get_zeroed_page() with kzalloc() and free_page() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/chsc.c | 6 +++--- drivers/s390/cio/qdio.h | 2 +- drivers/s390/cio/qdio_main.c | 6 +++--- drivers/s390/cio/qdio_setup.c | 2 +- drivers/s390/cio/qdio_thinint.c | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index c3186c0a372b..d98f629e8929 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -1143,7 +1143,7 @@ int __init chsc_init(void) int ret; sei_page = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); - chsc_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + chsc_page = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!sei_page || !chsc_page) { ret = -ENOMEM; goto out_err; @@ -1153,7 +1153,7 @@ int __init chsc_init(void) goto out_err; return ret; out_err: - free_page((unsigned long)chsc_page); + kfree(chsc_page); kfree(sei_page); return ret; } @@ -1161,7 +1161,7 @@ int __init chsc_init(void) void __init chsc_init_cleanup(void) { crw_unregister_handler(CRW_RSC_CSS); - free_page((unsigned long)chsc_page); + kfree(chsc_page); kfree(sei_page); } diff --git a/drivers/s390/cio/qdio.h b/drivers/s390/cio/qdio.h index 4bd4c00c9c0c..dff5f53a8795 100644 --- a/drivers/s390/cio/qdio.h +++ b/drivers/s390/cio/qdio.h @@ -244,7 +244,7 @@ struct qdio_irq { int perf_stat_enabled; struct qdr *qdr; - unsigned long chsc_page; + void *chsc_page; struct qdio_q *input_qs[QDIO_MAX_QUEUES_PER_IRQ]; struct qdio_q *output_qs[QDIO_MAX_QUEUES_PER_IRQ]; diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c index c1e09fa34e77..d137bf8c7066 100644 --- a/drivers/s390/cio/qdio_main.c +++ b/drivers/s390/cio/qdio_main.c @@ -934,7 +934,7 @@ int qdio_free(struct ccw_device *cdev) qdio_free_queues(irq_ptr); free_page((unsigned long) irq_ptr->qdr); - free_page(irq_ptr->chsc_page); + kfree(irq_ptr->chsc_page); kfree(irq_ptr->ccw); free_page((unsigned long) irq_ptr); return 0; @@ -986,7 +986,7 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, * qdio_establish. In case of low memory and swap on a zfcp disk * we may not be able to allocate memory otherwise. */ - irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL); + irq_ptr->chsc_page = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!irq_ptr->chsc_page) goto err_chsc; @@ -1006,7 +1006,7 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, err_queues: free_page((unsigned long) irq_ptr->qdr); err_qdr: - free_page(irq_ptr->chsc_page); + kfree(irq_ptr->chsc_page); err_chsc: err_dbf: kfree(irq_ptr->ccw); diff --git a/drivers/s390/cio/qdio_setup.c b/drivers/s390/cio/qdio_setup.c index ea09aadaae4e..bd80703c174c 100644 --- a/drivers/s390/cio/qdio_setup.c +++ b/drivers/s390/cio/qdio_setup.c @@ -253,7 +253,7 @@ int qdio_setup_get_ssqd(struct qdio_irq *irq_ptr, if (!ssqd) return -ENOMEM; } else { - ssqd = (struct chsc_ssqd_area *)irq_ptr->chsc_page; + ssqd = irq_ptr->chsc_page; } rc = chsc_ssqd(*schid, ssqd); diff --git a/drivers/s390/cio/qdio_thinint.c b/drivers/s390/cio/qdio_thinint.c index e167aa75c3df..a2ff51887537 100644 --- a/drivers/s390/cio/qdio_thinint.c +++ b/drivers/s390/cio/qdio_thinint.c @@ -136,7 +136,7 @@ static struct airq_struct tiqdio_airq = { static int set_subchannel_ind(struct qdio_irq *irq_ptr, int reset) { - struct chsc_scssc_area *scssc = (void *)irq_ptr->chsc_page; + struct chsc_scssc_area *scssc = irq_ptr->chsc_page; dma64_t summary_indicator_addr, subchannel_indicator_addr; int rc; From db111020cc720a39480f67b9bb8308dd7c0c7099 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:13 +0300 Subject: [PATCH 011/102] s390/cmf: Use kmalloc() for the CMB area alloc_cmb() allocates the channel measurement block area shared by devices using the basic channel measurement format. This memory can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. The measurement block origin must be 32-byte aligned. Each CMB is 32 bytes, so kmalloc() provides the required alignment without rounding the allocation to a power-of-two number of pages. Reject an empty area before allocating it. Replace use of __get_free_pages() with kmalloc() and free_pages() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/cmf.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c index 92ab3d546fe4..8fd41a7dfc20 100644 --- a/drivers/s390/cio/cmf.c +++ b/drivers/s390/cio/cmf.c @@ -482,6 +482,9 @@ static int alloc_cmb(struct ccw_device *cdev) ssize_t size; struct cmb_data *cmb_data; + if (cmb_area.num_channels <= 0) + return -ENOMEM; + /* Allocate private cmb_data. */ cmb_data = kzalloc_obj(struct cmb_data); if (!cmb_data) @@ -501,12 +504,12 @@ static int alloc_cmb(struct ccw_device *cdev) WARN_ON(!list_empty(&cmb_area.list)); spin_unlock(&cmb_area.lock); - mem = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); + mem = kmalloc(size, GFP_KERNEL); spin_lock(&cmb_area.lock); if (cmb_area.mem) { /* ok, another thread was faster */ - free_pages((unsigned long)mem, get_order(size)); + kfree(mem); } else if (!mem) { /* no luck */ ret = -ENOMEM; @@ -547,10 +550,8 @@ static void free_cmb(struct ccw_device *cdev) list_del_init(&priv->cmb_list); if (list_empty(&cmb_area.list)) { - ssize_t size; - size = sizeof(struct cmb) * cmb_area.num_channels; cmf_activate(NULL, CMF_OFF); - free_pages((unsigned long)cmb_area.mem, get_order(size)); + kfree(cmb_area.mem); cmb_area.mem = NULL; } spin_unlock_irq(cdev->ccwlock); From 91e4448b300a97422f0233d6273649d912c1e741 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:14 +0300 Subject: [PATCH 012/102] s390/idals: Use kmalloc() for IDAL data buffers idal_buffer_alloc() allocates the data chunks of an IDAL buffer that is used for channel I/O. These buffers can be allocated with kmalloc() as there's nothing special about them to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. Replace use of __get_free_pages() with kmalloc() and free_pages() with kfree(). While on it, tell kmemleak to ignore the data chunks as only their DMA addresses are retained in the IDAL buffer. Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- arch/s390/include/asm/idals.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/arch/s390/include/asm/idals.h b/arch/s390/include/asm/idals.h index 06e1ec2afd5a..213f430a8d33 100644 --- a/arch/s390/include/asm/idals.h +++ b/arch/s390/include/asm/idals.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -147,9 +148,11 @@ static inline struct idal_buffer *idal_buffer_alloc(size_t size, int page_order) ib->data[i] = dma64_add(ib->data[i - 1], IDA_BLOCK_SIZE); continue; } - vaddr = (void *)__get_free_pages(GFP_KERNEL, page_order); + vaddr = kmalloc(PAGE_SIZE << page_order, GFP_KERNEL); if (!vaddr) goto error; + /* Only DMA addresses are retained in ib->data. */ + kmemleak_ignore(vaddr); ib->data[i] = virt_to_dma64(vaddr); } return ib; @@ -157,7 +160,7 @@ static inline struct idal_buffer *idal_buffer_alloc(size_t size, int page_order) while (i >= nr_chunks) { i -= nr_chunks; vaddr = dma64_to_virt(ib->data[i]); - free_pages((unsigned long)vaddr, ib->page_order); + kfree(vaddr); } kfree(ib); return ERR_PTR(-ENOMEM); @@ -175,7 +178,7 @@ static inline void idal_buffer_free(struct idal_buffer *ib) nr_chunks = (PAGE_SIZE << ib->page_order) >> IDA_SIZE_SHIFT; for (i = 0; i < nr_ptrs; i += nr_chunks) { vaddr = dma64_to_virt(ib->data[i]); - free_pages((unsigned long)vaddr, ib->page_order); + kfree(vaddr); } kfree(ib); } From c76ff72b390b553da029ed79852553e3b917a408 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:15 +0300 Subject: [PATCH 013/102] s390/qdio_main: Use kzalloc() for the IRQ structure qdio_allocate() allocates the QDIO irq structure. This memory can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. Replace use of get_zeroed_page() with kzalloc() and free_page() with kfree(). While on it, drop the kmemleak exemption for the CCW. The IRQ structure is now tracked, so kmemleak can find its pointer to the CCW. Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/qdio_main.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c index d137bf8c7066..28eb8602ca97 100644 --- a/drivers/s390/cio/qdio_main.c +++ b/drivers/s390/cio/qdio_main.c @@ -12,11 +12,11 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include @@ -936,7 +936,7 @@ int qdio_free(struct ccw_device *cdev) free_page((unsigned long) irq_ptr->qdr); kfree(irq_ptr->chsc_page); kfree(irq_ptr->ccw); - free_page((unsigned long) irq_ptr); + kfree(irq_ptr); return 0; } EXPORT_SYMBOL_GPL(qdio_free); @@ -961,7 +961,7 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, no_output_qs > QDIO_MAX_QUEUES_PER_IRQ) return -EINVAL; - irq_ptr = (void *) get_zeroed_page(GFP_KERNEL); + irq_ptr = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!irq_ptr) return -ENOMEM; @@ -969,9 +969,6 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, if (!irq_ptr->ccw) goto err_ccw; - /* kmemleak doesn't scan the page-allocated irq_ptr: */ - kmemleak_not_leak(irq_ptr->ccw); - irq_ptr->cdev = cdev; mutex_init(&irq_ptr->setup_mutex); if (qdio_allocate_dbf(irq_ptr)) @@ -1011,7 +1008,7 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, err_dbf: kfree(irq_ptr->ccw); err_ccw: - free_page((unsigned long) irq_ptr); + kfree(irq_ptr); return rc; } EXPORT_SYMBOL_GPL(qdio_allocate); From b03a58a24497c1b132bbded96d8b416b3a9470be Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:16 +0300 Subject: [PATCH 014/102] s390/qdio_main: Use kzalloc() for the QDR qdio_allocate() allocates the queue description record (QDR). This memory can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. Replace use of get_zeroed_page() with kzalloc() and free_page() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/qdio_main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c index 28eb8602ca97..889f63f94a4a 100644 --- a/drivers/s390/cio/qdio_main.c +++ b/drivers/s390/cio/qdio_main.c @@ -933,7 +933,7 @@ int qdio_free(struct ccw_device *cdev) mutex_unlock(&irq_ptr->setup_mutex); qdio_free_queues(irq_ptr); - free_page((unsigned long) irq_ptr->qdr); + kfree(irq_ptr->qdr); kfree(irq_ptr->chsc_page); kfree(irq_ptr->ccw); kfree(irq_ptr); @@ -988,7 +988,7 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, goto err_chsc; /* qdr is used in ccw1.cda which is u32 */ - irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA); + irq_ptr->qdr = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!irq_ptr->qdr) goto err_qdr; @@ -1001,7 +1001,7 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, return 0; err_queues: - free_page((unsigned long) irq_ptr->qdr); + kfree(irq_ptr->qdr); err_qdr: kfree(irq_ptr->chsc_page); err_chsc: From a0152eeabf40a0af8118cad9e0402714c15d12a2 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:17 +0300 Subject: [PATCH 015/102] s390/qdio_setup: Use kzalloc() for QDIO buffers qdio_alloc_buffers() allocates the QDIO buffers. This memory can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. Replace use of get_zeroed_page() with kzalloc() and free_page() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/qdio_setup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/qdio_setup.c b/drivers/s390/cio/qdio_setup.c index bd80703c174c..b6f2ff202fe6 100644 --- a/drivers/s390/cio/qdio_setup.c +++ b/drivers/s390/cio/qdio_setup.c @@ -35,7 +35,7 @@ void qdio_free_buffers(struct qdio_buffer **buf, unsigned int count) int pos; for (pos = 0; pos < count; pos += QBUFF_PER_PAGE) - free_page((unsigned long) buf[pos]); + kfree(buf[pos]); } EXPORT_SYMBOL_GPL(qdio_free_buffers); @@ -49,7 +49,7 @@ int qdio_alloc_buffers(struct qdio_buffer **buf, unsigned int count) int pos; for (pos = 0; pos < count; pos += QBUFF_PER_PAGE) { - buf[pos] = (void *) get_zeroed_page(GFP_KERNEL); + buf[pos] = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!buf[pos]) { qdio_free_buffers(buf, count); return -ENOMEM; From 77e5421c112a99610f4649c3e63ccc33627b30d0 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:18 +0300 Subject: [PATCH 016/102] s390/qdio_setup: Use kmalloc() for the storage list __qdio_allocate_qs() allocates the storage list information block of a queue. This memory can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. Replace use of __get_free_page() with kmalloc() and free_page() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/qdio_setup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/qdio_setup.c b/drivers/s390/cio/qdio_setup.c index b6f2ff202fe6..d8e9c0872d15 100644 --- a/drivers/s390/cio/qdio_setup.c +++ b/drivers/s390/cio/qdio_setup.c @@ -83,7 +83,7 @@ static void __qdio_free_queues(struct qdio_q **queues, unsigned int count) for (i = 0; i < count; i++) { q = queues[i]; - free_page((unsigned long)q->sl_page); + kfree(q->sl_page); kmem_cache_free(qdio_q_cache, q); } } @@ -109,7 +109,7 @@ static int __qdio_allocate_qs(struct qdio_q **irq_ptr_qs, int nr_queues) return -ENOMEM; } - q->sl_page = (void *)__get_free_page(GFP_KERNEL); + q->sl_page = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!q->sl_page) { kmem_cache_free(qdio_q_cache, q); __qdio_free_queues(irq_ptr_qs, i); From 4380decdce1f8bed35b0d4b7b23c574f95c528b5 Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:19 +0300 Subject: [PATCH 017/102] s390/qdio_setup: Use kmalloc() for the SSQD request qdio_setup_get_ssqd() allocates the request block for the Store Subchannel QDIO Data (SSQD) CHSC command. This memory can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. Replace use of __get_free_page() with kmalloc() and free_page() with kfree(). Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/qdio_setup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/qdio_setup.c b/drivers/s390/cio/qdio_setup.c index d8e9c0872d15..e1c5c6a8b3b2 100644 --- a/drivers/s390/cio/qdio_setup.c +++ b/drivers/s390/cio/qdio_setup.c @@ -249,7 +249,7 @@ int qdio_setup_get_ssqd(struct qdio_irq *irq_ptr, DBF_EVENT("getssqd:%4x", schid->sch_no); if (!irq_ptr) { - ssqd = (struct chsc_ssqd_area *)__get_free_page(GFP_KERNEL); + ssqd = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!ssqd) return -ENOMEM; } else { @@ -270,7 +270,7 @@ int qdio_setup_get_ssqd(struct qdio_irq *irq_ptr, out: if (!irq_ptr) - free_page((unsigned long)ssqd); + kfree(ssqd); return rc; } From e95e9d86f2be30206a95bd3b8488079231e3e98a Mon Sep 17 00:00:00 2001 From: "Mike Rapoport (Microsoft)" Date: Wed, 16 Sep 2026 15:03:20 +0300 Subject: [PATCH 018/102] s390/scm: Use kmalloc() for SCM information scm_update_information() allocates the response buffer for the CHSC Store SCM Information command. This buffer can be allocated with kmalloc() as there's nothing special about it to go directly to the page allocator. kmalloc() provides a better API that does not require ugly casts and kfree() does not need to know the size of the freed object. Performance difference between kmalloc() and __get_free_pages() is not measurable as both allocators take an object/page from a per-CPU list for fast path allocations. For the slow path the performance is anyway determined by the amount of reclaim involved rather than by what allocator is used. Replace use of __get_free_page() with kmalloc() and free_page() with kfree(). While on it, use __free(kfree) for the local response buffer. Link: https://lore.kernel.org/all/635405e4-9423-4a25-a6e7-e03c8ea0bcbe@redhat.com Assisted-by: copilot:claude-opus Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Heiko Carstens Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/scm.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/s390/cio/scm.c b/drivers/s390/cio/scm.c index 171212a6d2d9..b79fe023ab8d 100644 --- a/drivers/s390/cio/scm.c +++ b/drivers/s390/cio/scm.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "chsc.h" @@ -224,12 +225,12 @@ static int scm_add(struct chsc_scm_info *scm_info, size_t num) int scm_update_information(void) { - struct chsc_scm_info *scm_info; + struct chsc_scm_info *scm_info __free(kfree) = NULL; u64 token = 0; size_t num; int ret; - scm_info = (void *)__get_free_page(GFP_KERNEL | GFP_DMA); + scm_info = kmalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); if (!scm_info) return -ENOMEM; @@ -250,8 +251,6 @@ int scm_update_information(void) token = scm_info->restok; } while (token); - free_page((unsigned long)scm_info); - return ret; } From 2fc0d80b9d961a9f43c7c3fa840079e4cd629e47 Mon Sep 17 00:00:00 2001 From: Maximilian Immanuel Brandtner Date: Thu, 24 Sep 2026 12:47:31 +0200 Subject: [PATCH 019/102] s390/sclp: Introduce dedicated sclp-vt220 event buffer type Introduce sclp-vt220 event buffer type to replace the byte-manipulation behavior in the receiver function with well defined primitives. Signed-off-by: Maximilian Immanuel Brandtner Reviewed-by: Peter Oberparleiter Reviewed-by: Hendrik Brueckner Signed-off-by: Heiko Carstens --- drivers/s390/char/sclp_vt220.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/drivers/s390/char/sclp_vt220.c b/drivers/s390/char/sclp_vt220.c index 4e2bf8186a5d..cd5e5dba460d 100644 --- a/drivers/s390/char/sclp_vt220.c +++ b/drivers/s390/char/sclp_vt220.c @@ -52,6 +52,12 @@ struct sclp_vt220_sccb { struct evbuf_header evbuf; }; +struct sclp_vt220_evbuf { + struct evbuf_header header; + char type; + char data[]; +} __packed; + #define SCLP_VT220_MAX_CHARS_PER_BUFFER (PAGE_SIZE - \ sizeof(struct sclp_vt220_request) - \ sizeof(struct sclp_vt220_sccb)) @@ -519,25 +525,25 @@ static void sclp_vt220_handle_input(const char *buffer, unsigned int count) /* * Called by the SCLP to report incoming event buffers. */ -static void -sclp_vt220_receiver_fn(struct evbuf_header *evbuf) +static void sclp_vt220_receiver_fn(struct evbuf_header *evbuf) { - char *buffer; + struct sclp_vt220_evbuf *buffer; unsigned int count; - buffer = (char *) ((addr_t) evbuf + sizeof(struct evbuf_header)); - count = evbuf->length - sizeof(struct evbuf_header); + if (evbuf->length < offsetof(struct sclp_vt220_evbuf, data)) + return; - switch (*buffer) { + buffer = (struct sclp_vt220_evbuf *)evbuf; + count = evbuf->length - offsetof(struct sclp_vt220_evbuf, data); + + switch (buffer->type) { case SCLP_VT220_SESSION_ENDED: case SCLP_VT220_SESSION_STARTED: sclp_vt220_reset_session(); break; case SCLP_VT220_SESSION_DATA: /* Send input to line discipline */ - buffer++; - count--; - sclp_vt220_handle_input(buffer, count); + sclp_vt220_handle_input(buffer->data, count); tty_flip_buffer_push(&sclp_vt220_port); break; } From 0287e5076147ba6a9b0d242e1666e1619fdecdb0 Mon Sep 17 00:00:00 2001 From: Maximilian Immanuel Brandtner Date: Thu, 24 Sep 2026 12:47:32 +0200 Subject: [PATCH 020/102] s390/sclp: Implement resize for sclp-vt220 console Add support for sclp vt220 resize events to enable host-initiated terminal resizing when using the sclp-vt220 console. This allows QEMU to dynamically resize the console. On older kernel versions, these events are safely ignored as the kernel drops events with undefined tty codes. Signed-off-by: Maximilian Immanuel Brandtner Reviewed-by: Hendrik Brueckner Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/char/sclp_vt220.c | 63 +++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/drivers/s390/char/sclp_vt220.c b/drivers/s390/char/sclp_vt220.c index cd5e5dba460d..b95af0974624 100644 --- a/drivers/s390/char/sclp_vt220.c +++ b/drivers/s390/char/sclp_vt220.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include "sclp.h" @@ -58,6 +59,11 @@ struct sclp_vt220_evbuf { char data[]; } __packed; +struct sclp_vt220_resize_data_t { + u16 rows; + u16 cols; +} __packed; + #define SCLP_VT220_MAX_CHARS_PER_BUFFER (PAGE_SIZE - \ sizeof(struct sclp_vt220_request) - \ sizeof(struct sclp_vt220_sccb)) @@ -95,6 +101,15 @@ static int __initdata sclp_vt220_init_count; * another buffer */ static int sclp_vt220_flush_later; +/* Work struct required for scheduling resize */ +static struct work_struct sclp_vt220_resize_work; + +/* Current vt220 terminal winsize */ +static struct winsize sclp_vt220_winsize = { + .ws_row = 24, + .ws_col = 80, +}; + static void sclp_vt220_receiver_fn(struct evbuf_header *evbuf); static int __sclp_vt220_emit(struct sclp_vt220_request *request); static void sclp_vt220_emit_current(void); @@ -474,6 +489,7 @@ sclp_vt220_write(struct tty_struct *tty, const u8 *buf, size_t count) #define SCLP_VT220_SESSION_ENDED 0x01 #define SCLP_VT220_SESSION_STARTED 0x80 #define SCLP_VT220_SESSION_DATA 0x00 +#define SCLP_VT220_SESSION_RESIZE 0x08 #ifdef CONFIG_MAGIC_SYSRQ @@ -522,6 +538,38 @@ static void sclp_vt220_handle_input(const char *buffer, unsigned int count) #endif +static void sclp_vt220_resize(struct work_struct *work) +{ + struct tty_struct *tty; + struct winsize ws; + + spin_lock_irq(&sclp_vt220_lock); + ws = sclp_vt220_winsize; + spin_unlock_irq(&sclp_vt220_lock); + + tty = tty_port_tty_get(&sclp_vt220_port); + if (!tty) + return; + + tty_do_resize(tty, &ws); + tty_kref_put(tty); +} + +static void sclp_vt220_resize_sched(void *buffer, unsigned int count) +{ + struct sclp_vt220_resize_data_t *data = buffer; + unsigned long flags; + + if (count < sizeof(*data)) + return; + + spin_lock_irqsave(&sclp_vt220_lock, flags); + sclp_vt220_winsize.ws_row = data->rows; + sclp_vt220_winsize.ws_col = data->cols; + schedule_work(&sclp_vt220_resize_work); + spin_unlock_irqrestore(&sclp_vt220_lock, flags); +} + /* * Called by the SCLP to report incoming event buffers. */ @@ -546,6 +594,9 @@ static void sclp_vt220_receiver_fn(struct evbuf_header *evbuf) sclp_vt220_handle_input(buffer->data, count); tty_flip_buffer_push(&sclp_vt220_port); break; + case SCLP_VT220_SESSION_RESIZE: + sclp_vt220_resize_sched(buffer->data, count); + break; } } @@ -557,10 +608,12 @@ sclp_vt220_open(struct tty_struct *tty, struct file *filp) { if (tty->count == 1) { tty_port_tty_set(&sclp_vt220_port, tty); - if (!tty->winsize.ws_row && !tty->winsize.ws_col) { - tty->winsize.ws_row = 24; - tty->winsize.ws_col = 80; - } + + mutex_lock(&tty->winsize_mutex); + spin_lock_irq(&sclp_vt220_lock); + tty->winsize = sclp_vt220_winsize; + spin_unlock_irq(&sclp_vt220_lock); + mutex_unlock(&tty->winsize_mutex); } return 0; } @@ -753,6 +806,8 @@ static int __init sclp_vt220_tty_init(void) rc = tty_register_driver(driver); if (rc) goto out_init; + + INIT_WORK(&sclp_vt220_resize_work, sclp_vt220_resize); rc = sclp_register(&sclp_vt220_register_input); if (rc) goto out_reg; From 2b97e984b18a003256ed7c4aa20bf4b6661e4750 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 21 Sep 2026 17:57:55 +0200 Subject: [PATCH 021/102] s390/percpu: Fix comment typo This is all preempt_disable() / preempt_enable() pairs and not preempt_disable() / preempt_disable() pairs. Acked-by: Alexander Gordeev Tested-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index feb58acf87d6..31012abe8292 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -63,7 +63,7 @@ /* * Macros to be used for percpu code section based on atomic instructions. * - * Avoid the need to use preempt_disable() / preempt_disable() pairs and the + * Avoid the need to use preempt_disable() / preempt_enable() pairs and the * conditional preempt_schedule_notrace() function calls which come with * this. The idea is that this_cpu operations based on atomic instructions are * guarded with mviy instructions: From 965869bd2a4f2df44c84fc44f2d2038d14b45978 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 21 Sep 2026 17:57:56 +0200 Subject: [PATCH 022/102] s390/percpu: Add sanity check to GEN_MVIY macro Emit an error if the GEN_MVIY macro fails to parse the passed register string. This will make it more obvious if something similar which was fixed with [1] happens again. [1] commit 101782f8945a ("s390/percpu: Fix MVIY_PERCPU() with older binutils") Acked-by: Alexander Gordeev Tested-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 31012abe8292..9888b38dedab 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -100,12 +100,17 @@ */ #define MVIY_PERCPU(disp, dispalt, reg) \ - ".macro GEN_MVIY disp reg\n" \ + ".macro GEN_MVIY disp, reg\n" \ + ".set .Lreg,255\n" \ ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ " .ifc \\reg,%%r\\rs\n" \ - " mviy \\disp(%%r0),\\rs\n" \ + " .set .Lreg,\\rs\n" \ " .endif\n" \ ".endr\n" \ + ".if .Lreg == 255\n" \ + " .error \"Illegal register number\"\n" \ + ".endif\n" \ + "mviy \\disp(%%r0),.Lreg\n" \ ".endm\n" \ ALTERNATIVE("GEN_MVIY " disp ", " reg "\n", \ "GEN_MVIY " dispalt ", " reg "\n", \ From ee30fcd726fb3f1fed370a0be446a8b44f8a09b7 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 21 Sep 2026 17:57:57 +0200 Subject: [PATCH 023/102] s390/lowcore: Remove _AC() from LOWCORE_ALT_ADDRESS Remove the _AC() conversion from LOWCORE_ALT_ADDRESS. This changes the type of LOWCORE_ALT_ADDRESS from unsigned long to int in C code, however looking at the current users this should not cause any changes. This allows to use __stringify(LOWCORE_ALT_ADDRESS) for inline assemblies, which otherwise wouldn't work because of the UL suffix. Acked-by: Alexander Gordeev Tested-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/include/asm/lowcore.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 5cef215d30e7..f0364a9f8a19 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -21,7 +21,7 @@ #define LC_ORDER 1 #define LC_PAGES 2 -#define LOWCORE_ALT_ADDRESS _AC(0x70000, UL) +#define LOWCORE_ALT_ADDRESS 0x70000 #ifndef __ASSEMBLER__ From d6fe39764bd690949ae640c49164cca07b99d7c9 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 21 Sep 2026 17:57:58 +0200 Subject: [PATCH 024/102] s390/percpu: Let MVIY_PERCPU() calculate alternative displacement Let MVIY_PERCPU() calculate the alternative lowcore displacement instead of passing the same value(s) again and again in all percpu inline assemblies. This shortens the inline assemblies and makes them a bit more readable. Acked-by: Alexander Gordeev Tested-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 68 +++++++++++++++------------------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 9888b38dedab..3bc448c4c2dd 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -99,7 +99,9 @@ * MVIY_ALT(...) <- end of percpu code section */ -#define MVIY_PERCPU(disp, dispalt, reg) \ +#define LC_ALT_ADDR __stringify(LOWCORE_ALT_ADDRESS) + +#define MVIY_PERCPU(disp, reg) \ ".macro GEN_MVIY disp, reg\n" \ ".set .Lreg,255\n" \ ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ @@ -112,19 +114,19 @@ ".endif\n" \ "mviy \\disp(%%r0),.Lreg\n" \ ".endm\n" \ - ALTERNATIVE("GEN_MVIY " disp ", " reg "\n", \ - "GEN_MVIY " dispalt ", " reg "\n", \ + ALTERNATIVE("GEN_MVIY " disp " , " reg "\n", \ + "GEN_MVIY " disp "+" LC_ALT_ADDR " , " reg "\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) \ ".purgem GEN_MVIY\n" -#define MVIY_ALT(disp, dispalt) \ - ALTERNATIVE(" mviy " disp "(%%r0),0\n", \ - " mviy " dispalt "(%%r0),0\n", \ +#define MVIY_ALT(disp) \ + ALTERNATIVE(" mviy " disp "(%%r0),0\n", \ + " mviy " disp "+" LC_ALT_ADDR "(%%r0),0\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) -#define AG_ALT(disp, dispalt, reg) \ - ALTERNATIVE(" ag " reg ", " disp "(%%r0)\n", \ - " ag " reg ", " dispalt "(%%r0)\n", \ +#define AG_ALT(disp, reg) \ + ALTERNATIVE(" ag " reg ", " disp "(%%r0)\n", \ + " ag " reg ", " disp "+" LC_ALT_ADDR "(%%r0)\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) #ifndef MARCH_HAS_Z196_FEATURES @@ -153,33 +155,29 @@ do { \ if (__builtin_constant_p(val__) && \ ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ asm volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ + AG_ALT("%[disppcpo]", "%[ptr__]") \ op2 " 0(%[ptr__]),%[val__]\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + MVIY_ALT("%[disppcpr]") \ : [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "i" ((szcast)val__), \ [disppcpr] "i" (lc_pcpr), \ [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } else { \ asm volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + MVIY_PERCPU("%[disppcpr]", "%[ptr__]") \ + AG_ALT("%[disppcpo]", "%[ptr__]") \ op1 " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + MVIY_ALT("%[disppcpr]") \ : [old__] "=&d" (old__), \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ [disppcpr] "i" (lc_pcpr), \ [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } \ @@ -199,18 +197,16 @@ do { \ lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ + AG_ALT("%[disppcpo]","%[ptr__]") \ op " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + MVIY_ALT("%[disppcpr]") \ : [old__] "=&d" (old__), \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ [disppcpr] "i" (lc_pcpr), \ [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ old__ + val__; \ @@ -230,18 +226,16 @@ do { \ lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ + AG_ALT("%[disppcpo]","%[ptr__]") \ op " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + MVIY_ALT("%[disppcpr]") \ : [old__] "=&d" (old__), \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ [disppcpr] "i" (lc_pcpr), \ [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } while (0) @@ -263,16 +257,14 @@ do { \ lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ + AG_ALT("%[disppcpo]","%[ptr__]") \ op " %[res__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + MVIY_ALT("%[disppcpr]") \ : [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [disppcpr] "i" (lc_pcpr), \ [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ "m" (*ptr__), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ @@ -294,17 +286,15 @@ do { \ lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ - AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ + AG_ALT("%[disppcpo]","%[ptr__]") \ op " %[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + MVIY_ALT("%[disppcpr]") \ : [ptr__] "+&a" (ptr__), "=m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ [disppcpr] "i" (lc_pcpr), \ [disppcpo] "i" (lc_pcpo), \ - [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ - [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } while (0) From 301e3a0776511ed60cca1f7d4b14b6b44827c6f4 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 21 Sep 2026 17:57:59 +0200 Subject: [PATCH 025/102] s390/percpu/lowcore: Add and use LC_PERCPU lowcore offset defines Define LC_PERCPU_OFFSET and LC_PERCPU_REGISTER offset defines and use them in percpu code. This shortens the percpu inline assemblies and makes them a bit more readable. Note that because of header dependencies it is not possible to use asm-offsets. However to avoid adding more dependencies to asm-offsets it seems to be preferable to use the current approach in any way. Acked-by: Alexander Gordeev Tested-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/include/asm/entry-percpu.h | 2 +- arch/s390/include/asm/lowcore.h | 3 ++ arch/s390/include/asm/percpu.h | 47 ++++++++++------------------ 3 files changed, 20 insertions(+), 32 deletions(-) diff --git a/arch/s390/include/asm/entry-percpu.h b/arch/s390/include/asm/entry-percpu.h index 4ef47265cfce..89c9e44f6f83 100644 --- a/arch/s390/include/asm/entry-percpu.h +++ b/arch/s390/include/asm/entry-percpu.h @@ -49,7 +49,7 @@ static __always_inline bool percpu_code_check(struct pt_regs *regs) } if ((insn & 0xff0f) != 0xe300) return true; - disp = offsetof(struct lowcore, percpu_offset); + disp = LC_PERCPU_OFFSET; if (machine_has_relocated_lowcore()) disp += LOWCORE_ALT_ADDRESS; insn = (disp & 0xff000) >> 4 | (disp & 0x00fff) << 16 | 0x8; diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index f0364a9f8a19..f4517bbe405b 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -222,6 +222,9 @@ struct lowcore { __u8 pad_0x1900[0x2000-0x1900]; /* 0x1900 */ } __packed __aligned(8192); +#define LC_PERCPU_OFFSET offsetof(struct lowcore, percpu_offset) +#define LC_PERCPU_REGISTER offsetof(struct lowcore, percpu_register) + static __always_inline struct lowcore *get_lowcore(void) { struct lowcore *lc; diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 3bc448c4c2dd..770fe4be65dc 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -14,17 +14,16 @@ #define arch_raw_cpu_ptr(_ptr) \ ({ \ - unsigned long lc_percpu, tcp_ptr__; \ + unsigned long tcp_ptr__; \ \ tcp_ptr__ = (__force unsigned long)(_ptr); \ - lc_percpu = offsetof(struct lowcore, percpu_offset); \ asm_inline volatile( \ ALTERNATIVE("ag %[__ptr__],%[offzero](%%r0)\n", \ "ag %[__ptr__],%[offalt](%%r0)\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) \ : [__ptr__] "+d" (tcp_ptr__) \ - : [offzero] "i" (lc_percpu), \ - [offalt] "i" (lc_percpu + LOWCORE_ALT_ADDRESS), \ + : [offzero] "i" (LC_PERCPU_OFFSET), \ + [offalt] "i" (LC_PERCPU_OFFSET + LOWCORE_ALT_ADDRESS), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ (TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__; \ @@ -144,13 +143,10 @@ #define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ do { \ - unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ \ - lc_pcpr = offsetof(struct lowcore, percpu_register); \ - lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ if (__builtin_constant_p(val__) && \ ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ @@ -162,8 +158,8 @@ do { \ : [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "i" ((szcast)val__), \ - [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ + [disppcpr] "i" (LC_PERCPU_REGISTER), \ + [disppcpo] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } else { \ @@ -176,8 +172,8 @@ do { \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ - [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ + [disppcpr] "i" (LC_PERCPU_REGISTER), \ + [disppcpo] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } \ @@ -188,13 +184,10 @@ do { \ #define arch_this_cpu_add_return(pcp, val, op) \ ({ \ - unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ \ - lc_pcpr = offsetof(struct lowcore, percpu_register); \ - lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ @@ -205,8 +198,8 @@ do { \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ - [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ + [disppcpr] "i" (LC_PERCPU_REGISTER), \ + [disppcpo] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ old__ + val__; \ @@ -217,13 +210,10 @@ do { \ #define arch_this_cpu_to_op(pcp, val, op) \ do { \ - unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ \ - lc_pcpr = offsetof(struct lowcore, percpu_register); \ - lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ @@ -234,8 +224,8 @@ do { \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ - [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ + [disppcpr] "i" (LC_PERCPU_REGISTER), \ + [disppcpo] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } while (0) @@ -249,12 +239,10 @@ do { \ #define arch_this_cpu_read(pcp, op) \ ({ \ - unsigned long lc_pcpr, lc_pcpo, res__; \ typedef typeof(pcp) pcp_op_T__; \ + unsigned long res__; \ pcp_op_T__ *ptr__; \ \ - lc_pcpr = offsetof(struct lowcore, percpu_register); \ - lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ @@ -263,8 +251,8 @@ do { \ MVIY_ALT("%[disppcpr]") \ : [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ + : [disppcpr] "i" (LC_PERCPU_REGISTER), \ + [disppcpo] "i" (LC_PERCPU_OFFSET), \ "m" (*ptr__), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ @@ -278,12 +266,9 @@ do { \ #define arch_this_cpu_write(pcp, val, op) \ do { \ - unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ *ptr__, val__ = (val); \ \ - lc_pcpr = offsetof(struct lowcore, percpu_register); \ - lc_pcpo = offsetof(struct lowcore, percpu_offset); \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ @@ -293,8 +278,8 @@ do { \ : [ptr__] "+&a" (ptr__), "=m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ - [disppcpr] "i" (lc_pcpr), \ - [disppcpo] "i" (lc_pcpo), \ + [disppcpr] "i" (LC_PERCPU_REGISTER), \ + [disppcpo] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } while (0) From 02a8ff0f894f6960d5d5fb47169140a1ae5c024d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 21 Sep 2026 17:58:00 +0200 Subject: [PATCH 026/102] s390/percpu: Rename inline assembly symbolic names Rename percpu inline assembly symbolic names so they match (more) what is specified with input operands. A trivial change which is supposed to make the code a bit more readable. Acked-by: Alexander Gordeev Tested-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 60 +++++++++++++++++----------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 770fe4be65dc..0df47ec2e6c7 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -151,29 +151,29 @@ do { \ if (__builtin_constant_p(val__) && \ ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ asm volatile( \ - MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ - AG_ALT("%[disppcpo]", "%[ptr__]") \ + MVIY_PERCPU("%[lcreg]","%[ptr__]") \ + AG_ALT("%[lcoff]", "%[ptr__]") \ op2 " 0(%[ptr__]),%[val__]\n" \ - MVIY_ALT("%[disppcpr]") \ + MVIY_ALT("%[lcreg]") \ : [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "i" ((szcast)val__), \ - [disppcpr] "i" (LC_PERCPU_REGISTER), \ - [disppcpo] "i" (LC_PERCPU_OFFSET), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } else { \ asm volatile( \ - MVIY_PERCPU("%[disppcpr]", "%[ptr__]") \ - AG_ALT("%[disppcpo]", "%[ptr__]") \ + MVIY_PERCPU("%[lcreg]", "%[ptr__]") \ + AG_ALT("%[lcoff]", "%[ptr__]") \ op1 " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]") \ + MVIY_ALT("%[lcreg]") \ : [old__] "=&d" (old__), \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ - [disppcpr] "i" (LC_PERCPU_REGISTER), \ - [disppcpo] "i" (LC_PERCPU_OFFSET), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } \ @@ -190,16 +190,16 @@ do { \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ - AG_ALT("%[disppcpo]","%[ptr__]") \ + MVIY_PERCPU("%[lcreg]","%[ptr__]") \ + AG_ALT("%[lcoff]","%[ptr__]") \ op " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]") \ + MVIY_ALT("%[lcreg]") \ : [old__] "=&d" (old__), \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ - [disppcpr] "i" (LC_PERCPU_REGISTER), \ - [disppcpo] "i" (LC_PERCPU_OFFSET), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ old__ + val__; \ @@ -216,16 +216,16 @@ do { \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ - AG_ALT("%[disppcpo]","%[ptr__]") \ + MVIY_PERCPU("%[lcreg]","%[ptr__]") \ + AG_ALT("%[lcoff]","%[ptr__]") \ op " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]") \ + MVIY_ALT("%[lcreg]") \ : [old__] "=&d" (old__), \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ - [disppcpr] "i" (LC_PERCPU_REGISTER), \ - [disppcpo] "i" (LC_PERCPU_OFFSET), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } while (0) @@ -245,14 +245,14 @@ do { \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ - AG_ALT("%[disppcpo]","%[ptr__]") \ + MVIY_PERCPU("%[lcreg]","%[ptr__]") \ + AG_ALT("%[lcoff]","%[ptr__]") \ op " %[res__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]") \ + MVIY_ALT("%[lcreg]") \ : [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [disppcpr] "i" (LC_PERCPU_REGISTER), \ - [disppcpo] "i" (LC_PERCPU_OFFSET), \ + : [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (*ptr__), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ @@ -271,15 +271,15 @@ do { \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[disppcpr]","%[ptr__]") \ - AG_ALT("%[disppcpo]","%[ptr__]") \ + MVIY_PERCPU("%[lcreg]","%[ptr__]") \ + AG_ALT("%[lcoff]","%[ptr__]") \ op " %[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[disppcpr]") \ + MVIY_ALT("%[lcreg]") \ : [ptr__] "+&a" (ptr__), "=m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ - [disppcpr] "i" (LC_PERCPU_REGISTER), \ - [disppcpo] "i" (LC_PERCPU_OFFSET), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } while (0) From da9af84e66c3ace4f1a95421c1152f9df4e6abc5 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 21 Sep 2026 17:58:01 +0200 Subject: [PATCH 027/102] s390/percpu: Use __PCPU_BEGIN() and __PCPU_END() for inline assemblies Replace the use of MVIY_PERCPU(), AG_ALT(), and MVIY_ALT() within all percpu inline assemblies with __PCPU_BEGIN() and __PCPU_END(), since the begin and end of each inline assembly is the same. Also use consistent names: e.g. __PCPU_MVIY() and __PCPU_AG() instead of the previous MVIY_PERCPU() and AG_ALT(). Furthermore extract DEFINE_GR_NUM and UNDEF_GR_NUM which are used to convert register strings to register numbers. All of this should make the code more readable. No functional change. Acked-by: Alexander Gordeev Tested-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 83 +++++++++++++++++----------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 0df47ec2e6c7..78c3aaa138f7 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -92,42 +92,49 @@ * * Inline assemblies making use of this typically have a code sequence like: * - * MVIY_PERCPU(...) <- start of percpu code section - * AG_ALT(...) <- add percpu offset; must be the second instruction - * atomic_op <- atomic op - * MVIY_ALT(...) <- end of percpu code section + * __PCPU_BEGIN(...) <- start of percpu code section + * atomic_op <- atomic op + * __PCPU_END(...) <- end of percpu code section */ #define LC_ALT_ADDR __stringify(LOWCORE_ALT_ADDRESS) -#define MVIY_PERCPU(disp, reg) \ - ".macro GEN_MVIY disp, reg\n" \ - ".set .Lreg,255\n" \ - ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ - " .ifc \\reg,%%r\\rs\n" \ - " .set .Lreg,\\rs\n" \ +#define DEFINE_GR_NUM \ + ".macro _GR_NUM opd, gr\n" \ + ".set \\opd,255\n" \ + ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ + " .ifc \\gr,%%r\\rs\n" \ + " \\opd = \\rs\n" \ " .endif\n" \ ".endr\n" \ - ".if .Lreg == 255\n" \ + ".if \\opd == 255\n" \ " .error \"Illegal register number\"\n" \ ".endif\n" \ - "mviy \\disp(%%r0),.Lreg\n" \ - ".endm\n" \ - ALTERNATIVE("GEN_MVIY " disp " , " reg "\n", \ - "GEN_MVIY " disp "+" LC_ALT_ADDR " , " reg "\n", \ - ALT_FEATURE(MFEATURE_LOWCORE)) \ - ".purgem GEN_MVIY\n" + ".endm\n" -#define MVIY_ALT(disp) \ - ALTERNATIVE(" mviy " disp "(%%r0),0\n", \ - " mviy " disp "+" LC_ALT_ADDR "(%%r0),0\n", \ +#define UNDEF_GR_NUM \ + ".purgem _GR_NUM\n" + +#define __PCPU_MVIY(lcreg, imm) \ + ALTERNATIVE(" mviy " lcreg "(%%r0)," imm "\n", \ + " mviy " lcreg "+" LC_ALT_ADDR "(%%r0)," imm "\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) -#define AG_ALT(disp, reg) \ - ALTERNATIVE(" ag " reg ", " disp "(%%r0)\n", \ - " ag " reg ", " disp "+" LC_ALT_ADDR "(%%r0)\n", \ +#define __PCPU_AG(reg, lcoff) \ + ALTERNATIVE(" ag " reg ", " lcoff "(%%r0)\n", \ + " ag " reg ", " lcoff "+" LC_ALT_ADDR "(%%r0)\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) +#define __PCPU_BEGIN(lcreg, lcoff, reg) \ + DEFINE_GR_NUM \ + "_GR_NUM .Lreg, " reg "\n" \ + UNDEF_GR_NUM \ + __PCPU_MVIY(lcreg, ".Lreg") \ + __PCPU_AG(reg, lcoff) + +#define __PCPU_END(lcreg) \ + __PCPU_MVIY(lcreg, "0") + #ifndef MARCH_HAS_Z196_FEATURES #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) @@ -151,10 +158,9 @@ do { \ if (__builtin_constant_p(val__) && \ ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ asm volatile( \ - MVIY_PERCPU("%[lcreg]","%[ptr__]") \ - AG_ALT("%[lcoff]", "%[ptr__]") \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ op2 " 0(%[ptr__]),%[val__]\n" \ - MVIY_ALT("%[lcreg]") \ + __PCPU_END("%[lcreg]") \ : [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "i" ((szcast)val__), \ @@ -164,10 +170,9 @@ do { \ : "cc"); \ } else { \ asm volatile( \ - MVIY_PERCPU("%[lcreg]", "%[ptr__]") \ - AG_ALT("%[lcoff]", "%[ptr__]") \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ op1 " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[lcreg]") \ + __PCPU_END("%[lcreg]") \ : [old__] "=&d" (old__), \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ @@ -190,10 +195,9 @@ do { \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[lcreg]","%[ptr__]") \ - AG_ALT("%[lcoff]","%[ptr__]") \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ op " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[lcreg]") \ + __PCPU_END("%[lcreg]") \ : [old__] "=&d" (old__), \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ @@ -216,10 +220,9 @@ do { \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[lcreg]","%[ptr__]") \ - AG_ALT("%[lcoff]","%[ptr__]") \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ op " %[old__],%[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[lcreg]") \ + __PCPU_END("%[lcreg]") \ : [old__] "=&d" (old__), \ [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ @@ -245,10 +248,9 @@ do { \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[lcreg]","%[ptr__]") \ - AG_ALT("%[lcoff]","%[ptr__]") \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ op " %[res__],0(%[ptr__])\n" \ - MVIY_ALT("%[lcreg]") \ + __PCPU_END("%[lcreg]") \ : [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [lcreg] "i" (LC_PERCPU_REGISTER), \ @@ -271,10 +273,9 @@ do { \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - MVIY_PERCPU("%[lcreg]","%[ptr__]") \ - AG_ALT("%[lcoff]","%[ptr__]") \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ op " %[val__],0(%[ptr__])\n" \ - MVIY_ALT("%[lcreg]") \ + __PCPU_END("%[lcreg]") \ : [ptr__] "+&a" (ptr__), "=m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ : [val__] "d" (val__), \ From e8b55c9982df6dc5f28c427e4debabb4da3a5e64 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 21 Sep 2026 17:58:02 +0200 Subject: [PATCH 028/102] s390/percpu: Use percpu code section for this_cpu_cmpxchg128() Convert this_cpu_cmpxchg128() to make use of percpu code section functionality. This is just for the sake of completeness, since currently there is no in-kernel user of this_cpu_cmpxchg128(). Acked-by: Alexander Gordeev Tested-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 78c3aaa138f7..0eae65937c0d 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -312,15 +312,23 @@ do { \ #define this_cpu_cmpxchg128(pcp, oval, nval) \ ({ \ typedef typeof(pcp) pcp_op_T__; \ - u128 old__, new__, ret__; \ + u128 old__ = (oval), new__ = (nval); \ pcp_op_T__ *ptr__; \ - old__ = oval; \ - new__ = nval; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - ret__ = cmpxchg128((void *)ptr__, old__, new__); \ - preempt_enable_notrace(); \ - ret__; \ + \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ + " cdsg %[old__],%[new__],0(%[ptr__])\n" \ + __PCPU_END("%[lcreg]") \ + : [old__] "+&d" (old__), [ptr__] "+&a" (ptr__), \ + "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [new__] "d" (new__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "memory", "cc"); \ + old__; \ }) #define arch_this_cpu_xchg(pcp, nval) \ From d83b5cf355ca762ef19aa3a424af44e3e275ff52 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 21 Sep 2026 17:58:03 +0200 Subject: [PATCH 029/102] s390/percpu: Use percpu code section for this_cpu_xchg() Convert this_cpu_xchg() to make use of percpu code section functionality in order to get rid of preempt_disable() and preempt_enable() calls. Consider only four and eight byte usages, since there are no compare and swap instructions for one and two bytes available. For one and two byte use the previous simple implementation. Acked-by: Alexander Gordeev Tested-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 0eae65937c0d..237bf8c80843 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -331,7 +331,7 @@ do { \ old__; \ }) -#define arch_this_cpu_xchg(pcp, nval) \ +#define arch_this_cpu_xchg_simple(pcp, nval) \ ({ \ typeof(pcp) *ptr__; \ typeof(pcp) ret__; \ @@ -342,10 +342,34 @@ do { \ ret__; \ }) -#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval) -#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval) -#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval) -#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval) +#define arch_this_cpu_xchg(pcp, nval, ldop, csop) \ +({ \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ old__, new__ = (nval); \ + pcp_op_T__ *ptr__; \ + \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ + " " ldop " %[old__],0(%[ptr__])\n" \ + "0: " csop " %[old__],%[new__],0(%[ptr__])\n"\ + " jnz 0b\n" \ + __PCPU_END("%[lcreg]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [new__] "d" (new__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "memory", "cc"); \ + old__; \ +}) + +#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg_simple(pcp, nval) +#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg_simple(pcp, nval) +#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval, "l", "cs") +#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval, "lg", "csg") #include From 45b0097302d87a42189abe0b33d69fe18f6c7cb0 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 21 Sep 2026 17:58:04 +0200 Subject: [PATCH 030/102] s390/percpu: Use percpu code section for this_cpu_cmpxchg() Convert this_cpu_cmpxchg() to make use of percpu code section functionality in order to get rid of preempt_disable() and preempt_enable() calls. Consider only four and eight byte usages, since there are no compare and swap instructions for one and two bytes available. For one and two bytes use the previous simple implementation. Acked-by: Alexander Gordeev Tested-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 237bf8c80843..46368131b932 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -290,7 +290,7 @@ do { \ #define this_cpu_write_4(pcp, val) arch_this_cpu_write(pcp, val, "st") #define this_cpu_write_8(pcp, val) arch_this_cpu_write(pcp, val, "stg") -#define arch_this_cpu_cmpxchg(pcp, oval, nval) \ +#define arch_this_cpu_cmpxchg_simple(pcp, oval, nval) \ ({ \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ ret__; \ @@ -302,10 +302,32 @@ do { \ ret__; \ }) -#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) -#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) -#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) -#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval) +#define arch_this_cpu_cmpxchg(pcp, oval, nval, op) \ +({ \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ old__ = (oval), new__ = (nval); \ + pcp_op_T__ *ptr__; \ + \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ + op " %[old__],%[new__],0(%[ptr__])\n" \ + __PCPU_END("%[lcreg]") \ + : [old__] "+&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [new__] "d" (new__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "memory", "cc"); \ + old__; \ +}) + +#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg_simple(pcp, oval, nval) +#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg_simple(pcp, oval, nval) +#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval, "cs") +#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval, "csg") #define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n) From 7849fb83e4cf814710003e4c83e161f97cf7ab1c Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 21 Sep 2026 17:58:05 +0200 Subject: [PATCH 031/102] s390/percpu: Rework to simplify percpu_entry() and percpu_exit() The percpu code section functionality uses a rather complex method to figure out if the register, which contains the address of the current cpu's percpu variable, needs to be adjusted. If an interrupt happens within a percpu code section (indicated by a lowcore field), the instruction at the interrupted location is checked. If it is not a specific AG instruction, the register needs to be updated. This mechanism needs to take kprobes into account, and enforces a specific instruction ordering. Mark Rutland provided a different solution for arm64 [1] which comes without such limitations, but requires to use one more instruction, and two more registers. Given that this simplifies percpu_entry() and percpu_exit() it seems to be worth to go that route. Change s390 to implement a similar approach. This requires to encode three register numbers into the "percpu_register" field, which is used to indicate if a percpu code section is executed. The used mviy instruction can write only one byte, which allows to encode only two register numbers. Use a register pair for the inline assemblies, and only encode the even register number of the register pair to work around this. The generated code changes like this for e.g. a simple this_cpu_inc(): Old: c0 20 00 00 00 00 larl %r2,c6 <-- load address of percpu var b9 04 00 32 lgr %r3,%r2 <-- pointless copy of address eb 03 03 c0 00 52 mviy 960,3 <-- start of percpu code section - gpr 3 contains percpu var address e3 30 03 b8 00 08 ag %r3,952 <-- add percpu offset eb 01 30 00 00 7a agsi 0(%r3),1 <-- atomic inc eb 00 03 c0 00 52 mviy 960,0 <-- end of percpu code section New: c0 10 00 00 00 00 larl %r1,c6 <-- load address of percpu var eb 21 03 c0 00 52 mviy 960,33 <-- start of percpu code section 33 == 0x21: - gpr 1 contains percpu var address - gpr 2 used for percpu offset - gpr 2+1 == 3 used for current cpu's percpu var address e3 20 03 b8 00 04 lg %r2,952 <-- load percpu offset 41 32 10 00 la %r3,0(%r2,%r1) <-- generate current cpu's percpu var address eb 01 30 00 00 7a agsi 0(%r3),1 <-- atomic inc eb 00 03 c0 00 52 mviy 960,0 <-- end of percpu code section In the above "new" code example 33 (0x21) is used as indicator value to mark that a percpu code section is executed. This value implies that registers 2 and 3 will (only) hold the percpu offset and the current cpu's percpu var address. Those registers will be updated by percpu_exit() if the process was migrated to a different cpu to contain the percpu offset and percpu var address of the new cpu. Note that because of the pointless lgr instruction in the "old" code example the two code sequences have identical size, and it looks like only one more register is used. However this is because of suboptimal gcc code generation. [1] https://lore.kernel.org/all/20260908151741.394589-1-mark.rutland@arm.com/ Acked-by: Alexander Gordeev Tested-by: Mete Durlu Signed-off-by: Heiko Carstens --- arch/s390/include/asm/entry-percpu.h | 71 +++-------- arch/s390/include/asm/percpu.h | 177 +++++++++++++++++---------- arch/s390/kernel/irq.c | 10 +- arch/s390/kernel/nmi.c | 4 +- arch/s390/kernel/traps.c | 4 +- 5 files changed, 138 insertions(+), 128 deletions(-) diff --git a/arch/s390/include/asm/entry-percpu.h b/arch/s390/include/asm/entry-percpu.h index 89c9e44f6f83..f6f79ec4bee3 100644 --- a/arch/s390/include/asm/entry-percpu.h +++ b/arch/s390/include/asm/entry-percpu.h @@ -2,79 +2,42 @@ #ifndef ARCH_S390_ENTRY_PERCPU_H #define ARCH_S390_ENTRY_PERCPU_H -#include +#include #include #include #include -#include static __always_inline void percpu_entry(struct pt_regs *regs) { struct lowcore *lc = get_lowcore(); - if (user_mode(regs)) - return; regs->cpu = lc->cpu_nr; regs->percpu_register = lc->percpu_register; lc->percpu_register = 0; } -static __always_inline bool percpu_code_check(struct pt_regs *regs) -{ - unsigned int insn, disp; - struct kprobe *p; - - if (likely(user_mode(regs) || !regs->percpu_register)) - return false; - /* - * Within a percpu code section - check if the percpu base register - * needs to be updated. This is the case if the PSW does not point to - * the ADD instruction within the section. - * - AG %rx,percpu_offset_in_lowcore(%r0,%r0) - * which adds the percpu offset to the percpu base register. - */ - lockdep_assert_preemption_disabled(); -again: - insn = READ_ONCE(*(u16 *)psw_bits(regs->psw).ia); - if (unlikely(insn == BREAKPOINT_INSTRUCTION)) { - p = get_kprobe((void *)psw_bits(regs->psw).ia); - /* - * If the kprobe is concurrently removed on a different CPU - * it might not be found anymore. However text must have - * been restored - try again. - */ - if (!p) - goto again; - insn = p->opcode; - } - if ((insn & 0xff0f) != 0xe300) - return true; - disp = LC_PERCPU_OFFSET; - if (machine_has_relocated_lowcore()) - disp += LOWCORE_ALT_ADDRESS; - insn = (disp & 0xff000) >> 4 | (disp & 0x00fff) << 16 | 0x8; - if (*(u32 *)(psw_bits(regs->psw).ia + 2) != insn) - return true; - return false; -} - -static __always_inline void percpu_exit(struct pt_regs *regs, bool needs_fixup) +static __always_inline void percpu_exit(struct pt_regs *regs) { + unsigned char regval, regpcp, regoff, regptr; struct lowcore *lc = get_lowcore(); - unsigned char reg; - if (user_mode(regs)) + if (!regs->percpu_register) return; - reg = regs->percpu_register; - lc->percpu_register = reg; - if (likely(!needs_fixup)) - return; - /* Check if process has been migrated to a different CPU. */ + regval = regs->percpu_register; + lc->percpu_register = regval; + /* Migrated to a different CPU? */ if (regs->cpu == lc->cpu_nr) return; - /* Fixup percpu base register */ - regs->gprs[reg] -= __per_cpu_offset[regs->cpu]; - regs->gprs[reg] += lc->percpu_offset; + regpcp = FIELD_GET(PCPU_REG_PCP, regval); + regoff = FIELD_GET(PCPU_REG_OFF, regval); + regptr = regoff + 1; + /* + * Update register 'regoff' with the current CPU's percpu offset, + * and recalculate and update current CPU's percpu variable address + * contained in register 'regptr'. + */ + regs->gprs[regoff] = lc->percpu_offset; + regs->gprs[regptr] = regs->gprs[regpcp] + regs->gprs[regoff]; } #endif diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 46368131b932..c3995bd8d1de 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -67,28 +67,32 @@ * this. The idea is that this_cpu operations based on atomic instructions are * guarded with mviy instructions: * - * - The first mviy instruction writes the register number, which contains the - * percpu address variable to lowcore. This also indicates that a percpu - * code section is executed. + * - The first mviy instruction writes the register number of the percpu address + * variable and the even register number of a register pair (which encodes two + * registers: the even register for the percpu offset and the odd register for + * the percpu pointer) to lowcore. This also indicates that a percpu code + * section is executed. * - * - The first mviy instruction following the mviy instruction must be the ag - * instruction which adds the percpu offset to the percpu address register. + * - The mviy instruction is followed by the lg instruction which loads the + * percpu offset into the even register of the pair and the la instruction + * which adds the percpu offset and the percpu address into the odd register + * of the pair (the percpu pointer register). * * - Afterwards the atomic percpu operation follows. * * - Then a second mviy instruction writes a zero to lowcore, which indicates * the end of the percpu code section. * - * - In case of an interrupt/exception/nmi the register number which was - * written to lowcore is copied to the exception frame (pt_regs), and a zero - * is written to lowcore. + * - In case of an interrupt/exception/nmi the encoded register numbers which + * were written to lowcore are copied to the exception frame (pt_regs), and a + * zero is written to lowcore. * * - On return to the previous context it is checked if a percpu code section - * was executed (saved register number not zero), and if the process was - * migrated to a different cpu. If the percpu offset was already added to - * the percpu address register (instruction address does _not_ point to the - * ag instruction) the content of the percpu address register is adjusted so - * it points to percpu variable of the new cpu. + * was executed (saved register value not zero), and if the process was + * migrated to a different cpu. The content of the percpu offset register + * (even register of the pair) is reloaded with the current cpu's percpu + * offset and the percpu pointer register (odd register of the pair) is + * recalculated so it points to the percpu variable of the new cpu. * * Inline assemblies making use of this typically have a code sequence like: * @@ -115,22 +119,53 @@ #define UNDEF_GR_NUM \ ".purgem _GR_NUM\n" +#define PCPU_REG_PCP_SHIFT 0 +#define PCPU_REG_PCP GENMASK(3, 0) +#define PCPU_REG_OFF_SHIFT 4 +#define PCPU_REG_OFF GENMASK(7, 4) + #define __PCPU_MVIY(lcreg, imm) \ ALTERNATIVE(" mviy " lcreg "(%%r0)," imm "\n", \ " mviy " lcreg "+" LC_ALT_ADDR "(%%r0)," imm "\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) -#define __PCPU_AG(reg, lcoff) \ - ALTERNATIVE(" ag " reg ", " lcoff "(%%r0)\n", \ - " ag " reg ", " lcoff "+" LC_ALT_ADDR "(%%r0)\n", \ +#define __PCPU_LG(regoff, lcoff) \ + ALTERNATIVE(" lg " regoff ", " lcoff "(%%r0)\n", \ + " lg " regoff ", " lcoff "+" LC_ALT_ADDR "(%%r0)\n", \ ALT_FEATURE(MFEATURE_LOWCORE)) -#define __PCPU_BEGIN(lcreg, lcoff, reg) \ +#define __PCPU_LA(regptr, regoff, regpcp) \ + " la " regptr ",0(" regoff "," regpcp ")\n" + +#define __PCPU_CALC_REGVAL(regpcp, regoff) \ + "(" regpcp " << " __stringify(PCPU_REG_PCP_SHIFT) ") |" \ + "(" regoff " << " __stringify(PCPU_REG_OFF_SHIFT) ")" + +#define __PCPU_CHECK_REGS(regpcp, regoff, regptr) \ + ".if " regoff " & 1\n" \ + " .error \"Percpu offset register must be even\"\n" \ + ".endif\n" \ + ".if (" regpcp " == 0) || (" regoff " == 0)\n" \ + " .error \"Percpu address and offset register must be non-zero\"\n"\ + ".endif\n" \ + ".if " regptr " != " regoff " + 1\n" \ + " .error \"Percpu pointer and offset register must be a pair\"\n" \ + ".endif\n" \ + ".if (" regpcp " == " regoff ") || (" regpcp " == " regptr ")\n" \ + " .error \"Percpu registers must be distinct\"\n" \ + ".endif\n" + +#define __PCPU_BEGIN(lcreg, lcoff, regpcp, regoff, regptr) \ DEFINE_GR_NUM \ - "_GR_NUM .Lreg, " reg "\n" \ + "_GR_NUM .Lregpcp, " regpcp "\n" \ + "_GR_NUM .Lregoff, " regoff "\n" \ + "_GR_NUM .Lregptr, " regptr "\n" \ UNDEF_GR_NUM \ - __PCPU_MVIY(lcreg, ".Lreg") \ - __PCPU_AG(reg, lcoff) + __PCPU_CHECK_REGS(".Lregpcp", ".Lregoff", ".Lregptr") \ + ".set .Lregval, " __PCPU_CALC_REGVAL(".Lregpcp", ".Lregoff") "\n" \ + __PCPU_MVIY(lcreg, ".Lregval") \ + __PCPU_LG(regoff, lcoff) \ + __PCPU_LA(regptr, regoff, regpcp) #define __PCPU_END(lcreg) \ __PCPU_MVIY(lcreg, "0") @@ -151,6 +186,7 @@ #define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ do { \ typedef typeof(pcp) pcp_op_T__; \ + union register_pair rp__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ \ @@ -158,25 +194,27 @@ do { \ if (__builtin_constant_p(val__) && \ ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ asm volatile( \ - __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ - op2 " 0(%[ptr__]),%[val__]\n" \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op2 " 0(%N[pair__]),%[val__]\n" \ __PCPU_END("%[lcreg]") \ - : [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + : [pair__] "=&a" (rp__.pair), "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "i" ((szcast)val__), \ + : [val__] "i" ((szcast)val__), [ptr__] "a" (ptr__), \ [lcreg] "i" (LC_PERCPU_REGISTER), \ [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } else { \ asm volatile( \ - __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ - op1 " %[old__],%[val__],0(%[ptr__])\n" \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op1 " %[old__],%[val__],0(%N[pair__])\n" \ __PCPU_END("%[lcreg]") \ - : [old__] "=&d" (old__), \ - [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "d" (val__), \ + : [val__] "d" (val__), [ptr__] "a" (ptr__), \ [lcreg] "i" (LC_PERCPU_REGISTER), \ [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ @@ -190,18 +228,20 @@ do { \ #define arch_this_cpu_add_return(pcp, val, op) \ ({ \ typedef typeof(pcp) pcp_op_T__; \ + union register_pair rp__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ - op " %[old__],%[val__],0(%[ptr__])\n" \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[old__],%[val__],0(%N[pair__])\n" \ __PCPU_END("%[lcreg]") \ - : [old__] "=&d" (old__), \ - [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "d" (val__), \ + : [val__] "d" (val__), [ptr__] "a" (ptr__), \ [lcreg] "i" (LC_PERCPU_REGISTER), \ [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ @@ -215,18 +255,20 @@ do { \ #define arch_this_cpu_to_op(pcp, val, op) \ do { \ typedef typeof(pcp) pcp_op_T__; \ + union register_pair rp__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ - op " %[old__],%[val__],0(%[ptr__])\n" \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[old__],%[val__],0(%N[pair__])\n" \ __PCPU_END("%[lcreg]") \ - : [old__] "=&d" (old__), \ - [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "d" (val__), \ + : [val__] "d" (val__), [ptr__] "a" (ptr__), \ [lcreg] "i" (LC_PERCPU_REGISTER), \ [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ @@ -243,17 +285,20 @@ do { \ #define arch_this_cpu_read(pcp, op) \ ({ \ typedef typeof(pcp) pcp_op_T__; \ + union register_pair rp__; \ unsigned long res__; \ pcp_op_T__ *ptr__; \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ - op " %[res__],0(%[ptr__])\n" \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[res__],0(%N[pair__])\n" \ __PCPU_END("%[lcreg]") \ - : [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \ + : [res__] "=&d" (res__), [pair__] "=&a" (rp__.pair), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [lcreg] "i" (LC_PERCPU_REGISTER), \ + : [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (*ptr__), \ "m" (((struct lowcore *)0)->percpu_offset) \ @@ -269,16 +314,18 @@ do { \ #define arch_this_cpu_write(pcp, val, op) \ do { \ typedef typeof(pcp) pcp_op_T__; \ + union register_pair rp__; \ pcp_op_T__ *ptr__, val__ = (val); \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ - op " %[val__],0(%[ptr__])\n" \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[val__],0(%N[pair__])\n" \ __PCPU_END("%[lcreg]") \ - : [ptr__] "+&a" (ptr__), "=m" (*ptr__), \ + : [pair__] "=&a" (rp__.pair), "=m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "d" (val__), \ + : [val__] "d" (val__), [ptr__] "a" (ptr__), \ [lcreg] "i" (LC_PERCPU_REGISTER), \ [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ @@ -306,17 +353,19 @@ do { \ ({ \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ old__ = (oval), new__ = (nval); \ + union register_pair rp__; \ pcp_op_T__ *ptr__; \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ - op " %[old__],%[new__],0(%[ptr__])\n" \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[old__],%[new__],0(%N[pair__])\n" \ __PCPU_END("%[lcreg]") \ - : [old__] "+&d" (old__), \ - [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [new__] "d" (new__), \ + : [new__] "d" (new__), [ptr__] "a" (ptr__), \ [lcreg] "i" (LC_PERCPU_REGISTER), \ [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ @@ -335,17 +384,19 @@ do { \ ({ \ typedef typeof(pcp) pcp_op_T__; \ u128 old__ = (oval), new__ = (nval); \ + union register_pair rp__; \ pcp_op_T__ *ptr__; \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ - " cdsg %[old__],%[new__],0(%[ptr__])\n" \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + " cdsg %[old__],%[new__],0(%N[pair__])\n" \ __PCPU_END("%[lcreg]") \ - : [old__] "+&d" (old__), [ptr__] "+&a" (ptr__), \ + : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \ "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [new__] "d" (new__), \ + : [new__] "d" (new__), [ptr__] "a" (ptr__), \ [lcreg] "i" (LC_PERCPU_REGISTER), \ [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ @@ -368,19 +419,21 @@ do { \ ({ \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ old__, new__ = (nval); \ + union register_pair rp__; \ pcp_op_T__ *ptr__; \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ - __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]") \ - " " ldop " %[old__],0(%[ptr__])\n" \ - "0: " csop " %[old__],%[new__],0(%[ptr__])\n"\ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + " " ldop " %[old__],0(%N[pair__])\n" \ + "0: " csop " %[old__],%[new__],0(%N[pair__])\n"\ " jnz 0b\n" \ __PCPU_END("%[lcreg]") \ - : [old__] "=&d" (old__), \ - [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [new__] "d" (new__), \ + : [new__] "d" (new__), [ptr__] "a" (ptr__), \ [lcreg] "i" (LC_PERCPU_REGISTER), \ [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index c923496aa7b4..8f21bea349a6 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -144,9 +144,9 @@ static int irq_pending(struct pt_regs *regs) void noinstr do_io_irq(struct pt_regs *regs) { - bool from_idle, percpu_needs_fixup; struct pt_regs *old_regs; irqentry_state_t state; + bool from_idle; percpu_entry(regs); state = irqentry_enter(regs); @@ -174,21 +174,20 @@ void noinstr do_io_irq(struct pt_regs *regs) do_irq_async(regs, IO_INTERRUPT); } while (machine_is_lpar() && irq_pending(regs)); - percpu_needs_fixup = percpu_code_check(regs); irq_exit_rcu(); set_irq_regs(old_regs); irqentry_exit(regs, state); if (from_idle) regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); - percpu_exit(regs, percpu_needs_fixup); + percpu_exit(regs); } void noinstr do_ext_irq(struct pt_regs *regs) { - bool from_idle, percpu_needs_fixup; struct pt_regs *old_regs; irqentry_state_t state; + bool from_idle; percpu_entry(regs); state = irqentry_enter(regs); @@ -214,14 +213,13 @@ void noinstr do_ext_irq(struct pt_regs *regs) do_irq_async(regs, EXT_INTERRUPT); - percpu_needs_fixup = percpu_code_check(regs); irq_exit_rcu(); set_irq_regs(old_regs); irqentry_exit(regs, state); if (from_idle) regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); - percpu_exit(regs, percpu_needs_fixup); + percpu_exit(regs); } static void show_msi_interrupt(struct seq_file *p, int irq) diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 17297a8b63d9..56a15edc65d7 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -361,7 +361,6 @@ NOKPROBE_SYMBOL(s390_backup_mcck_info); */ void notrace s390_do_machine_check(struct pt_regs *regs) { - bool percpu_needs_fixup; static int ipd_count; static DEFINE_SPINLOCK(ipd_lock); static unsigned long long last_ipd; @@ -495,9 +494,8 @@ void notrace s390_do_machine_check(struct pt_regs *regs) if (mcck_pending) schedule_mcck_handler(); - percpu_needs_fixup = percpu_code_check(regs); irqentry_nmi_exit(regs, irq_state); - percpu_exit(regs, percpu_needs_fixup); + percpu_exit(regs); } NOKPROBE_SYMBOL(s390_do_machine_check); diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index b6ba4465f59d..dd051046170b 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -338,7 +338,6 @@ static void (*pgm_check_table[128])(struct pt_regs *regs); void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags) { struct lowcore *lc = get_lowcore(); - bool percpu_needs_fixup; irqentry_state_t state; struct pgm_stat *stat; unsigned int trapnr; @@ -400,9 +399,8 @@ void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags) pgm_check_table[trapnr](regs); out: local_irq_disable(); - percpu_needs_fixup = percpu_code_check(regs); irqentry_exit(regs, state); - percpu_exit(regs, percpu_needs_fixup); + percpu_exit(regs); } static int pgm_check_stat_show(struct seq_file *p, void *v) From 6dcc4ef892f266c5ecc97c69adfd8717d7d4be7c Mon Sep 17 00:00:00 2001 From: Niklas Schnelle Date: Mon, 28 Sep 2026 12:12:34 +0200 Subject: [PATCH 032/102] s390/configs: Enable NVMe-oF RDMA and TCP in defconfig NVMe-oF with RDMA and TCP transports is enabled in distributions both in host and target mode. Enable it for defconfigs as well to align them with distributions and simplify testing. As part of this also enable NVMe native multipath support which is also enabled in distributions and required for I/O policy support. Signed-off-by: Niklas Schnelle Reviewed-by: Halil Pasic Signed-off-by: Heiko Carstens --- arch/s390/configs/debug_defconfig | 14 ++++++++++++++ arch/s390/configs/defconfig | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 3dae71474333..3892d9dd75d9 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -451,6 +451,20 @@ CONFIG_BLK_DEV_RAM_SIZE=32768 CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_RBD=m CONFIG_BLK_DEV_NVME=m +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_VERBOSE_ERRORS=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_TCP=m +CONFIG_NVME_TCP_TLS=y +CONFIG_NVME_HOST_AUTH=y +CONFIG_NVME_TARGET=m +CONFIG_NVME_TARGET_DEBUGFS=y +CONFIG_NVME_TARGET_PASSTHRU=y +CONFIG_NVME_TARGET_LOOP=m +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_TCP=m +CONFIG_NVME_TARGET_TCP_TLS=y +CONFIG_NVME_TARGET_AUTH=y CONFIG_BLK_DEV_NULL_BLK=m CONFIG_ENCLOSURE_SERVICES=m CONFIG_GENWQE=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 6f5722634b4d..95720e8ad80d 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -441,6 +441,20 @@ CONFIG_BLK_DEV_RAM_SIZE=32768 CONFIG_VIRTIO_BLK=y CONFIG_BLK_DEV_RBD=m CONFIG_BLK_DEV_NVME=m +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_VERBOSE_ERRORS=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_TCP=m +CONFIG_NVME_TCP_TLS=y +CONFIG_NVME_HOST_AUTH=y +CONFIG_NVME_TARGET=m +CONFIG_NVME_TARGET_DEBUGFS=y +CONFIG_NVME_TARGET_PASSTHRU=y +CONFIG_NVME_TARGET_LOOP=m +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_TCP=m +CONFIG_NVME_TARGET_TCP_TLS=y +CONFIG_NVME_TARGET_AUTH=y CONFIG_BLK_DEV_NULL_BLK=m CONFIG_ENCLOSURE_SERVICES=m CONFIG_GENWQE=m From f63eb4ee4a30fc5d8de07a229d92310e3c8d560c Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Tue, 29 Sep 2026 15:58:29 +0200 Subject: [PATCH 033/102] s390/vmcp: Initialize full session info to zero The cma_alloc bit as well as the response can contain old data. cma_release will not free the kmalloced memory in that case and the response code might leak kernel stack data to CAP_SYS_ADMIN users. Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Heiko Carstens Signed-off-by: Heiko Carstens --- drivers/s390/char/vmcp.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/s390/char/vmcp.c b/drivers/s390/char/vmcp.c index b277f9d0c321..2cf94f23af2a 100644 --- a/drivers/s390/char/vmcp.c +++ b/drivers/s390/char/vmcp.c @@ -104,13 +104,11 @@ static int vmcp_open(struct inode *inode, struct file *file) if (!capable(CAP_SYS_ADMIN)) return -EPERM; - session = kmalloc_obj(*session); + session = kzalloc_obj(*session); if (!session) return -ENOMEM; session->bufsize = PAGE_SIZE; - session->response = NULL; - session->resp_size = 0; mutex_init(&session->mutex); file->private_data = session; return nonseekable_open(inode, file); From 5e9b339f323216d71f3ede8a0a427abf7859c486 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Fri, 2 Oct 2026 12:52:36 +0200 Subject: [PATCH 034/102] s390/zcrypt: Fix and improve zcrypt reply message verification checks Add or improve checks related to buffer sizes and reply sizes to the handling of replies from the crypto cards for CCA, EP11 (AP message type 6) and ICA (AP type 50) messages. The verification code related to reply field length was not designed well and thus firmware deficiencies could lead to unexpected behavior in the zcrypt device driver. Thus improve the code to more closely inspect especially length fields at message replies. Add length validation before accessing reply message structures in zcrypt_msgtype6_receive(), zcrypt_msgtype6_receive_ep11(), and zcrypt_msgtype50_receive() to prevent out-of-bounds reads and potential kernel memory disclosure. Also rework these three functions to validate reply lengths more carefully before copying data back into the request buffer. Use size_t for length calculations, reject inconsistent reply sizes, and add defensive handling for short invalid replies. For XCRB replies, validate both reply segments and derive the effective message length from the covered range instead of trusting only the second segment. Additionally improve some of the later invoked evaluation functions which further check the payload. Add comments about length assumptions and for the type50 processing rework the payload processing completely with removing a misplaced BUG_ON(). Fixes: 5e55a488c87a ("s390/zcrypt: Separate msgtype implementation from card modules.") Signed-off-by: Harald Freudenberger Cc: stable@vger.kernel.org Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/zcrypt_msgtype50.c | 95 +++++++---- drivers/s390/crypto/zcrypt_msgtype6.c | 208 ++++++++++++++++++------- 2 files changed, 221 insertions(+), 82 deletions(-) diff --git a/drivers/s390/crypto/zcrypt_msgtype50.c b/drivers/s390/crypto/zcrypt_msgtype50.c index d6fc2d8e7fad..5907fb2a6920 100644 --- a/drivers/s390/crypto/zcrypt_msgtype50.c +++ b/drivers/s390/crypto/zcrypt_msgtype50.c @@ -152,6 +152,11 @@ struct type80_hdr { unsigned char reserved3[8]; } __packed; +struct type80_reply { + struct type80_hdr hdr; + char data[]; +} __packed; + int get_rsa_modex_fc(struct ica_rsa_modexpo *mex, int *fcode) { if (!mex->inputdatalength) @@ -340,32 +345,43 @@ static int ICACRT_msg_to_type50CRT_msg(struct zcrypt_queue *zq, * @data: pointer to user output data * @length: size of user output data * - * Returns 0 on success or -EFAULT. + * Returns 0 on success or neg. errno value on failure. */ static int convert_type80(struct zcrypt_queue *zq, struct ap_message *reply, char __user *outputdata, unsigned int outputdatalength) { - struct type80_hdr *t80h = reply->msg; - unsigned char *data; + struct type80_reply *msg = reply->msg; + size_t payload_len; - if (t80h->len < sizeof(*t80h) + outputdatalength) { - /* The result is too short, the CEXxA card may not do that.. */ + /* + * reply->len is always >= sizeof(struct type80_reply) here and + * outputdatalength (the modulus size) is guaranteed to be equal to + * inputdatalength and 0 < outputdatalength <= CEX3A_MAX_MOD_SIZE, + * also payload size may be >= outputdatalength. + */ + + payload_len = reply->len - sizeof(msg->hdr); + + if (outputdatalength > payload_len) { + /* We expect at least outputdatalength bytes, broken card ? */ zq->online = 0; pr_err("Crypto dev=%02x.%04x code=0x%02x => online=0 rc=EAGAIN\n", AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), t80h->code); + AP_QID_QUEUE(zq->queue->qid), msg->hdr.code); ZCRYPT_DBF_ERR("%s dev=%02x.%04x code=0x%02x => online=0 rc=EAGAIN\n", __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), t80h->code); + AP_QID_QUEUE(zq->queue->qid), msg->hdr.code); ap_send_online_uevent(&zq->queue->ap_dev, zq->online); return -EAGAIN; } - BUG_ON(t80h->len > CEX3A_MAX_RESPONSE_SIZE); - data = reply->msg + t80h->len - outputdatalength; - if (copy_to_user(outputdata, data, outputdatalength)) + + if (copy_to_user(outputdata, + msg->data + payload_len - outputdatalength, + outputdatalength)) return -EFAULT; + return 0; } @@ -374,14 +390,19 @@ static int convert_response(struct zcrypt_queue *zq, char __user *outputdata, unsigned int outputdatalength) { - /* Response type byte is the second byte in the response. */ - unsigned char rtype = ((unsigned char *)reply->msg)[1]; + struct type80_reply *msg = reply->msg; - switch (rtype) { + /* reply->len is always >= sizeof(struct error_hdr) here */ + + switch (msg->hdr.type) { case TYPE82_RSP_CODE: case TYPE88_RSP_CODE: return convert_error(zq, reply); case TYPE80_RSP_CODE: + if (msg->hdr.code) + return convert_error(zq, reply); + if (reply->len < sizeof(struct type80_reply)) + return -EINVAL; return convert_type80(zq, reply, outputdata, outputdatalength); default: /* Unknown response type, this should NEVER EVER happen */ @@ -389,11 +410,11 @@ static int convert_response(struct zcrypt_queue *zq, pr_err("Crypto dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", AP_QID_CARD(zq->queue->qid), AP_QID_QUEUE(zq->queue->qid), - (int)rtype); + (int)msg->hdr.type); ZCRYPT_DBF_ERR( "%s dev=%02x.%04x unknown response type 0x%02x => online=0 rc=EAGAIN\n", __func__, AP_QID_CARD(zq->queue->qid), - AP_QID_QUEUE(zq->queue->qid), (int)rtype); + AP_QID_QUEUE(zq->queue->qid), (int)msg->hdr.type); ap_send_online_uevent(&zq->queue->ap_dev, zq->online); return -EAGAIN; } @@ -416,26 +437,46 @@ static void zcrypt_msgtype50_receive(struct ap_queue *aq, .reply_code = REP82_ERROR_MACHINE_FAILURE, }; struct type80_hdr *t80h; - int len; + size_t len; /* Copy the reply message to the request message buffer. */ if (!reply) goto out; /* ap_msg->rc indicates the error */ + t80h = reply->msg; - if (t80h->type == TYPE80_RSP_CODE) { - len = t80h->len; - if (len > reply->bufsize || len > msg->bufsize || - len != reply->len) { - pr_debug("len mismatch => EMSGSIZE\n"); - msg->rc = -EMSGSIZE; + + if (reply->len < sizeof(*t80h) || + t80h->type != TYPE80_RSP_CODE) { + if (reply->len < sizeof(error_reply)) { + /* total broken reply, use static error reply instead */ + memcpy(msg->msg, &error_reply, sizeof(error_reply)); + msg->len = sizeof(error_reply); goto out; + } else { + /* malformed reply, convert function will handle this */ + len = reply->len; + goto copy_len_and_out; } - memcpy(msg->msg, reply->msg, len); - msg->len = len; - } else { - memcpy(msg->msg, reply->msg, sizeof(error_reply)); - msg->len = sizeof(error_reply); } + + len = t80h->len; + +copy_len_and_out: + if (len != reply->len) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu rpl.len %zu mismatch, msg.rc=%d\n", + len, reply->len, msg->rc); + goto out; + } + if (len > reply->bufsize || len > msg->bufsize) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu exceeds buf %zu/%zu, msg.rc=%d\n", + len, reply->bufsize, msg->bufsize, msg->rc); + goto out; + } + memcpy(msg->msg, reply->msg, len); + msg->len = len; + out: complete(&msg->response.work); } diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c index 3df1d676de5d..5962224bfb8b 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.c +++ b/drivers/s390/crypto/zcrypt_msgtype6.c @@ -766,6 +766,15 @@ static int convert_type86_rng(struct zcrypt_queue *zq, if (msg->cprbx.ccp_rtcode != 0 || msg->cprbx.ccp_rscode != 0) return -EINVAL; + /* + * Note that offset2 and count2 have already been checked in + * zcrypt_msgtype6_receive(). So only check for valid count2 + * and for not exceeding the hard coded rng buffer size. + */ + if (!msg->fmt2.count2) + return -EINVAL; + if (msg->fmt2.count2 > ZCRYPT_RNG_BUFFER_SIZE) + return -EMSGSIZE; memcpy(buffer, data + msg->fmt2.offset2, msg->fmt2.count2); return msg->fmt2.count2; } @@ -777,11 +786,17 @@ static int convert_response_ica(struct zcrypt_queue *zq, { struct type86x_reply *msg = reply->msg; + /* reply->len is always >= sizeof(struct error_hdr) here */ + switch (msg->hdr.type) { case TYPE82_RSP_CODE: case TYPE88_RSP_CODE: return convert_error(zq, reply); case TYPE86_RSP_CODE: + if (msg->hdr.reply_code) + return convert_error(zq, reply); + if (reply->len < sizeof(struct type86x_reply)) + return -EINVAL; if (msg->cprbx.ccp_rtcode && msg->cprbx.ccp_rscode == 0x14f && outputdatalength > 256) { @@ -792,8 +807,6 @@ static int convert_response_ica(struct zcrypt_queue *zq, return -EINVAL; } } - if (msg->hdr.reply_code) - return convert_error(zq, reply); if (msg->cprbx.cprb_ver_id == 0x02) return convert_type86_ica(zq, reply, outputdata, outputdatalength); @@ -820,6 +833,8 @@ static int convert_response_xcrb(bool userspace, struct zcrypt_queue *zq, { struct type86x_reply *msg = reply->msg; + /* reply->len is always >= sizeof(struct error_hdr) here */ + switch (msg->hdr.type) { case TYPE82_RSP_CODE: case TYPE88_RSP_CODE: @@ -827,9 +842,16 @@ static int convert_response_xcrb(bool userspace, struct zcrypt_queue *zq, return convert_error(zq, reply); case TYPE86_RSP_CODE: if (msg->hdr.reply_code) { - xcrb->status = msg->fmt2.apfs; + if (reply->len < sizeof(struct type86_fmt2_msg)) + xcrb->status = 0x0008044DL; + else + xcrb->status = msg->fmt2.apfs; return convert_error(zq, reply); } + if (reply->len < sizeof(struct type86_fmt2_msg) + + offsetof(struct CPRBX, cprb_ver_id) + + sizeof(msg->cprbx.cprb_ver_id)) + return -EINVAL; if (msg->cprbx.cprb_ver_id == 0x02) return convert_type86_xcrb(userspace, zq, reply, xcrb); fallthrough; /* wrong cprb version is an unknown response */ @@ -854,6 +876,8 @@ static int convert_response_ep11_xcrb(bool userspace, struct zcrypt_queue *zq, { struct type86_ep11_reply *msg = reply->msg; + /* reply->len is always >= sizeof(struct error_hdr) here */ + switch (msg->hdr.type) { case TYPE82_RSP_CODE: case TYPE87_RSP_CODE: @@ -861,6 +885,8 @@ static int convert_response_ep11_xcrb(bool userspace, struct zcrypt_queue *zq, case TYPE86_RSP_CODE: if (msg->hdr.reply_code) return convert_error(zq, reply); + if (reply->len < sizeof(struct type86_ep11_reply)) + return -EINVAL; if (msg->cprbx.cprb_ver_id == 0x04) return convert_type86_ep11_xcrb(userspace, zq, reply, xcrb); fallthrough; /* wrong cprb version is an unknown resp */ @@ -885,6 +911,8 @@ static int convert_response_rng(struct zcrypt_queue *zq, { struct type86x_reply *msg = reply->msg; + /* reply->len is always >= sizeof(struct error_hdr) here */ + switch (msg->hdr.type) { case TYPE82_RSP_CODE: case TYPE88_RSP_CODE: @@ -892,6 +920,8 @@ static int convert_response_rng(struct zcrypt_queue *zq, case TYPE86_RSP_CODE: if (msg->hdr.reply_code) return -EINVAL; + if (reply->len < sizeof(struct type86x_reply)) + return -EINVAL; if (msg->cprbx.cprb_ver_id == 0x02) return convert_type86_rng(zq, reply, data); fallthrough; /* wrong cprb version is an unknown response */ @@ -928,48 +958,85 @@ static void zcrypt_msgtype6_receive(struct ap_queue *aq, }; struct ap_response_type *resp_type = &msg->response; struct type86x_reply *t86r; - int len; + size_t minlen, len; /* Copy the reply message to the request message buffer. */ if (!reply) goto out; /* ap_msg->rc indicates the error */ + t86r = reply->msg; - if (t86r->hdr.type == TYPE86_RSP_CODE && - t86r->cprbx.cprb_ver_id == 0x02) { - switch (resp_type->type) { - case CEXXC_RESPONSE_TYPE_ICA: - len = sizeof(struct type86x_reply) + t86r->length; - if (len > reply->bufsize || len > msg->bufsize || - len != reply->len) { - pr_debug("len mismatch => EMSGSIZE\n"); - msg->rc = -EMSGSIZE; - goto out; - } - memcpy(msg->msg, reply->msg, len); - msg->len = len; - break; - case CEXXC_RESPONSE_TYPE_XCRB: - if (t86r->fmt2.count2) - len = t86r->fmt2.offset2 + t86r->fmt2.count2; - else - len = t86r->fmt2.offset1 + t86r->fmt2.count1; - if (len > reply->bufsize || len > msg->bufsize || - len != reply->len) { - pr_debug("len mismatch => EMSGSIZE\n"); - msg->rc = -EMSGSIZE; - goto out; - } - memcpy(msg->msg, reply->msg, len); - msg->len = len; - break; - default: + minlen = sizeof(t86r->hdr) + sizeof(t86r->fmt2) + + offsetof(struct CPRBX, cprb_ver_id) + + sizeof(t86r->cprbx.cprb_ver_id); + + if (reply->len < minlen || + t86r->hdr.type != TYPE86_RSP_CODE || + t86r->cprbx.cprb_ver_id != 0x02) { + if (reply->len < sizeof(error_reply)) { + /* total broken reply, use static error reply instead */ memcpy(msg->msg, &error_reply, sizeof(error_reply)); msg->len = sizeof(error_reply); + goto out; + } else { + /* malformed reply, convert function will handle this */ + len = reply->len; + goto copy_len_and_out; } - } else { - memcpy(msg->msg, reply->msg, sizeof(error_reply)); - msg->len = sizeof(error_reply); } + + switch (resp_type->type) { + case CEXXC_RESPONSE_TYPE_ICA: + if (reply->len < sizeof(struct type86x_reply)) { + msg->rc = -EMSGSIZE; + pr_debug("rpl.len %zu < struct type86_reply, msg.rc=%d\n", + reply->len, msg->rc); + goto out; + } + len = sizeof(struct type86x_reply) + (size_t)t86r->length; + break; + case CEXXC_RESPONSE_TYPE_XCRB: + len = (size_t)t86r->fmt2.offset1 + (size_t)t86r->fmt2.count1; + if (len > reply->len) { + msg->rc = -EMSGSIZE; + pr_debug("offset1 %u count1 %u rpl.len %zu mismatch, msg.rc=%d\n", + t86r->fmt2.offset1, t86r->fmt2.count1, + reply->len, msg->rc); + goto out; + } + if (t86r->fmt2.count2) { + len = (size_t)t86r->fmt2.offset2 + + (size_t)t86r->fmt2.count2; + if (len > reply->len) { + msg->rc = -EMSGSIZE; + pr_debug("offset2 %u count2 %u rpl.len %zu mismatch, msg.rc=%d\n", + t86r->fmt2.offset2, t86r->fmt2.count2, + reply->len, msg->rc); + goto out; + } + } + break; + default: + memcpy(msg->msg, &error_reply, sizeof(error_reply)); + msg->len = sizeof(error_reply); + goto out; + } + +copy_len_and_out: + if (len != reply->len) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu rpl.len %zu mismatch, msg.rc=%d\n", + len, reply->len, msg->rc); + goto out; + } + if (len > reply->bufsize || len > msg->bufsize) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu exceeds buf %zu/%zu, msg.rc=%d\n", + len, reply->bufsize, msg->bufsize, msg->rc); + goto out; + } + memcpy(msg->msg, reply->msg, len); + msg->len = len; + out: complete(&resp_type->work); } @@ -992,34 +1059,65 @@ static void zcrypt_msgtype6_receive_ep11(struct ap_queue *aq, }; struct ap_response_type *resp_type = &msg->response; struct type86_ep11_reply *t86r; - int len; + size_t minlen, len; /* Copy the reply message to the request message buffer. */ if (!reply) goto out; /* ap_msg->rc indicates the error */ + t86r = reply->msg; - if (t86r->hdr.type == TYPE86_RSP_CODE && - t86r->cprbx.cprb_ver_id == 0x04) { - switch (resp_type->type) { - case CEXXC_RESPONSE_TYPE_EP11: - len = t86r->fmt2.offset1 + t86r->fmt2.count1; - if (len > reply->bufsize || len > msg->bufsize || - len != reply->len) { - pr_debug("len mismatch => EMSGSIZE\n"); - msg->rc = -EMSGSIZE; - goto out; - } - memcpy(msg->msg, reply->msg, len); - msg->len = len; - break; - default: + minlen = sizeof(t86r->hdr) + sizeof(t86r->fmt2) + + offsetof(struct ep11_cprb, cprb_ver_id) + + sizeof(t86r->cprbx.cprb_ver_id); + + if (reply->len < minlen || + t86r->hdr.type != TYPE86_RSP_CODE || + t86r->cprbx.cprb_ver_id != 0x04) { + if (reply->len < sizeof(error_reply)) { + /* total broken reply, use static error reply instead */ memcpy(msg->msg, &error_reply, sizeof(error_reply)); msg->len = sizeof(error_reply); + goto out; + } else { + /* malformed reply, convert function will handle this */ + len = reply->len; + goto copy_len_and_out; } - } else { - memcpy(msg->msg, reply->msg, sizeof(error_reply)); - msg->len = sizeof(error_reply); } + + switch (resp_type->type) { + case CEXXC_RESPONSE_TYPE_EP11: + len = (size_t)t86r->fmt2.offset1 + (size_t)t86r->fmt2.count1; + if (len > reply->len) { + msg->rc = -EMSGSIZE; + pr_debug("offset1 %u count1 %u rpl.len %zu mismatch, msg.rc=%d\n", + t86r->fmt2.offset1, t86r->fmt2.count1, + reply->len, msg->rc); + goto out; + } + break; + default: + memcpy(msg->msg, &error_reply, sizeof(error_reply)); + msg->len = sizeof(error_reply); + goto out; + } + +copy_len_and_out: + if (len != reply->len) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu rpl.len %zu mismatch, msg.rc=%d\n", + len, reply->len, msg->rc); + goto out; + } + if (len > reply->bufsize || len > msg->bufsize) { + msg->rc = -EMSGSIZE; + pr_debug("len %zu exceeds buf %zu/%zu, msg.rc=%d\n", + len, reply->bufsize, msg->bufsize, msg->rc); + goto out; + } + memcpy(msg->msg, reply->msg, len); + msg->len = len; + out: complete(&resp_type->work); } From 74a31be974dde1d12ed9db131b413c519f8fc429 Mon Sep 17 00:00:00 2001 From: Shirong Zhao Date: Mon, 5 Oct 2026 13:58:25 +0800 Subject: [PATCH 035/102] s390/fs3270: Return -ENOTTY for unknown ioctls in fs3270_ioctl() fs3270_ioctl() initializes rc to 0 and its cmd switch has no default branch, so an unrecognized ioctl command returns success without doing anything. Follow the ioctl convention and reject unknown commands with -ENOTTY. [hca@linux.ibm.com: shortened commit description] Signed-off-by: Shirong Zhao Reviewed-by: Heiko Carstens Signed-off-by: Heiko Carstens --- drivers/s390/char/fs3270.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/s390/char/fs3270.c b/drivers/s390/char/fs3270.c index f8e2cc17ac53..25f0a85d92fa 100644 --- a/drivers/s390/char/fs3270.c +++ b/drivers/s390/char/fs3270.c @@ -355,6 +355,9 @@ static long fs3270_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (copy_to_user(argp, &iocb, sizeof(struct raw3270_iocb))) rc = -EFAULT; break; + default: + rc = -ENOTTY; + break; } mutex_unlock(&fs3270_mutex); return rc; From 833ee20e712bd0c71435feddaa7b578799fe50c6 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 5 Oct 2026 11:22:56 +0200 Subject: [PATCH 036/102] s390/dis: Mark non-NUL-terminated character arrays with __nonstring With [1] the generated string initializers were changed to use an array instead of a non-NUL-terminated string. The reason was that GCC 15 warns about string initializers that truncate the NUL terminator with -Wunterminated-string-initialization. The better solution is to use the __nonstring annotation, which exists to deal exactly with this problem. Therefore use it and basically revert [1] in order to generate more readable initializers: Before: [LONG_INSN_ALGHSIK] = { 'a', 'l', 'g', 'h', 's', 'i', 'k', }, After: [LONG_INSN_ALGHSIK] = "alghsik", [1] commit 7cbae7ea3de5 ("s390/tools: Use array instead of string initializer") Reviewed-by: Jens Remus Signed-off-by: Heiko Carstens --- arch/s390/kernel/dis.c | 4 ++-- arch/s390/tools/gen_opcode_table.c | 23 ++++------------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index 1cec93895b3a..9c349eb884bf 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -51,7 +51,7 @@ struct s390_operand { struct s390_insn { union { - const char name[5]; + const char name[5] __nonstring; struct { unsigned char zero; unsigned int offset; @@ -340,7 +340,7 @@ static const unsigned char formats[][6] = { [INSTR_VSI_URDV] = { V_32, D_20, B_16, U8_8, 0, 0 }, }; -static char long_insn_name[][7] = LONG_INSN_INITIALIZER; +static char long_insn_name[][7] __nonstring_array = LONG_INSN_INITIALIZER; static struct s390_insn opcode[] = OPCODE_TABLE_INITIALIZER; static struct s390_opcode_offset opcode_offset[] = OPCODE_OFFSET_INITIALIZER; diff --git a/arch/s390/tools/gen_opcode_table.c b/arch/s390/tools/gen_opcode_table.c index 7d76c417f83f..820905eca78a 100644 --- a/arch/s390/tools/gen_opcode_table.c +++ b/arch/s390/tools/gen_opcode_table.c @@ -201,17 +201,6 @@ static int cmp_long_insn(const void *a, const void *b) return strcmp(((struct insn *)a)->name, ((struct insn *)b)->name); } -static void print_insn_name(const char *name) -{ - size_t i, len; - - len = strlen(name); - printf("{"); - for (i = 0; i < len; i++) - printf(" \'%c\',", name[i]); - printf(" }"); -} - static void print_long_insn(struct gen_opcode *desc) { struct insn *insn; @@ -234,9 +223,7 @@ static void print_long_insn(struct gen_opcode *desc) insn = &desc->insn[i]; if (insn->name_len < 6) continue; - printf("\t[LONG_INSN_%s] = ", insn->upper); - print_insn_name(insn->name); - printf(", \\\n"); + printf("\t[LONG_INSN_%s] = \"%s\", \\\n", insn->upper, insn->name); } printf("}\n\n"); } @@ -249,12 +236,10 @@ static void print_opcode(struct insn *insn, int nr) if (insn->type->byte != 0) opcode += 2; printf("\t[%4d] = { .opfrag = 0x%s, .format = INSTR_%s, ", nr, opcode, insn->format); - if (insn->name_len < 6) { - printf(".name = "); - print_insn_name(insn->name); - } else { + if (insn->name_len < 6) + printf(".name = \"%s\"", insn->name); + else printf(".offset = LONG_INSN_%s", insn->upper); - } printf(" }, \\\n"); } From 99a95985dfc85eecad97f0991a2401d2fcef294c Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Tue, 6 Oct 2026 13:13:03 +0200 Subject: [PATCH 037/102] s390/cpum_cf: Check session mask before cpu offline cfset_offline_cpu() stops counter sets for every active session without checking whether the outgoing cpu belongs to the session mask. For example, run an hwctr session on cpu0 and a cpu bound perf event on cpu1: lshwc -l 9999 -i 5 0:b perf stat -C 1 -e cpum_cf/CPU_CYCLES/ sleep 120 chcpu -d 1 chcpu -e 1 Although the hwctr session does not include cpu1, cfset_offline_cpu() decrements its counter set reference count. The subsequent perf teardown decrements it again, causing it to underflow. The perf event keeps the percpu state alive across hotplug. When cpu1 comes online, the hwctr session is started there and the reference count moves from -1 to 0. Stopping the hwctr session then decrements it to -1 again, so the zero triggered counter set disable path is skipped. Check the session mask before stopping counter sets on the outgoing cpu. This prevents the unmatched decrement and the resulting reference count underflow. Fixes: a029a4eab39e ("s390/cpumf: Allow concurrent access for CPU Measurement Counter Facility") Reported-by: Christian Borntraeger Cc: stable@vger.kernel.org # v5.14+ Signed-off-by: Sumanth Korikkar Reviewed-by: Heiko Carstens Signed-off-by: Heiko Carstens --- arch/s390/kernel/perf_cpum_cf.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 69fa303cd479..1d63dc3b0ab9 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -1797,6 +1797,9 @@ static int cfset_offline_cpu(unsigned int cpu) if (!list_empty(&cfset_session.head)) { list_for_each_entry(rp, &cfset_session.head, node) { + if (!cpumask_test_cpu(cpu, &rp->mask)) + continue; + p.sets = rp->ctrset; cfset_ioctl_off(&p); cpumask_clear_cpu(cpu, &rp->mask); From c8b2bb9385d08fbb734146b7902838ca79097127 Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Tue, 6 Oct 2026 13:13:04 +0200 Subject: [PATCH 038/102] s390/cpum_cf: Honor hwctr session cpu mask during hotplug cfset_online_cpu() and cfset_offline_cpu() cpu hotplug callbacks start and stop counter sets for every active hwctr session without checking the session cpu mask, and dynamically modify that mask on each hotplug event. For example, assume cpu0 and cpu1 are online and cpu2 is offline. Start an hwctr session restricted to cpu0 and cpu1, and then bring cpu2 online: lshwc -l 9999 -i 5 0-1:b chcpu -e 2 Although cpu2 is not part of the requested mask, cfset_online_cpu() starts the counter sets on cpu2 and adds it to the session mask. cpu2 can then appear in lshwc output (s390-tools) and is handled as part of the session until it is stopped. Keep the userspace provided cpu mask unchanged and update counter sets only when the hotplugged cpu belongs to that mask. For all cpu requests, lshwc should supply the possible cpu mask so that cpus coming online later remain part of the session. Note: A corresponding lshwc (s390-tools) change is needed to supply the possible cpu mask for allcpu sessions, ensuring that cpus coming online later remain part of the session. Without it, newly online cpus would be silently excluded from an active session. Since this userspace change is not yet available, the patch is not suitable for stable kernels and intentionally omits a Fixes: tag. The prerequisite bug fix is in "s390/cpum_cf: Check session mask before cpu offline". Reported-by: Christian Borntraeger Signed-off-by: Sumanth Korikkar Reviewed-by: Heiko Carstens Signed-off-by: Heiko Carstens --- arch/s390/kernel/perf_cpum_cf.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 1d63dc3b0ab9..a2334af51544 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -1768,8 +1768,9 @@ static struct miscdevice cfset_dev = { .mode = 0666, }; -/* Hotplug add of a CPU. Scan through all active processes and add - * that CPU to the list of CPUs supplied with ioctl(..., START, ...). +/* + * Start counter sets on a newly online CPU for each active hwctr session that + * includes this CPU in its requested mask. */ static int cfset_online_cpu(unsigned int cpu) { @@ -1778,17 +1779,19 @@ static int cfset_online_cpu(unsigned int cpu) if (!list_empty(&cfset_session.head)) { list_for_each_entry(rp, &cfset_session.head, node) { + if (!cpumask_test_cpu(cpu, &rp->mask)) + continue; + p.sets = rp->ctrset; cfset_ioctl_on(&p); - cpumask_set_cpu(cpu, &rp->mask); } } return 0; } -/* Hotplug remove of a CPU. Scan through all active processes and clear - * that CPU from the list of CPUs supplied with ioctl(..., START, ...). - * Adjust reference counts. +/* + * Stop counter sets on an outgoing CPU for each active hwctr session that + * includes this CPU in its requested mask. */ static int cfset_offline_cpu(unsigned int cpu) { @@ -1802,7 +1805,6 @@ static int cfset_offline_cpu(unsigned int cpu) p.sets = rp->ctrset; cfset_ioctl_off(&p); - cpumask_clear_cpu(cpu, &rp->mask); } } return 0; From 0c920885e593ee0be7579c22adf7f6a1bbef71be Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 6 Oct 2026 16:01:12 +0200 Subject: [PATCH 039/102] s390/appldata: Emulate virtual timer with delayed work Emulate the virtual timer in appldata using delayed work: the work is scheduled for the minimum possible wall-clock time until the configured CPU-time interval elapses (remaining CPU time / number of online CPUs), with a minimum delay of 100ms. The work then reads per-CPU statistics of all online CPUs to check whether enough CPU time has elapsed, and if so runs the registered callbacks. This is slightly more expensive, but allows to subsequently remove the entire vtimer infrastructure. Reviewed-by: Gerald Schaefer Tested-by: Gerald Schaefer Signed-off-by: Heiko Carstens --- arch/s390/appldata/appldata_base.c | 171 ++++++++++++++++++----------- 1 file changed, 107 insertions(+), 64 deletions(-) diff --git a/arch/s390/appldata/appldata_base.c b/arch/s390/appldata/appldata_base.c index 9cba4633c3f3..836c51da18f4 100644 --- a/arch/s390/appldata/appldata_base.c +++ b/arch/s390/appldata/appldata_base.c @@ -28,19 +28,14 @@ #include #include #include +#include #include -#include #include #include "appldata.h" - -#define APPLDATA_CPU_INTERVAL 10000 /* default (CPU) time for - sampling interval in - milliseconds */ - -#define TOD_MICRO 0x01000 /* nr. of TOD clock units - for 1 microsecond */ +/* Default CPU time for sampling interval */ +#define APPLDATA_CPU_INTERVAL (10 * NSEC_PER_SEC) /* * /proc entries (sysctl) @@ -64,22 +59,14 @@ static const struct ctl_table appldata_table[] = { }, }; -/* - * Timer - */ -static struct vtimer_list appldata_timer; +static void appldata_work_fn(struct work_struct *work); +static DECLARE_DELAYED_WORK(appldata_work, appldata_work_fn); -static DEFINE_SPINLOCK(appldata_timer_lock); -static int appldata_interval = APPLDATA_CPU_INTERVAL; +static DEFINE_MUTEX(appldata_timer_lock); +static u64 appldata_interval = APPLDATA_CPU_INTERVAL; static int appldata_timer_active; -/* - * Work queue - */ -static struct workqueue_struct *appldata_wq; -static void appldata_work_fn(struct work_struct *work); -static DECLARE_WORK(appldata_work, appldata_work_fn); - +static u64 appldata_cputime_start; /* * Ops list @@ -89,14 +76,34 @@ static LIST_HEAD(appldata_ops_list); /*************************** timer, work, DIAG *******************************/ -/* - * appldata_timer_function() - * - * schedule work and reschedule timer - */ -static void appldata_timer_function(unsigned long data) +static u64 appldata_total_cpu_time_ns(void) { - queue_work(appldata_wq, (struct work_struct *) data); + u64 total = 0; + int cpu; + + for_each_possible_cpu(cpu) { + total += kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; + total += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; + total += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; + total += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; + total += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; + } + return total; +} + +static void appldata_schedule_work(u64 remaining) +{ + unsigned int ncpus = num_online_cpus(); + unsigned long delay = HZ / 10; + + /* + * At most ncpus CPUs consume CPU time simultaneously, so the + * minimum wall-clock time until the remaining CPU time elapses + * is remaining / ncpus. + * Make sure the work is not scheduled more than once per 100ms. + */ + delay = max(delay, nsecs_to_jiffies(remaining / ncpus)); + mod_delayed_work(system_percpu_wq, &appldata_work, delay); } /* @@ -106,9 +113,27 @@ static void appldata_timer_function(unsigned long data) */ static void appldata_work_fn(struct work_struct *work) { - struct list_head *lh; + u64 now, elapsed, interval; struct appldata_ops *ops; + struct list_head *lh; + bool expired = false; + now = appldata_total_cpu_time_ns(); + mutex_lock(&appldata_timer_lock); + if (appldata_timer_active) { + interval = appldata_interval; + elapsed = now - appldata_cputime_start; + if (elapsed < interval) { + appldata_schedule_work(interval - elapsed); + } else { + appldata_cputime_start = now; + appldata_schedule_work(interval); + expired = true; + } + } + mutex_unlock(&appldata_timer_lock); + if (!expired) + return; mutex_lock(&appldata_ops_mutex); list_for_each(lh, &appldata_ops_list) { ops = list_entry(lh, struct appldata_ops, list); @@ -162,33 +187,52 @@ int appldata_diag(char record_nr, u16 function, unsigned long buffer, #define APPLDATA_MOD_TIMER 2 /* - * __appldata_vtimer_setup() + * __appldata_timer_setup() * - * Add, delete or modify virtual timers on all online cpus. - * The caller needs to get the appldata_timer_lock spinlock. + * Add, delete or modify the appldata delayed work. */ -static void __appldata_vtimer_setup(int cmd) +static void __appldata_timer_setup(int cmd) { - u64 timer_interval = (u64) appldata_interval * 1000 * TOD_MICRO; + static DEFINE_MUTEX(appldata_cancel_lock); + u64 now, elapsed, remaining; switch (cmd) { case APPLDATA_ADD_TIMER: - if (appldata_timer_active) - break; - appldata_timer.expires = timer_interval; - add_virt_timer_periodic(&appldata_timer); - appldata_timer_active = 1; + mutex_lock(&appldata_cancel_lock); + mutex_lock(&appldata_timer_lock); + if (!appldata_timer_active) { + appldata_cputime_start = appldata_total_cpu_time_ns(); + appldata_schedule_work(appldata_interval); + appldata_timer_active = 1; + } + mutex_unlock(&appldata_timer_lock); + mutex_unlock(&appldata_cancel_lock); break; case APPLDATA_DEL_TIMER: - del_virt_timer(&appldata_timer); - if (!appldata_timer_active) + mutex_lock(&appldata_cancel_lock); + mutex_lock(&appldata_timer_lock); + if (!appldata_timer_active) { + mutex_unlock(&appldata_timer_lock); + mutex_unlock(&appldata_cancel_lock); break; + } appldata_timer_active = 0; + mutex_unlock(&appldata_timer_lock); + cancel_delayed_work_sync(&appldata_work); + mutex_unlock(&appldata_cancel_lock); break; case APPLDATA_MOD_TIMER: - if (!appldata_timer_active) - break; - mod_virt_timer_periodic(&appldata_timer, timer_interval); + mutex_lock(&appldata_timer_lock); + if (appldata_timer_active) { + now = appldata_total_cpu_time_ns(); + elapsed = now - appldata_cputime_start; + remaining = appldata_interval - elapsed; + if (elapsed >= appldata_interval) + remaining = 1; + appldata_schedule_work(remaining); + } + mutex_unlock(&appldata_timer_lock); + break; } } @@ -215,12 +259,10 @@ appldata_timer_handler(const struct ctl_table *ctl, int write, if (rc < 0 || !write) return rc; - spin_lock(&appldata_timer_lock); if (timer_active) - __appldata_vtimer_setup(APPLDATA_ADD_TIMER); + __appldata_timer_setup(APPLDATA_ADD_TIMER); else - __appldata_vtimer_setup(APPLDATA_DEL_TIMER); - spin_unlock(&appldata_timer_lock); + __appldata_timer_setup(APPLDATA_DEL_TIMER); return 0; } @@ -234,7 +276,7 @@ static int appldata_interval_handler(const struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos) { - int interval = appldata_interval; + int interval = appldata_interval / NSEC_PER_MSEC; int rc; struct ctl_table ctl_entry = { .procname = ctl->procname, @@ -247,10 +289,10 @@ appldata_interval_handler(const struct ctl_table *ctl, int write, if (rc < 0 || !write) return rc; - spin_lock(&appldata_timer_lock); - appldata_interval = interval; - __appldata_vtimer_setup(APPLDATA_MOD_TIMER); - spin_unlock(&appldata_timer_lock); + mutex_lock(&appldata_timer_lock); + appldata_interval = interval * NSEC_PER_MSEC; + mutex_unlock(&appldata_timer_lock); + __appldata_timer_setup(APPLDATA_MOD_TIMER); return 0; } @@ -392,19 +434,20 @@ void appldata_unregister_ops(struct appldata_ops *ops) /******************************* init / exit *********************************/ -/* - * appldata_init() - * - * init timer, register /proc entries - */ +static int appldata_cpu_online(unsigned int cpu) +{ + __appldata_timer_setup(APPLDATA_MOD_TIMER); + return 0; +} + static int __init appldata_init(void) { - init_virt_timer(&appldata_timer); - appldata_timer.function = appldata_timer_function; - appldata_timer.data = (unsigned long) &appldata_work; - appldata_wq = alloc_ordered_workqueue("appldata", 0); - if (!appldata_wq) - return -ENOMEM; + int rc; + + rc = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "s390/appldata:online", + appldata_cpu_online, NULL); + if (rc < 0) + return rc; register_sysctl(appldata_proc_name, appldata_table); return 0; } From 13efc382cc7cc0dfba557b55dc65477c2d24ddf3 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 6 Oct 2026 16:01:13 +0200 Subject: [PATCH 040/102] s390/vtime: Remove vtimer infrastructure The only user is gone - remove the vtimer infrastructure. Signed-off-by: Heiko Carstens --- arch/s390/include/asm/vtime.h | 2 + arch/s390/include/asm/vtimer.h | 30 ----- arch/s390/kernel/process.c | 1 - arch/s390/kernel/smp.c | 2 +- arch/s390/kernel/time.c | 2 +- arch/s390/kernel/vtime.c | 212 ++------------------------------- 6 files changed, 12 insertions(+), 237 deletions(-) delete mode 100644 arch/s390/include/asm/vtimer.h diff --git a/arch/s390/include/asm/vtime.h b/arch/s390/include/asm/vtime.h index da116a93d3b6..22e494b22192 100644 --- a/arch/s390/include/asm/vtime.h +++ b/arch/s390/include/asm/vtime.h @@ -8,6 +8,8 @@ DECLARE_PER_CPU(u64, mt_cycles[8]); +void vtime_init(void); + static inline void update_timer_sys(void) { struct lowcore *lc = get_lowcore(); diff --git a/arch/s390/include/asm/vtimer.h b/arch/s390/include/asm/vtimer.h deleted file mode 100644 index e601adaa6320..000000000000 --- a/arch/s390/include/asm/vtimer.h +++ /dev/null @@ -1,30 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright IBM Corp. 2003, 2012 - * Virtual CPU timer - * - * Author(s): Jan Glauber - */ - -#ifndef _ASM_S390_TIMER_H -#define _ASM_S390_TIMER_H - -#define VTIMER_MAX_SLICE (0x7fffffffffffffffULL) - -struct vtimer_list { - struct list_head entry; - u64 expires; - u64 interval; - void (*function)(unsigned long); - unsigned long data; -}; - -extern void init_virt_timer(struct vtimer_list *timer); -extern void add_virt_timer(struct vtimer_list *timer); -extern void add_virt_timer_periodic(struct vtimer_list *timer); -extern int mod_virt_timer(struct vtimer_list *timer, u64 expires); -extern int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires); -extern int del_virt_timer(struct vtimer_list *timer); -extern void vtime_init(void); - -#endif /* _ASM_S390_TIMER_H */ diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 416650ae4871..4c73d2528fcb 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 32499cad86f0..f7c262a17c03 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -48,7 +48,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 2b989bebd220..dd8a88cc9c4b 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -48,9 +48,9 @@ #include #include #include -#include #include #include +#include #include "entry.h" union tod_clock __bootdata_preserved(tod_clock_base); diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index efcbf406f03e..d9fd39c5732e 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Virtual cpu timer based timer functions. + * Virtual CPU time accounting * * Copyright IBM Corp. 2004, 2012 * Author(s): Jan Glauber @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -22,19 +21,14 @@ #include "entry.h" -static void virt_timer_expire(void); - -static LIST_HEAD(virt_timer_list); -static DEFINE_SPINLOCK(virt_timer_lock); -static atomic64_t virt_timer_current; -static atomic64_t virt_timer_elapsed; +#define CPU_TIMER_MAX 0x7fffffffffffffffUL DEFINE_PER_CPU(u64, mt_cycles[8]); static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 }; static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 }; static DEFINE_PER_CPU(unsigned long, mt_scaling_jiffies); -static inline void set_vtimer(u64 expires) +static inline void cpu_timer_init(u64 value) { struct lowcore *lc = get_lowcore(); u64 timer; @@ -42,18 +36,9 @@ static inline void set_vtimer(u64 expires) asm volatile( " stpt %0\n" /* Store current cpu timer value */ " spt %1" /* Set new value imm. afterwards */ - : "=Q" (timer) : "Q" (expires)); + : "=Q" (timer) : "Q" (value)); lc->system_timer += lc->last_update_timer - timer; - lc->last_update_timer = expires; -} - -static inline int virt_timer_forward(u64 elapsed) -{ - lockdep_assert_irqs_disabled(); - if (list_empty(&virt_timer_list)) - return 0; - elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed); - return elapsed >= atomic64_read(&virt_timer_current); + lc->last_update_timer = value; } static void update_mt_scaling(void) @@ -125,7 +110,7 @@ static inline void vtime_reset_last_update(struct lowcore *lc) * Update process times based on virtual cpu times stored by entry.S * to the lowcore fields user_timer, system_timer & steal_clock. */ -static int do_account_vtime(struct task_struct *tsk) +static void do_account_vtime(struct task_struct *tsk) { u64 timer, clock, user, guest, system, hardirq, softirq; struct lowcore *lc = get_lowcore(); @@ -172,8 +157,6 @@ static int do_account_vtime(struct task_struct *tsk) account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ); if (softirq) account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ); - - return virt_timer_forward(user + guest + system + hardirq + softirq); } void vtime_task_switch(struct task_struct *prev) @@ -203,8 +186,7 @@ void vtime_flush(struct task_struct *tsk) struct lowcore *lc = get_lowcore(); u64 steal, avg_steal; - if (do_account_vtime(tsk)) - virt_timer_expire(); + do_account_vtime(tsk); steal = lc->steal_timer; avg_steal = lc->avg_steal_timer; @@ -247,187 +229,9 @@ void vtime_account_hardirq(struct task_struct *tsk) get_lowcore()->hardirq_timer += vtime_delta(); } -/* - * Sorted add to a list. List is linear searched until first bigger - * element is found. - */ -static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) -{ - struct vtimer_list *tmp; - - list_for_each_entry(tmp, head, entry) { - if (tmp->expires > timer->expires) { - list_add_tail(&timer->entry, &tmp->entry); - return; - } - } - list_add_tail(&timer->entry, head); -} - -/* - * Handler for expired virtual CPU timer. - */ -static void virt_timer_expire(void) -{ - struct vtimer_list *timer, *tmp; - unsigned long elapsed; - LIST_HEAD(cb_list); - - /* walk timer list, fire all expired timers */ - spin_lock(&virt_timer_lock); - elapsed = atomic64_read(&virt_timer_elapsed); - list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) { - if (timer->expires < elapsed) - /* move expired timer to the callback queue */ - list_move_tail(&timer->entry, &cb_list); - else - timer->expires -= elapsed; - } - if (!list_empty(&virt_timer_list)) { - timer = list_first_entry(&virt_timer_list, - struct vtimer_list, entry); - atomic64_set(&virt_timer_current, timer->expires); - } - atomic64_sub(elapsed, &virt_timer_elapsed); - spin_unlock(&virt_timer_lock); - - /* Do callbacks and recharge periodic timers */ - list_for_each_entry_safe(timer, tmp, &cb_list, entry) { - list_del_init(&timer->entry); - timer->function(timer->data); - if (timer->interval) { - /* Recharge interval timer */ - timer->expires = timer->interval + - atomic64_read(&virt_timer_elapsed); - spin_lock(&virt_timer_lock); - list_add_sorted(timer, &virt_timer_list); - spin_unlock(&virt_timer_lock); - } - } -} - -void init_virt_timer(struct vtimer_list *timer) -{ - timer->function = NULL; - INIT_LIST_HEAD(&timer->entry); -} -EXPORT_SYMBOL(init_virt_timer); - -static inline int vtimer_pending(struct vtimer_list *timer) -{ - return !list_empty(&timer->entry); -} - -static void internal_add_vtimer(struct vtimer_list *timer) -{ - if (list_empty(&virt_timer_list)) { - /* First timer, just program it. */ - atomic64_set(&virt_timer_current, timer->expires); - atomic64_set(&virt_timer_elapsed, 0); - list_add(&timer->entry, &virt_timer_list); - } else { - /* Update timer against current base. */ - timer->expires += atomic64_read(&virt_timer_elapsed); - if (likely((s64) timer->expires < - (s64) atomic64_read(&virt_timer_current))) - /* The new timer expires before the current timer. */ - atomic64_set(&virt_timer_current, timer->expires); - /* Insert new timer into the list. */ - list_add_sorted(timer, &virt_timer_list); - } -} - -static void __add_vtimer(struct vtimer_list *timer, int periodic) -{ - unsigned long flags; - - timer->interval = periodic ? timer->expires : 0; - spin_lock_irqsave(&virt_timer_lock, flags); - internal_add_vtimer(timer); - spin_unlock_irqrestore(&virt_timer_lock, flags); -} - -/* - * add_virt_timer - add a oneshot virtual CPU timer - */ -void add_virt_timer(struct vtimer_list *timer) -{ - __add_vtimer(timer, 0); -} -EXPORT_SYMBOL(add_virt_timer); - -/* - * add_virt_timer_int - add an interval virtual CPU timer - */ -void add_virt_timer_periodic(struct vtimer_list *timer) -{ - __add_vtimer(timer, 1); -} -EXPORT_SYMBOL(add_virt_timer_periodic); - -static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic) -{ - unsigned long flags; - int rc; - - BUG_ON(!timer->function); - - if (timer->expires == expires && vtimer_pending(timer)) - return 1; - spin_lock_irqsave(&virt_timer_lock, flags); - rc = vtimer_pending(timer); - if (rc) - list_del_init(&timer->entry); - timer->interval = periodic ? expires : 0; - timer->expires = expires; - internal_add_vtimer(timer); - spin_unlock_irqrestore(&virt_timer_lock, flags); - return rc; -} - -/* - * returns whether it has modified a pending timer (1) or not (0) - */ -int mod_virt_timer(struct vtimer_list *timer, u64 expires) -{ - return __mod_vtimer(timer, expires, 0); -} -EXPORT_SYMBOL(mod_virt_timer); - -/* - * returns whether it has modified a pending timer (1) or not (0) - */ -int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires) -{ - return __mod_vtimer(timer, expires, 1); -} -EXPORT_SYMBOL(mod_virt_timer_periodic); - -/* - * Delete a virtual timer. - * - * returns whether the deleted timer was pending (1) or not (0) - */ -int del_virt_timer(struct vtimer_list *timer) -{ - unsigned long flags; - - if (!vtimer_pending(timer)) - return 0; - spin_lock_irqsave(&virt_timer_lock, flags); - list_del_init(&timer->entry); - spin_unlock_irqrestore(&virt_timer_lock, flags); - return 1; -} -EXPORT_SYMBOL(del_virt_timer); - -/* - * Start the virtual CPU timer on the current CPU. - */ void vtime_init(void) { - /* set initial cpu timer */ - set_vtimer(VTIMER_MAX_SLICE); + cpu_timer_init(CPU_TIMER_MAX); /* Setup initial MT scaling values */ if (smp_cpu_mtid) { __this_cpu_write(mt_scaling_jiffies, jiffies); From 7b88e8b6d714cd1b2b6185e2963c8ec4d98542d0 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 6 Oct 2026 17:46:02 +0200 Subject: [PATCH 041/102] s390/zcrypt: Guard domain index uses against speculative bypass The previous array_index_nospec() placement only covered the admin permission check, leaving ap_test_config_usage_domain() and ap_test_config_ctrl_domain() in the CCA path exposed to speculative execution. Move the sanitization to before all permission and config checks in both the CCA and EP11 paths. Hold the sanitized domain value in another variable so that the later use can choose to use either the original value (subject to speculative execution) or the sanitized value (which may in case of AUTOSEL_DOM reflect the wrong value). Before that, check the domain value to be either AUTOSEL_DOM or in range 0...AP_DOMAINS-1 and return with -EINVAL in case this check does not pass. Fixes: e935cd525af4 ("s390/zcrypt: Close speculative mem read possibility") Signed-off-by: Harald Freudenberger Cc: stable@vger.kernel.org Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/zcrypt_api.c | 35 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index ec6a4c2f9f04..80435fd8e6ad 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -854,7 +854,7 @@ static long _zcrypt_send_cprb(u32 xflags, struct ap_perms *perms, struct ica_xcRB *xcrb) { bool userspace = xflags & ZCRYPT_XFLAG_USERSPACE; - unsigned int card, domain, func_code = 0; + unsigned int card, domain, _dom, func_code = 0; unsigned int wgt = 0, pref_wgt = 0; struct zcrypt_queue *zq, *pref_zq; struct zcrypt_card *zc, *pref_zc; @@ -877,10 +877,17 @@ static long _zcrypt_send_cprb(u32 xflags, struct ap_perms *perms, print_hex_dump_debug("ccareq: ", DUMP_PREFIX_ADDRESS, 16, 1, ap_msg.msg, ap_msg.len, false); + /* Check domain for either AUTOSEL_DOM or in range 0...AP_DOMAINS-1 */ + if (domain != AUTOSEL_DOM && domain >= AP_DOMAINS) { + rc = -EINVAL; + goto out; + } + /* Spectre-v1: sanitize domain unconditionally for later use */ + _dom = array_index_nospec(domain, AP_DOMAINS); + if (perms != &ap_perms && domain < AP_DOMAINS) { if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { - domain = array_index_nospec(domain, AP_DOMAINS); - if (!test_bit_inv(domain, perms->adm)) { + if (!test_bit_inv(_dom, perms->adm)) { rc = -ENODEV; goto out; } @@ -893,10 +900,11 @@ static long _zcrypt_send_cprb(u32 xflags, struct ap_perms *perms, * If a valid target domain is set and this domain is NOT a usage * domain but a control only domain, autoselect target domain. */ - if (domain < AP_DOMAINS && - !ap_test_config_usage_domain(domain) && - ap_test_config_ctrl_domain(domain)) - domain = AUTOSEL_DOM; + if (domain < AP_DOMAINS) { + if (!ap_test_config_usage_domain(_dom) && + ap_test_config_ctrl_domain(_dom)) + domain = AUTOSEL_DOM; + } pref_zc = NULL; pref_zq = NULL; @@ -1041,7 +1049,7 @@ static long _zcrypt_send_ep11_cprb(u32 xflags, struct ap_perms *perms, struct ep11_target_dev *targets = NULL; unsigned short target_num; unsigned int wgt = 0, pref_wgt = 0; - unsigned int func_code = 0, domain; + unsigned int func_code = 0, domain, _dom; struct ap_message ap_msg; int cpen, qpen, qid = 0, rc; struct module *mod; @@ -1078,10 +1086,17 @@ static long _zcrypt_send_ep11_cprb(u32 xflags, struct ap_perms *perms, print_hex_dump_debug("ep11req: ", DUMP_PREFIX_ADDRESS, 16, 1, ap_msg.msg, ap_msg.len, false); + /* Check domain for either AUTOSEL_DOM or in range 0...AP_DOMAINS-1 */ + if (domain != AUTOSEL_DOM && domain >= AP_DOMAINS) { + rc = -EINVAL; + goto out; + } + /* Spectre-v1: sanitize domain unconditionally for later use */ + _dom = array_index_nospec(domain, AP_DOMAINS); + if (perms != &ap_perms && domain < AP_DOMAINS) { if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { - domain = array_index_nospec(domain, AP_DOMAINS); - if (!test_bit_inv(domain, perms->adm)) { + if (!test_bit_inv(_dom, perms->adm)) { rc = -ENODEV; goto out; } From 9fd7fca66735307fd7afa3450882a55ef80f2add Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 6 Oct 2026 17:46:03 +0200 Subject: [PATCH 042/102] s390/zcrypt: Fix out-of-bounds ptr advance in cca_query_crypto_facility() The FQ reply parser code blindly advanced the walk pointer by a length value read directly from the hardware reply payload without checking that the advance stayed within the allocated reply buffer. A corrupt or malicious device response could push ptr beyond the cprbmem region, causing an out-of-bounds dereference or kernel memory exposure via the subsequent memcpy(). Fix this by tracking the remaining reply buffer space in a variable and validating each device-supplied length field against it before advancing or dereferencing the pointer. Fixes: 2004b57cde6b ("s390/zcrypt: code cleanup") Signed-off-by: Harald Freudenberger Cc: stable@vger.kernel.org Reviewed-by: Holger Dengler Signed-off-by: Heiko Carstens --- drivers/s390/crypto/zcrypt_ccamisc.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c index 19909bf43dc9..862947431164 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.c +++ b/drivers/s390/crypto/zcrypt_ccamisc.c @@ -1627,6 +1627,7 @@ int cca_query_crypto_facility(u16 cardnr, u16 domain, u8 subfunc_code[2]; u8 lvdata[]; } __packed * prepparm; + size_t datalen; /* get already prepared memory for 2 cprbs with param block each */ rc = alloc_and_prep_cprbmem(parmbsize, &mem, @@ -1673,27 +1674,44 @@ int cca_query_crypto_facility(u16 cardnr, u16 domain, prepcblk->rpl_parmb = (u8 __user *)ptr; prepparm = (struct fqrepparm *)ptr; ptr = prepparm->lvdata; + datalen = parmbsize - 2 * sizeof(u8); /* check and possibly copy reply rule array */ len = *((u16 *)ptr); + if (len > datalen) { + ZCRYPT_DBF_ERR("%s reply rule array len %u exceeds datalen %zu\n", + __func__, len, datalen); + rc = -EIO; + goto out; + } + datalen -= sizeof(u16); + ptr += sizeof(u16); if (len > sizeof(u16)) { - ptr += sizeof(u16); len -= sizeof(u16); if (rarray && rarraylen && *rarraylen > 0) { *rarraylen = (len > *rarraylen ? *rarraylen : len); memcpy(rarray, ptr, *rarraylen); } + datalen -= len; ptr += len; } /* check and possible copy reply var array */ len = *((u16 *)ptr); + if (len > datalen) { + ZCRYPT_DBF_ERR("%s reply var array len %u exceeds datalen %zu\n", + __func__, len, datalen); + rc = -EIO; + goto out; + } + datalen -= sizeof(u16); + ptr += sizeof(u16); if (len > sizeof(u16)) { - ptr += sizeof(u16); len -= sizeof(u16); if (varray && varraylen && *varraylen > 0) { *varraylen = (len > *varraylen ? *varraylen : len); memcpy(varray, ptr, *varraylen); } + datalen -= len; ptr += len; } From 946b9dcff33bd4669464674ae1c00683b875be55 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 9 Oct 2026 17:21:31 +0200 Subject: [PATCH 043/102] s390/ptrace: Preserve ILC when updating system call number When an attached debugger changes gpr2 via PTRACE_POKEUSR while the traced process has PIF_SYSCALL set, the int_code in pt_regs is also updated. In case of an svc it contains the instruction length code (ILC), as well as the system call number. The ptrace code sets the ILC unconditionally to a value which matches the svc instruction. However the svc instruction also may have been executed via the execute or execute relative long instruction, which have a different length. Therefore preserve the ILC and only change the system call number using syscall_set_nr(). This makes sure that in case of a PSW rewind the instruction address points to the correct instruction. Fixes: 56e62a737028 ("s390: convert to generic entry") Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/kernel/ptrace.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 125ca4c4e30c..d8d5e843859c 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "entry.h" @@ -345,11 +346,9 @@ static int __poke_user(struct task_struct *child, addr_t addr, addr_t data) return -EINVAL; } - if (test_pt_regs_flag(regs, PIF_SYSCALL) && - addr == offsetof(struct user, regs.gprs[2])) { - struct pt_regs *regs = task_pt_regs(child); - - regs->int_code = 0x20000 | (data & 0xffff); + if (addr == offsetof(struct user, regs.gprs[2])) { + if (test_pt_regs_flag(regs, PIF_SYSCALL)) + syscall_set_nr(child, regs, data); } *(addr_t *)((addr_t) ®s->psw + addr) = data; } else if (addr < offsetof(struct user, regs.orig_gpr2)) { From 4044466ff004c7ad5fd47be717cd933077e6e874 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 2 Oct 2026 12:27:50 +0200 Subject: [PATCH 044/102] s390/extable: Fix mvcos instruction decoding If either the b1 or b2 field of the mvcos instruction is zero, register zero is not used for address computation; zero is used instead, according to the architecture. ex_handler_ua_mvcos() incorrectly uses the real contents of register zero. Fix this. This shouldn't be a problem for any generated code, since the compiler would only generate such code if it is possible to prove at compile time that either the source or destination address is within the first 4kb of the designated address space. Such code should not exist (also confirmed by an allyesconfig build and decoding all ~7000 mvcos instructions). Fixes: c488f5187a24 ("s390/uaccess: Shorten raw_copy_from_user() / raw_copy_to_user() inline assemblies") Cc: stable@vger.kernel.org Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/mm/extable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index 7498e858c401..848855da9a70 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -106,9 +106,9 @@ static bool ex_handler_ua_mvcos(const struct exception_table_entry *ex, regs->psw.addr = extable_fixup(ex); insn = (struct insn_ssf *)regs->psw.addr; if (from) - uaddr = regs->gprs[insn->b2] + insn->d2; + uaddr = (insn->b2 ? regs->gprs[insn->b2] : 0) + insn->d2; else - uaddr = regs->gprs[insn->b1] + insn->d1; + uaddr = (insn->b1 ? regs->gprs[insn->b1] : 0) + insn->d1; remainder = PAGE_SIZE - (uaddr & (PAGE_SIZE - 1)); if (regs->gprs[insn->r3] <= remainder) remainder = 0; From 2ba01cf7b2598571486425a0605f1434909b532d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 9 Oct 2026 15:26:04 +0200 Subject: [PATCH 045/102] s390/uprobes: Fix off-by-one bug in emulated PER handling The uprobes code emulates some instructions, and with that also PER storage-alteration handling, in case a debugger is attached. The PER storage-alteration handling contains an off-by-one bug, which may lead to PER traps which should not happen. Adjust the comparison in sim_stor_event() to fix that. Fixes: 8d1a2427d8fc ("s390/uprobes: fix user space PER events") Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/kernel/uprobes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/uprobes.c b/arch/s390/kernel/uprobes.c index c624f3361e43..b2b8a377ced6 100644 --- a/arch/s390/kernel/uprobes.c +++ b/arch/s390/kernel/uprobes.c @@ -241,7 +241,7 @@ static void sim_stor_event(struct pt_regs *regs, void *addr, int len) return; if (!(current->thread.per_user.control & PER_EVENT_STORE)) return; - if ((void *)current->thread.per_user.start > (addr + len)) + if ((void *)current->thread.per_user.start > (addr + len - 1)) return; if ((void *)current->thread.per_user.end < addr) return; From 31c66bd273303590fe69fad370e72a12faa0bf76 Mon Sep 17 00:00:00 2001 From: Tobias Schumacher Date: Fri, 9 Oct 2026 08:15:48 +0200 Subject: [PATCH 046/102] s390/pci: Fix double-free and NULL deref in zpci MSI domain cleanup zpci_remove_parent_msi_domain() dereferences zbus->msi_parent_domain unconditionally and does not clear it afterwards. zpci_bus_create_pci_bus() removes the domain when pci_create_root_bus() fails, then zpci_bus_release() removes it again on the last kref_put(), reading ->fwnode from the freed irq_domain and freeing it twice. If zpci_alloc_domain() or zpci_create_parent_msi_domain() fails, no domain is created at all; zbus is kzalloc'd, so the same release path dereferences NULL. Return early when there is no domain, and clear the pointer after removing one. Fixes: f770950a4709 ("s390/pci: Migrate s390 IRQ logic to IRQ domain API") Cc: stable@vger.kernel.org Reviewed-by: Niklas Schnelle Signed-off-by: Tobias Schumacher Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_irq.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index 9c9ed3d8d959..c9520a16ca75 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -533,9 +533,13 @@ void zpci_remove_parent_msi_domain(struct zpci_bus *zbus) { struct fwnode_handle *fn; + if (!zbus->msi_parent_domain) + return; + fn = zbus->msi_parent_domain->fwnode; irq_domain_remove(zbus->msi_parent_domain); irq_domain_free_fwnode(fn); + zbus->msi_parent_domain = NULL; } static void __init cpu_enable_directed_irq(void *unused) From 4a185d99179a307d856dc43610cfe72395d0c42a Mon Sep 17 00:00:00 2001 From: Tobias Schumacher Date: Fri, 9 Oct 2026 08:15:49 +0200 Subject: [PATCH 047/102] s390/pci: Fix resource leak in zpci MSI setup If airq_iv_create() fails in __alloc_airq(), the zpci_sbv bit allocated by airq_iv_alloc_bit() is never freed. This permanently leaks one of the ZPCI_NR_DEVICES summary bits, reducing system capacity with each failed device hotplug. In systems with repeated device insertion failures or under memory pressure, all summary bits can be exhausted, preventing new PCI devices from being added until reboot. Add proper error handling to free the zpci_sbv bit and reset zdev->aisb if the AIBV creation fails. Fixes: f770950a4709 ("s390/pci: Migrate s390 IRQ logic to IRQ domain API") Cc: stable@vger.kernel.org Reviewed-by: Niklas Schnelle Signed-off-by: Tobias Schumacher Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_irq.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index c9520a16ca75..134f8f4a5cfa 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -313,8 +313,11 @@ static int __alloc_airq(struct zpci_dev *zdev, int msi_vecs, zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_PTR | AIRQ_IV_DATA | AIRQ_IV_BITLOCK, NULL); - if (!zdev->aibv) + if (!zdev->aibv) { + airq_iv_free_bit(zpci_sbv, *bit); + zdev->aisb = -1UL; return -ENOMEM; + } /* Wire up shortcut pointer */ zpci_ibv[*bit] = zdev->aibv; From 1844913582fec337c54b54fdd3c92ce225930515 Mon Sep 17 00:00:00 2001 From: Tobias Schumacher Date: Fri, 9 Oct 2026 08:15:50 +0200 Subject: [PATCH 048/102] s390/pci: Fix MSI directed-mode teardown IRQ bit count On s390 with directed interrupts enabled, zpci_msi_teardown_directed() frees the platform's maximum number of MSI bits (zdev->max_msi) instead of the actual allocated count (zdev->msi_nr_irqs). This corrupts the shared IRQ bitmap used by all PCI functions, causing lost interrupts and heap corruption. Fix zpci_msi_teardown_directed() to only free the actual allocated IRQ bit count. Fixes: f770950a4709 ("s390/pci: Migrate s390 IRQ logic to IRQ domain API") Cc: stable@vger.kernel.org Reviewed-by: Niklas Schnelle Signed-off-by: Tobias Schumacher Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_irq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index 134f8f4a5cfa..d5763c5feb09 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -342,7 +342,7 @@ static struct airq_struct zpci_airq = { static void zpci_msi_teardown_directed(struct zpci_dev *zdev) { - airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->max_msi); + airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs); zdev->msi_first_bit = -1U; zdev->msi_nr_irqs = 0; } From 422d75ad5bc96b8eb9cb3cfd9d68fc685a0a9b75 Mon Sep 17 00:00:00 2001 From: Tobias Schumacher Date: Fri, 9 Oct 2026 08:15:51 +0200 Subject: [PATCH 049/102] s390/pci: Fix use-after-free race in zpci floating interrupt cleanup zpci_clear_irq() stops the adapter from raising new interrupts for the function, but a zpci_floating_irq_handler() already running on another CPU can still be scanning zdev->aibv when zpci_msi_teardown_floating() releases it. Clear the zpci_ibv[] entry so no further handler picks the vector up, then wait for a grace period before releasing it. The handler runs inside the rcu_read_lock() section that do_airq_interrupt() holds across airq->handler(), so synchronize_rcu() drains any handler still in flight. Release the vector and free the summary bit only after the grace period, so neither can be reused while a reader still holds the old pointer. zpci_ibv served both delivery modes, indexed by function under FLOATING and by cpu under DIRECTED. Only the floating vectors are published to and torn down under the interrupt handler, so split the directed vectors out into zpci_dibv and annotate zpci_ibv __rcu, which lets sparse check the accessors above. Fixes: f770950a4709 ("s390/pci: Migrate s390 IRQ logic to IRQ domain API") Cc: stable@vger.kernel.org Signed-off-by: Tobias Schumacher Reviewed-by: Niklas Schnelle Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_irq.c | 58 +++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index d5763c5feb09..c5fabad38139 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -22,12 +22,11 @@ static enum {FLOATING, DIRECTED} irq_delivery; */ static struct airq_iv *zpci_sbv; -/* - * interrupt bit vectors - * FLOATING - interrupt bit vector per function - * DIRECTED - interrupt bit vector per cpu - */ -static struct airq_iv **zpci_ibv; +/* FLOATING - interrupt bit vector per function */ +static struct airq_iv __rcu **zpci_ibv; + +/* DIRECTED - interrupt bit vector per cpu */ +static struct airq_iv **zpci_dibv; /* Modify PCI: Register floating adapter interruptions */ static int zpci_set_airq(struct zpci_dev *zdev) @@ -169,7 +168,7 @@ static struct irq_chip zpci_irq_chip = { static void zpci_handle_cpu_local_irq(bool rescan) { - struct airq_iv *dibv = zpci_ibv[smp_processor_id()]; + struct airq_iv *dibv = zpci_dibv[smp_processor_id()]; union zpci_sic_iib iib = {{0}}; struct irq_domain *msi_domain; irq_hw_number_t hwirq; @@ -279,7 +278,9 @@ static void zpci_floating_irq_handler(struct airq_struct *airq, } /* Scan the adapter interrupt vector for this device. */ - aibv = zpci_ibv[si]; + aibv = rcu_dereference(zpci_ibv[si]); + if (!aibv) + continue; for (ai = 0;;) { ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv)); if (ai == -1UL) @@ -299,7 +300,7 @@ static int __alloc_airq(struct zpci_dev *zdev, int msi_vecs, { if (irq_delivery == DIRECTED) { /* Allocate cpu vector bits */ - *bit = airq_iv_alloc(zpci_ibv[0], msi_vecs); + *bit = airq_iv_alloc(zpci_dibv[0], msi_vecs); if (*bit == -1UL) return -EIO; } else { @@ -320,7 +321,7 @@ static int __alloc_airq(struct zpci_dev *zdev, int msi_vecs, } /* Wire up shortcut pointer */ - zpci_ibv[*bit] = zdev->aibv; + rcu_assign_pointer(zpci_ibv[*bit], zdev->aibv); /* Each function has its own interrupt vector */ *bit = 0; } @@ -342,13 +343,16 @@ static struct airq_struct zpci_airq = { static void zpci_msi_teardown_directed(struct zpci_dev *zdev) { - airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs); + airq_iv_free(zpci_dibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs); zdev->msi_first_bit = -1U; zdev->msi_nr_irqs = 0; } static void zpci_msi_teardown_floating(struct zpci_dev *zdev) { + rcu_assign_pointer(zpci_ibv[zdev->aisb], NULL); + synchronize_rcu(); + airq_iv_release(zdev->aibv); zdev->aibv = NULL; airq_iv_free_bit(zpci_sbv, zdev->aisb); @@ -428,9 +432,9 @@ static int zpci_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, if (irq_delivery == DIRECTED) { for_each_possible_cpu(cpu) { - airq_iv_set_ptr(zpci_ibv[cpu], bit + i, + airq_iv_set_ptr(zpci_dibv[cpu], bit + i, (unsigned long)zbus->msi_parent_domain); - airq_iv_set_data(zpci_ibv[cpu], bit + i, hwirq + i); + airq_iv_set_data(zpci_dibv[cpu], bit + i, hwirq + i); } } else { airq_iv_set_ptr(zdev->aibv, bit + i, @@ -455,8 +459,8 @@ static void zpci_msi_clear_airq(struct irq_data *d, int i) if (irq_delivery == DIRECTED) { for_each_possible_cpu(cpu) { - airq_iv_set_ptr(zpci_ibv[cpu], bit + i, 0); - airq_iv_set_data(zpci_ibv[cpu], bit + i, 0); + airq_iv_set_ptr(zpci_dibv[cpu], bit + i, 0); + airq_iv_set_data(zpci_dibv[cpu], bit + i, 0); } } else { airq_iv_set_ptr(zdev->aibv, bit + i, 0); @@ -550,7 +554,7 @@ static void __init cpu_enable_directed_irq(void *unused) union zpci_sic_iib iib = {{0}}; union zpci_sic_iib ziib = {{0}}; - iib.cdiib.dibv_addr = virt_to_phys(zpci_ibv[smp_processor_id()]->vector); + iib.cdiib.dibv_addr = virt_to_phys(zpci_dibv[smp_processor_id()]->vector); zpci_set_irq_ctrl(SIC_IRQ_MODE_SET_CPU, 0, &iib); zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC, &ziib); @@ -570,8 +574,8 @@ static int __init zpci_directed_irq_init(void) iib.diib.disb_addr = virt_to_phys(zpci_sbv->vector); zpci_set_irq_ctrl(SIC_IRQ_MODE_DIRECT, 0, &iib); - zpci_ibv = kzalloc_objs(*zpci_ibv, num_possible_cpus()); - if (!zpci_ibv) + zpci_dibv = kzalloc_objs(*zpci_dibv, num_possible_cpus()); + if (!zpci_dibv) return -ENOMEM; for_each_possible_cpu(cpu) { @@ -579,12 +583,12 @@ static int __init zpci_directed_irq_init(void) * Per CPU IRQ vectors look the same but bit-allocation * is only done on the first vector. */ - zpci_ibv[cpu] = airq_iv_create(cache_line_size() * BITS_PER_BYTE, - AIRQ_IV_PTR | - AIRQ_IV_DATA | - AIRQ_IV_CACHELINE | - (!cpu ? AIRQ_IV_ALLOC : 0), NULL); - if (!zpci_ibv[cpu]) + zpci_dibv[cpu] = airq_iv_create(cache_line_size() * BITS_PER_BYTE, + AIRQ_IV_PTR | + AIRQ_IV_DATA | + AIRQ_IV_CACHELINE | + (!cpu ? AIRQ_IV_ALLOC : 0), NULL); + if (!zpci_dibv[cpu]) return -ENOMEM; } on_each_cpu(cpu_enable_directed_irq, NULL, 1); @@ -660,10 +664,12 @@ void __init zpci_irq_exit(void) if (irq_delivery == DIRECTED) { for_each_possible_cpu(cpu) { - airq_iv_release(zpci_ibv[cpu]); + airq_iv_release(zpci_dibv[cpu]); } + kfree(zpci_dibv); + } else { + kfree(zpci_ibv); } - kfree(zpci_ibv); if (zpci_sbv) airq_iv_release(zpci_sbv); unregister_adapter_interrupt(&zpci_airq); From 769f66f595c11185c66913ea5d96420504bfebc3 Mon Sep 17 00:00:00 2001 From: Tobias Schumacher Date: Fri, 9 Oct 2026 08:15:52 +0200 Subject: [PATCH 050/102] s390/pci: Add error cleanup in zpci_directed_irq_init() If per-CPU airq_iv allocation fails in the loop, previously allocated vectors and arrays leak. Add proper error path to release all resources on failure. The error path frees zpci_sbv, but SIC_IRQ_MODE_DIRECT has already programmed its address into the hardware. Set the directed interrupt mode only after all allocations have succeeded, so a failed init leaves no freed memory registered with the hardware. Fixes: e979ce7bced2 ("s390/pci: provide support for CPU directed interrupts") Cc: stable@vger.kernel.org Signed-off-by: Tobias Schumacher Reviewed-by: Niklas Schnelle Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_irq.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index c5fabad38139..bcf11585df8d 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -569,14 +569,9 @@ static int __init zpci_directed_irq_init(void) if (!zpci_sbv) return -ENOMEM; - iib.diib.isc = PCI_ISC; - iib.diib.nr_cpus = num_possible_cpus(); - iib.diib.disb_addr = virt_to_phys(zpci_sbv->vector); - zpci_set_irq_ctrl(SIC_IRQ_MODE_DIRECT, 0, &iib); - zpci_dibv = kzalloc_objs(*zpci_dibv, num_possible_cpus()); if (!zpci_dibv) - return -ENOMEM; + goto out_free_sbv; for_each_possible_cpu(cpu) { /* @@ -589,13 +584,30 @@ static int __init zpci_directed_irq_init(void) AIRQ_IV_CACHELINE | (!cpu ? AIRQ_IV_ALLOC : 0), NULL); if (!zpci_dibv[cpu]) - return -ENOMEM; + goto out_free_dibv; } + + iib.diib.isc = PCI_ISC; + iib.diib.nr_cpus = num_possible_cpus(); + iib.diib.disb_addr = virt_to_phys(zpci_sbv->vector); + zpci_set_irq_ctrl(SIC_IRQ_MODE_DIRECT, 0, &iib); on_each_cpu(cpu_enable_directed_irq, NULL, 1); zpci_irq_chip.irq_set_affinity = zpci_set_irq_affinity; return 0; + +out_free_dibv: + for_each_possible_cpu(cpu) { + if (zpci_dibv[cpu]) + airq_iv_release(zpci_dibv[cpu]); + } + kfree(zpci_dibv); + zpci_dibv = NULL; +out_free_sbv: + airq_iv_release(zpci_sbv); + zpci_sbv = NULL; + return -ENOMEM; } static int __init zpci_floating_irq_init(void) From 343d1acd1abd0d9320f8b4252c736246ba764d4e Mon Sep 17 00:00:00 2001 From: Tobias Schumacher Date: Fri, 9 Oct 2026 08:15:53 +0200 Subject: [PATCH 051/102] s390/pci: Set MSI_FLAG_NO_AFFINITY at IRQ init time MSI_FLAG_NO_AFFINITY is added to zpci_msi_parent_ops.required_flags from zpci_create_parent_msi_domain(), which runs for every new PCI bus, including buses created at runtime from a hotplug availability event. That is a non-atomic read-modify-write on a field which msi_lib_init_dev_msi_info() reads without a common lock while setting up MSI for a device on an already existing bus: required_flags = pops->required_flags; The stored value is always the same, so no caller observes a change, but the race need not exist: irq_delivery is decided once in zpci_irq_init() and never changes afterwards. Set the flag there instead, before any parent domain exists. Reviewed-by: Niklas Schnelle Signed-off-by: Tobias Schumacher Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_irq.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index bcf11585df8d..b890b55be873 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -523,9 +523,6 @@ int zpci_create_parent_msi_domain(struct zpci_bus *zbus) return -ENOMEM; } - if (irq_delivery == FLOATING) - zpci_msi_parent_ops.required_flags |= MSI_FLAG_NO_AFFINITY; - zbus->msi_parent_domain = msi_create_parent_irq_domain(&info, &zpci_msi_parent_ops); if (!zbus->msi_parent_domain) { irq_domain_free_fwnode(info.fwnode); @@ -636,6 +633,9 @@ int __init zpci_irq_init(void) if (s390_pci_force_floating) irq_delivery = FLOATING; + if (irq_delivery == FLOATING) + zpci_msi_parent_ops.required_flags |= MSI_FLAG_NO_AFFINITY; + if (irq_delivery == DIRECTED) zpci_airq.handler = zpci_directed_irq_handler; From 9126b292d5e941c8bb192f4cd6f13980b624cd28 Mon Sep 17 00:00:00 2001 From: Tobias Schumacher Date: Fri, 9 Oct 2026 08:15:54 +0200 Subject: [PATCH 052/102] s390/pci: Drop the unused index argument of zpci_msi_clear_airq() zpci_msi_domain_free() passes its loop index to zpci_msi_clear_airq(), which adds it to an offset that already accounts for it. zpci_msi_domain_alloc() stores hwirq + i for each vector, so zpci_decode_hwirq_msi_index() hands back msi_index + i and bit is already zdev->msi_first_bit + msi_index + i. The doubled index never selected a wrong entry. An irq domain's free() callback is only ever invoked from irq_domain_free_irqs_hierarchy(), which walks the range itself and passes a count of one. So, the loop in zpci_msi_domain_free() runs once with an index of zero. Drop the parameter and the addition. No functional change. Reviewed-by: Niklas Schnelle Signed-off-by: Tobias Schumacher Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_irq.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index b890b55be873..4b8ebb987580 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -446,7 +446,7 @@ static int zpci_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, return 0; } -static void zpci_msi_clear_airq(struct irq_data *d, int i) +static void zpci_msi_clear_airq(struct irq_data *d) { struct msi_desc *desc = irq_data_get_msi_desc(d); struct zpci_dev *zdev = to_zpci_dev(desc->dev); @@ -459,12 +459,12 @@ static void zpci_msi_clear_airq(struct irq_data *d, int i) if (irq_delivery == DIRECTED) { for_each_possible_cpu(cpu) { - airq_iv_set_ptr(zpci_dibv[cpu], bit + i, 0); - airq_iv_set_data(zpci_dibv[cpu], bit + i, 0); + airq_iv_set_ptr(zpci_dibv[cpu], bit, 0); + airq_iv_set_data(zpci_dibv[cpu], bit, 0); } } else { - airq_iv_set_ptr(zdev->aibv, bit + i, 0); - airq_iv_set_data(zdev->aibv, bit + i, 0); + airq_iv_set_ptr(zdev->aibv, bit, 0); + airq_iv_set_data(zdev->aibv, bit, 0); } } @@ -476,7 +476,7 @@ static void zpci_msi_domain_free(struct irq_domain *domain, unsigned int virq, for (i = 0; i < nr_irqs; i++) { d = irq_domain_get_irq_data(domain, virq + i); - zpci_msi_clear_airq(d, i); + zpci_msi_clear_airq(d); irq_domain_reset_irq_data(d); } } From 823e889478e423c18c2b35efa68234268b2338cb Mon Sep 17 00:00:00 2001 From: Tobias Schumacher Date: Fri, 9 Oct 2026 08:15:55 +0200 Subject: [PATCH 053/102] s390/pci: Unregister the adapter interrupt first in zpci_irq_exit() zpci_irq_exit() frees the interrupt bit vectors before unregistering the adapter interrupt whose handler uses them. This is only called from the pci_base_init() error path, where no handler is expected to run, but unregister first anyway so the vectors are never freed under a live handler. Signed-off-by: Tobias Schumacher Reviewed-by: Niklas Schnelle Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_irq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c index 4b8ebb987580..16ae80ed2d13 100644 --- a/arch/s390/pci/pci_irq.c +++ b/arch/s390/pci/pci_irq.c @@ -674,6 +674,7 @@ void __init zpci_irq_exit(void) { unsigned int cpu; + unregister_adapter_interrupt(&zpci_airq); if (irq_delivery == DIRECTED) { for_each_possible_cpu(cpu) { airq_iv_release(zpci_dibv[cpu]); @@ -684,5 +685,4 @@ void __init zpci_irq_exit(void) } if (zpci_sbv) airq_iv_release(zpci_sbv); - unregister_adapter_interrupt(&zpci_airq); } From fc9bb0ecd310b53af5116415952a1fb89d4660b9 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:16 +0200 Subject: [PATCH 054/102] s390/kprobes: Prevent kprobes on instructions with exception table entry The mvcos exception handler ex_handler_ua_mvcos() decodes the faulting instruction, assuming it is an mvcos instruction. In case the instruction is kprobed the decoded instruction is a breakpoint instruction instead, which leads to incorrect instruction decoding and potential register corruption. Fix this by simply preventing to set a kprobe on such instructions, similar like arm64 is doing it. Fixes: c488f5187a24 ("s390/uaccess: Shorten raw_copy_from_user() / raw_copy_to_user() inline assemblies") Cc: stable@vger.kernel.org Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/kernel/kprobes.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index c450120b4474..e806b124e4ba 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -85,6 +85,9 @@ static bool can_probe(unsigned long paddr) if (!kallsyms_lookup_size_offset(paddr, NULL, &offset)) return false; + if (s390_search_extables(paddr)) + return false; + /* Decode instructions */ addr = paddr - offset; while (addr < paddr) { From 646dc01d634bc182529d7cd5f5346869aa7d58ee Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:17 +0200 Subject: [PATCH 055/102] s390/early: Fix early warning handling When adding support for __WARN_printf() the early program check handler was not taken into account. Add the missing support to the early program check handler. This avoids that the kernel crashes instead of printing a warning. Fixes: 04dabb4261c3 ("s390/bug: Implement __WARN_printf()") Fixes: 79996065cfa2 ("s390/bug: Skip __WARN_trap() in call traces") Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/kernel/early.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index b27239c03d79..1fc153d8385b 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -153,16 +153,25 @@ static __init void setup_topology(void) void __init __do_early_pgm_check(struct pt_regs *regs) { struct lowcore *lc = get_lowcore(); - unsigned long ip; regs->int_code = lc->pgm_int_code; regs->int_parm_long = lc->trans_exc_code; regs->last_break = lc->pgm_last_break; - ip = __rewind_psw(regs->psw, regs->int_code >> 16); + regs->monitor_code = lc->monitor_code; /* Monitor Event? Might be a warning */ if ((regs->int_code & PGM_INT_CODE_MASK) == 0x40) { - if (report_bug(ip, regs) == BUG_TRAP_TYPE_WARN) + enum bug_trap_type btt; + unsigned long ip; + + if (regs->monitor_code == MONCODE_BUG_ARG) { + regs->psw.addr = regs->gprs[14]; + btt = report_bug_entry((struct bug_entry *)regs->gprs[2], regs); + } else { + ip = __rewind_psw(regs->psw, regs->int_code >> 16); + btt = report_bug(ip, regs); + } + if (btt == BUG_TRAP_TYPE_WARN) return; } if (fixup_exception(regs)) From 6128f9cd1ae0d298d795269c4553bfd73aff954a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:18 +0200 Subject: [PATCH 056/102] s390: Clear pt_regs flags field Clear the pt_regs flags field in the early program check handler as well as in the stack_invalid handler. Without this the pt_regs flags field will have random values, depending on previous stack contents. This matters currently for bug reporting and disassembly, where flags like PIF_PSW_ADDR_ADJUSTED may then be incorrectly set, which could lead to a partially incorrect disassembly or wrong code markers. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/kernel/early.c | 1 + arch/s390/kernel/entry.S | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 1fc153d8385b..b8e3c065fbec 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -154,6 +154,7 @@ void __init __do_early_pgm_check(struct pt_regs *regs) { struct lowcore *lc = get_lowcore(); + regs->flags = 0; regs->int_code = lc->pgm_int_code; regs->int_parm_long = lc->trans_exc_code; regs->last_break = lc->pgm_last_break; diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 10dd9bbdf985..f51ecd8e7d21 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -621,6 +621,7 @@ SYM_CODE_START(stack_invalid) stmg %r0,%r7,__PT_R0(%r11) stmg %r8,%r9,__PT_PSW(%r11) mvc __PT_R8(64,%r11),0(%r14) + xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) GET_LC %r2 mvc __PT_ORIG_GPR2(8,%r11),__LC_PGM_LAST_BREAK(%r2) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) From 352cb5c6272043a9e1db2fce5cd8f5661e654a59 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:19 +0200 Subject: [PATCH 057/102] s390/bug: Indicate modified PSW in disassembly With [1] the PSW address is set to the return address of the calling __WARN_trap() function in order to provide a more useful disassembly of the caller rather than the helper itself. However, since regs->psw.addr is modified, regs->int_code has nothing to do with the exception. In order to make that more obvious change the disassembler output. Introduce a new PIF_WARN_TRAP_SKIPPED pt_regs flag and set it whenever __WARN_trap() is skipped. In the disassembler output print a "WARN RET" prefix to make it obvious that the PSW points to the return address of the warning trap function. Also do not print the interruption code and instruction length code for this case, since they have nothing to do with the modified PSW. Example output: Krnl Code: 000003ffe185bc1c: e3f0ff98ff71 lay %r15,-104(%r15) 000003ffe185bc22: e3e0f0980024 stg %r14,152(%r15) 000003ffe185bc28: c0e5ff45ed94 brasl %r14,000003ffe0119750 WARN RET >000003ffe185bc2e: c0e5ffa052b9 brasl %r14,000003ffe0c661a0 000003ffe185bc34: c020fffe86d6 larl %r2,000003ffe182c9e0 000003ffe185bc3a: e548f0a80006 mvghi 168(%r15),6 000003ffe185bc40: e548f0a00005 mvghi 160(%r15),5 000003ffe185bc46: a7690004 lghi %r6,4 [1] 79996065cfa2 ("s390/bug: Skip __WARN_trap() in call traces") Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/include/asm/ptrace.h | 2 ++ arch/s390/kernel/dis.c | 23 ++++++++++------------- arch/s390/kernel/dumpstack.c | 7 +++++-- arch/s390/kernel/early.c | 9 +++++++-- arch/s390/kernel/traps.c | 1 + 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index 6411e3584283..c844b8aa0fd9 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -17,11 +17,13 @@ #define PIF_PSW_ADDR_ADJUSTED 1 /* psw address has been adjusted */ #define PIF_SYSCALL_RET_SET 2 /* return value was set via ptrace */ #define PIF_FTRACE_FULL_REGS 3 /* all register contents valid (ftrace) */ +#define PIF_WARN_TRAP_SKIPPED 4 /* __WARN_trap() was skipped */ #define _PIF_SYSCALL BIT(PIF_SYSCALL) #define _PIF_ADDR_PSW_ADJUSTED BIT(PIF_PSW_ADDR_ADJUSTED) #define _PIF_SYSCALL_RET_SET BIT(PIF_SYSCALL_RET_SET) #define _PIF_FTRACE_FULL_REGS BIT(PIF_FTRACE_FULL_REGS) +#define _PIF_WARN_TRAP_SKIPPED BIT(PIF_WARN_TRAP_SKIPPED) #define PSW32_MASK_PER _AC(0x40000000, UL) #define PSW32_MASK_DAT _AC(0x04000000, UL) diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index 9c349eb884bf..f17c44e1740b 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -540,34 +540,31 @@ void show_code(struct pt_regs *regs) start += 2; } /* Decode the instructions. */ - ptr = buffer; - ptr += sprintf(ptr, "%s Code:", mode); + printk("%s Code:\n", mode); hops = 0; while (start < end && hops < 8) { + ptr = buffer; opsize = insn_length(code[start]); - if (start + opsize == 32) - *ptr++ = '*'; + if (start == 32 && test_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED)) + ptr += sprintf(ptr, " WARN RET >"); + else if (start + opsize == 32 && !test_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED)) + ptr += sprintf(ptr, " *"); else if (start == 32) - *ptr++ = '>'; + ptr += sprintf(ptr, " >"); else - *ptr++ = ' '; + ptr += sprintf(ptr, " "); addr = pswaddr + start - 32; ptr += sprintf(ptr, "%px: ", (void *)addr); if (start + opsize >= end) break; for (i = 0; i < opsize; i++) ptr += sprintf(ptr, "%02x", code[start + i]); - *ptr++ = '\t'; - if (i < 6) - *ptr++ = '\t'; + ptr += sprintf(ptr, "%*s", 13 - 2 * opsize, " "); ptr += print_insn(ptr, code + start, addr); start += opsize; - pr_cont("%s", buffer); - ptr = buffer; - ptr += sprintf(ptr, "\n "); + pr_cont("%s\n", buffer); hops++; } - pr_cont("\n"); } void print_fn_code(unsigned char *code, unsigned long len) diff --git a/arch/s390/kernel/dumpstack.c b/arch/s390/kernel/dumpstack.c index f9d52e05e01e..4ec08264b3ef 100644 --- a/arch/s390/kernel/dumpstack.c +++ b/arch/s390/kernel/dumpstack.c @@ -203,8 +203,11 @@ void __noreturn die(struct pt_regs *regs, const char *str) console_verbose(); spin_lock_irq(&die_lock); bust_spinlocks(1); - printk("%s: %04x ilc:%d [#%d]", str, regs->int_code & 0xffff, - regs->int_code >> 17, ++die_counter); + if (test_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED)) + printk("%s: [#%d]", str, ++die_counter); + else + printk("%s: %04x ilc:%d [#%d]", str, regs->int_code & 0xffff, + regs->int_code >> 17, ++die_counter); pr_cont("SMP "); if (debug_pagealloc_enabled()) pr_cont("DEBUG_PAGEALLOC"); diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index b8e3c065fbec..d875b3b304be 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -167,6 +167,7 @@ void __init __do_early_pgm_check(struct pt_regs *regs) if (regs->monitor_code == MONCODE_BUG_ARG) { regs->psw.addr = regs->gprs[14]; + set_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED); btt = report_bug_entry((struct bug_entry *)regs->gprs[2], regs); } else { ip = __rewind_psw(regs->psw, regs->int_code >> 16); @@ -183,8 +184,12 @@ void __init __do_early_pgm_check(struct pt_regs *regs) * some basic information in case it is too early for printk(). */ register_early_console(); - early_printk("PANIC: early exception %04x PSW: %016lx %016lx\n", - regs->int_code & 0xffff, regs->psw.mask, regs->psw.addr); + if (test_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED)) + early_printk("PANIC: early exception PSW: %016lx %016lx\n", + regs->psw.mask, regs->psw.addr); + else + early_printk("PANIC: early exception %04x PSW: %016lx %016lx\n", + regs->int_code & 0xffff, regs->psw.mask, regs->psw.addr); show_regs(regs); disabled_wait(); } diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index dd051046170b..0187994712bc 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -267,6 +267,7 @@ static void monitor_event_exception(struct pt_regs *regs) return; if (regs->monitor_code == MONCODE_BUG_ARG) { regs->psw.addr = regs->gprs[14]; + set_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED); btt = report_bug_entry((struct bug_entry *)regs->gprs[2], regs); } else { btt = report_bug(regs->psw.addr - (regs->int_code >> 16), regs); From f9ef2f6c877a9faef8528058eb62fcefde83ed9a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:20 +0200 Subject: [PATCH 058/102] s390/tools: Rename gen_facilities to gen_bitmaps Rename gen_facilities.c to gen_bitmaps.c and make it more generic so it can be used for various bitmaps. Add command line parsing, and use it to explicitly specify that the facilities header should be generated. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/tools/.gitignore | 1 + arch/s390/tools/Makefile | 8 +-- .../tools/{gen_facilities.c => gen_bitmaps.c} | 49 +++++++++++++------ 3 files changed, 38 insertions(+), 20 deletions(-) rename arch/s390/tools/{gen_facilities.c => gen_bitmaps.c} (83%) diff --git a/arch/s390/tools/.gitignore b/arch/s390/tools/.gitignore index e6af51d9d183..e07d27143172 100644 --- a/arch/s390/tools/.gitignore +++ b/arch/s390/tools/.gitignore @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only gen_facilities +gen_bitmaps gen_opcode_table relocs diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile index f2862364fb42..ac267bb46550 100644 --- a/arch/s390/tools/Makefile +++ b/arch/s390/tools/Makefile @@ -10,17 +10,17 @@ PHONY += kapi kapi: $(kapi-hdrs-y) -hostprogs += gen_facilities +hostprogs += gen_bitmaps hostprogs += gen_opcode_table -HOSTCFLAGS_gen_facilities.o += $(LINUXINCLUDE) +HOSTCFLAGS_gen_bitmaps.o += $(LINUXINCLUDE) -filechk_facility-defs.h = $(obj)/gen_facilities +filechk_facility-defs.h = $(obj)/gen_bitmaps --facility filechk_dis-defs.h = \ $(obj)/gen_opcode_table < $(srctree)/arch/$(ARCH)/tools/opcodes.txt -$(kapi)/facility-defs.h: $(obj)/gen_facilities FORCE +$(kapi)/facility-defs.h: $(obj)/gen_bitmaps FORCE $(call filechk,facility-defs.h) $(kapi)/dis-defs.h: $(obj)/gen_opcode_table FORCE diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_bitmaps.c similarity index 83% rename from arch/s390/tools/gen_facilities.c rename to arch/s390/tools/gen_bitmaps.c index 32dd5a57240d..4bc5f19dd405 100644 --- a/arch/s390/tools/gen_facilities.c +++ b/arch/s390/tools/gen_bitmaps.c @@ -1,24 +1,25 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Simple program to generate defines out of facility lists that use the bit - * numbering scheme from the Princples of Operations: most significant bit - * has bit number 0. + * Simple program to generate bitmaps, for example facility lists. * * Copyright IBM Corp. 2015, 2018 * */ +#include #include #include #include #include -struct facility_def { +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) + +struct bitmap_def { char *name; int *bits; }; -static struct facility_def facility_defs[] = { +static struct bitmap_def facility_defs[] = { { /* * FACILITIES_ALS contains the list of facilities that are @@ -124,7 +125,7 @@ static struct facility_def facility_defs[] = { }, }; -static void print_facility_list(struct facility_def *def) +static void print_bitmap(struct bitmap_def *def) { unsigned int high, bit, dword, i; unsigned long long *array; @@ -151,25 +152,41 @@ static void print_facility_list(struct facility_def *def) free(array); } -static void print_facility_lists(void) +static void print_header(const char *guard, struct bitmap_def *defs, size_t num) { unsigned int i; - for (i = 0; i < sizeof(facility_defs) / sizeof(facility_defs[0]); i++) - print_facility_list(&facility_defs[i]); -} - -int main(int argc, char **argv) -{ - printf("#ifndef __ASM_S390_FACILITY_DEFS__\n"); - printf("#define __ASM_S390_FACILITY_DEFS__\n"); + printf("#ifndef __ASM_S390_%s_DEFS__\n", guard); + printf("#define __ASM_S390_%s_DEFS__\n", guard); printf("/*\n"); printf(" * DO NOT MODIFY.\n"); printf(" *\n"); printf(" * This file was generated by %s\n", __FILE__); printf(" */\n\n"); printf("#include \n\n"); - print_facility_lists(); + for (i = 0; i < num; i++) + print_bitmap(&defs[i]); printf("\n#endif\n"); +} + +static const struct option long_options[] = { + { "facility", no_argument, NULL, 'f' }, + { NULL, 0, NULL, 0 } +}; + +int main(int argc, char **argv) +{ + int opt; + + opt = getopt_long(argc, argv, "", long_options, NULL); + if (opt == -1) + return 1; + switch (opt) { + case 'f': + print_header("FACILITY", facility_defs, ARRAY_SIZE(facility_defs)); + break; + default: + return 1; + } return 0; } From 319ccf083b9942abdb69ec6749a3cce08ebc62bf Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:21 +0200 Subject: [PATCH 059/102] s390/tools: Generate bitmap of nullified program checks Add PGM_CHECKS_NULLIFIED bitmap to gen_bitmaps and add a --pgmcheck command line option to generate the pgmcheck-defs.h header file. The bitmap specifies which program checks are nullified, and will be used with future changes. Change gen_bitmaps.c so it can create bitmaps with different bit numbering: MSB0 bit numbering like before (most significant bit is bit 0), and LSB0 bit numbering (least significant bit is bit 0), which is used for the new bitmap. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/include/asm/Kbuild | 1 + arch/s390/tools/Makefile | 8 +++++- arch/s390/tools/gen_bitmaps.c | 54 +++++++++++++++++++++++++++++++++-- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild index 0c1fc47c3ba0..d13dd7e3dc01 100644 --- a/arch/s390/include/asm/Kbuild +++ b/arch/s390/include/asm/Kbuild @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 generated-y += dis-defs.h generated-y += facility-defs.h +generated-y += pgmcheck-defs.h generated-y += syscall_table.h generated-y += unistd_nr.h diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile index ac267bb46550..ba1e42d005e7 100644 --- a/arch/s390/tools/Makefile +++ b/arch/s390/tools/Makefile @@ -4,7 +4,9 @@ # kapi := arch/$(ARCH)/include/generated/asm -kapi-hdrs-y := $(kapi)/facility-defs.h $(kapi)/dis-defs.h +kapi-hdrs-y := $(kapi)/dis-defs.h +kapi-hdrs-y += $(kapi)/facility-defs.h +kapi-hdrs-y += $(kapi)/pgmcheck-defs.h PHONY += kapi @@ -16,6 +18,7 @@ hostprogs += gen_opcode_table HOSTCFLAGS_gen_bitmaps.o += $(LINUXINCLUDE) filechk_facility-defs.h = $(obj)/gen_bitmaps --facility +filechk_pgmcheck-defs.h = $(obj)/gen_bitmaps --pgmcheck filechk_dis-defs.h = \ $(obj)/gen_opcode_table < $(srctree)/arch/$(ARCH)/tools/opcodes.txt @@ -23,6 +26,9 @@ filechk_dis-defs.h = \ $(kapi)/facility-defs.h: $(obj)/gen_bitmaps FORCE $(call filechk,facility-defs.h) +$(kapi)/pgmcheck-defs.h: $(obj)/gen_bitmaps FORCE + $(call filechk,pgmcheck-defs.h) + $(kapi)/dis-defs.h: $(obj)/gen_opcode_table FORCE $(call filechk,dis-defs.h) diff --git a/arch/s390/tools/gen_bitmaps.c b/arch/s390/tools/gen_bitmaps.c index 4bc5f19dd405..2e5dc491b19e 100644 --- a/arch/s390/tools/gen_bitmaps.c +++ b/arch/s390/tools/gen_bitmaps.c @@ -17,6 +17,8 @@ struct bitmap_def { char *name; int *bits; + int inverted; + int maxbit; }; static struct bitmap_def facility_defs[] = { @@ -27,6 +29,7 @@ static struct bitmap_def facility_defs[] = { * -march=. */ .name = "FACILITIES_ALS", + .inverted = 1, .bits = (int[]){ 0, /* N3 instructions */ 1, /* z/Arch mode installed */ @@ -71,6 +74,7 @@ static struct bitmap_def facility_defs[] = { * guest facilities supported by KVM. */ .name = "FACILITIES_KVM", + .inverted = 1, .bits = (int[]){ 0, /* N3 instructions */ 1, /* z/Arch mode installed */ @@ -116,6 +120,7 @@ static struct bitmap_def facility_defs[] = { */ .name = "FACILITIES_KVM_CPUMODEL", + .inverted = 1, .bits = (int[]){ 12, /* AP Query Configuration Information */ 15, /* AP Facilities Test */ @@ -125,17 +130,56 @@ static struct bitmap_def facility_defs[] = { }, }; +static struct bitmap_def pgm_check_defs[] = { + { + .name = "PGM_CHECKS_NULLIFIED", + .inverted = 0, + .maxbit = 127, + .bits = (int[]){ + 0x10, /* Segment Translation */ + 0x11, /* Page Translation */ + 0x16, /* Trace Table */ + 0x20, /* AFX Translation */ + 0x21, /* ASX Translation */ + 0x22, /* LX Translation */ + 0x23, /* EX Translation */ + 0x24, /* Primary Authority */ + 0x25, /* Secondary Authority */ + 0x26, /* LFX Translation */ + 0x27, /* LSX Translation */ + 0x29, /* ALEN Translation */ + 0x2a, /* ALE Sequence */ + 0x2b, /* ASTE Validity */ + 0x2c, /* ASTE Sequence */ + 0x2d, /* Extended Authority */ + 0x2e, /* LSTE Sequence */ + 0x2f, /* ASTE Instance */ + 0x30, /* Stack Full */ + 0x31, /* Stack Empty */ + 0x32, /* Stack Specification */ + 0x33, /* Stack Type */ + 0x34, /* Stack Operation */ + 0x38, /* ASCE Type */ + 0x39, /* Region First Translation */ + 0x3a, /* Region Second Translation */ + 0x3b, /* Region Third Translation */ + 0x3d, /* Secure Storage Access */ + -1 /* END */ + } + }, +}; + static void print_bitmap(struct bitmap_def *def) { unsigned int high, bit, dword, i; unsigned long long *array; - array = calloc(1, 8); + high = def->maxbit ? def->maxbit / 64 : 0; + array = calloc(high + 1, 8); if (!array) exit(EXIT_FAILURE); - high = 0; for (i = 0; def->bits[i] != -1; i++) { - bit = 63 - (def->bits[i] & 63); + bit = def->inverted ? 63 - (def->bits[i] & 63) : (def->bits[i] & 63); dword = def->bits[i] / 64; if (dword > high) { array = realloc(array, (dword + 1) * 8); @@ -171,6 +215,7 @@ static void print_header(const char *guard, struct bitmap_def *defs, size_t num) static const struct option long_options[] = { { "facility", no_argument, NULL, 'f' }, + { "pgmcheck", no_argument, NULL, 'p' }, { NULL, 0, NULL, 0 } }; @@ -185,6 +230,9 @@ int main(int argc, char **argv) case 'f': print_header("FACILITY", facility_defs, ARRAY_SIZE(facility_defs)); break; + case 'p': + print_header("PGMCHECK", pgm_check_defs, ARRAY_SIZE(pgm_check_defs)); + break; default: return 1; } From 241c66ba76546d4d1ecd1a7e3cc2b54dc1c22ad1 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:22 +0200 Subject: [PATCH 060/102] s390/traps: Handle nullifying program checks in get_trap_ip() get_trap_ip() always rewinds the PSW address by the instruction length from the interruption code. For nullifying program checks the PSW already points at the faulting instruction, so rewinding is wrong. Add a nullified[] bitmap and return the unchanged PSW address for nullifying cases, or if PIF_PSW_ADDR_ADJUSTED is set. No functional change for existing callers, which only deal with suppressing, terminating, or completing program checks. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/kernel/traps.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 0187994712bc..7f56d5a66de8 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -41,15 +42,19 @@ struct pgm_stat { static DEFINE_PER_CPU_SHARED_ALIGNED(struct pgm_stat, pgm_stat); -static inline void __user *get_trap_ip(struct pt_regs *regs) +static inline unsigned long get_trap_ip(struct pt_regs *regs) { - unsigned long address; + static const unsigned long nullified[2] = { PGM_CHECKS_NULLIFIED }; + psw_t psw = regs->psw; + if (test_pt_regs_flag(regs, PIF_PSW_ADDR_ADJUSTED)) + return psw.addr; if (regs->int_code & 0x200) - address = current->thread.trap_tdb.data[3]; - else - address = regs->psw.addr; - return (void __user *)(address - (regs->int_code >> 16)); + psw.addr = current->thread.trap_tdb.data[3]; + if (test_bit(regs->int_code & PGM_INT_CODE_MASK, nullified)) + return psw.addr; + psw.addr = __rewind_psw(psw, regs->int_code >> 16); + return psw.addr; } #ifdef CONFIG_GENERIC_BUG @@ -62,7 +67,7 @@ int is_valid_bugaddr(unsigned long addr) void do_report_trap(struct pt_regs *regs, int si_signo, int si_code, char *str) { if (user_mode(regs)) { - force_sig_fault(si_signo, si_code, get_trap_ip(regs)); + force_sig_fault(si_signo, si_code, (void __user *)get_trap_ip(regs)); report_user_fault(regs, si_signo, 0); } else { if (!fixup_exception(regs)) @@ -148,12 +153,12 @@ static void translation_specification_exception(struct pt_regs *regs) static void illegal_op(struct pt_regs *regs) { int is_uprobe_insn = 0; - u16 __user *location; int signal = 0; u16 opcode; - location = get_trap_ip(regs); if (user_mode(regs)) { + u16 __user *location = (void __user *)get_trap_ip(regs); + if (get_user(opcode, location)) return; if (opcode == S390_BREAKPOINT_U16) { From 04e326f59a2717b048591f14f021cd9d46e6406e Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:23 +0200 Subject: [PATCH 061/102] s390/dis: Improve show_code() output Replace the '>' and '*' instruction markers with explicit 'PSW >' and 'PGM >' markers which identify both the PSW address and the faulting instruction. Previously, '>' marked the PSW address, while '*' marked the preceding instruction as a guess for the faulting instruction for suppressing exceptions: Krnl Code: 000003ff8dc7bff8: e310f0080004 lg %r1,8(%r15) *000003ff8dc7bffe: e3a0e0000004 lg %r10,0(%r14) >000003ff8dc7c004: a7280000 lhi %r2,0 000003ff8dc7c008: eb9ff0900024 stmg %r9,%r15,144(%r15) Now 'PGM >' explicitly marks the faulting instruction without guessing and 'PSW >' marks the PSW address: Krnl Code: 000003ff8dc7bff8: e310f0080004 lg %r1,8(%r15) PGM >000003ff8dc7bffe: e3a0e0000004 lg %r10,0(%r14) PSW >000003ff8dc7c004: a7280000 lhi %r2,0 000003ff8dc7c008: eb9ff0900024 stmg %r9,%r15,144(%r15) For nullifying exceptions where both addresses are the same, 'PGM+PSW >' is printed instead: Krnl Code: 000003ff8dc7bff8: e310f0080004 lg %r1,8(%r15) PGM+PSW >000003ff8dc7bffe: e3a0e0000004 lg %r10,0(%r14) 000003ff8dc7c004: a7280000 lhi %r2,0 000003ff8dc7c008: eb9ff0900024 stmg %r9,%r15,144(%r15) show_code() may also be called in other contexts where pt_regs does not originate from a program check (e.g. system calls). Introduce and set PIF_PGM_CHECK to indicate when in program check context, and print the PGM marker only in such a context. Note that corner cases are not taken into account. Like before kprobes is not considered, which means that the disassembly may contain unexpected instructions, or that disassembly and/or instruction length count as well as interruption do not match. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/include/asm/ptrace.h | 4 ++++ arch/s390/kernel/dis.c | 18 ++++++++++++------ arch/s390/kernel/early.c | 1 + arch/s390/kernel/traps.c | 4 +++- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index c844b8aa0fd9..3e56ef8c268b 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -18,12 +18,14 @@ #define PIF_SYSCALL_RET_SET 2 /* return value was set via ptrace */ #define PIF_FTRACE_FULL_REGS 3 /* all register contents valid (ftrace) */ #define PIF_WARN_TRAP_SKIPPED 4 /* __WARN_trap() was skipped */ +#define PIF_PGM_CHECK 5 /* inside a program check */ #define _PIF_SYSCALL BIT(PIF_SYSCALL) #define _PIF_ADDR_PSW_ADJUSTED BIT(PIF_PSW_ADDR_ADJUSTED) #define _PIF_SYSCALL_RET_SET BIT(PIF_SYSCALL_RET_SET) #define _PIF_FTRACE_FULL_REGS BIT(PIF_FTRACE_FULL_REGS) #define _PIF_WARN_TRAP_SKIPPED BIT(PIF_WARN_TRAP_SKIPPED) +#define _PIF_PGM_CHECK BIT(PIF_PGM_CHECK) #define PSW32_MASK_PER _AC(0x40000000, UL) #define PSW32_MASK_DAT _AC(0x04000000, UL) @@ -242,6 +244,8 @@ static inline void instruction_pointer_set(struct pt_regs *regs, regs->psw.addr = val; } +unsigned long get_trap_ip(struct pt_regs *regs); + int regs_query_register_offset(const char *name); const char *regs_query_register_name(unsigned int offset); diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index f17c44e1740b..d2d299ba8b86 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -503,7 +503,8 @@ static int copy_from_regs(struct pt_regs *regs, void *dst, void *src, int len) void show_code(struct pt_regs *regs) { char *mode = user_mode(regs) ? "User" : "Krnl"; - unsigned long addr, pswaddr; + unsigned long addr, pswaddr, trapaddr; + bool pgm_check; unsigned char code[64]; char buffer[128], *ptr; int start, end, opsize, hops, i; @@ -511,6 +512,9 @@ void show_code(struct pt_regs *regs) pswaddr = regs->psw.addr; if (test_pt_regs_flag(regs, PIF_PSW_ADDR_ADJUSTED)) pswaddr = __forward_psw(regs->psw, regs->int_code >> 16); + pgm_check = test_pt_regs_flag(regs, PIF_PGM_CHECK); + if (pgm_check) + trapaddr = get_trap_ip(regs); /* Get a snapshot of the 64 bytes surrounding the fault address. */ for (start = 32; start && pswaddr >= 34 - start; start -= 2) { addr = pswaddr - 34 + start; @@ -545,15 +549,17 @@ void show_code(struct pt_regs *regs) while (start < end && hops < 8) { ptr = buffer; opsize = insn_length(code[start]); + addr = pswaddr + start - 32; if (start == 32 && test_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED)) ptr += sprintf(ptr, " WARN RET >"); - else if (start + opsize == 32 && !test_pt_regs_flag(regs, PIF_WARN_TRAP_SKIPPED)) - ptr += sprintf(ptr, " *"); + else if (pgm_check && addr == trapaddr && start == 32) + ptr += sprintf(ptr, " PGM+PSW >"); + else if (pgm_check && addr == trapaddr) + ptr += sprintf(ptr, " PGM >"); else if (start == 32) - ptr += sprintf(ptr, " >"); + ptr += sprintf(ptr, " PSW >"); else ptr += sprintf(ptr, " "); - addr = pswaddr + start - 32; ptr += sprintf(ptr, "%px: ", (void *)addr); if (start + opsize >= end) break; @@ -562,7 +568,7 @@ void show_code(struct pt_regs *regs) ptr += sprintf(ptr, "%*s", 13 - 2 * opsize, " "); ptr += print_insn(ptr, code + start, addr); start += opsize; - pr_cont("%s\n", buffer); + printk("%s\n", buffer); hops++; } } diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index d875b3b304be..a603e034fa82 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -159,6 +159,7 @@ void __init __do_early_pgm_check(struct pt_regs *regs) regs->int_parm_long = lc->trans_exc_code; regs->last_break = lc->pgm_last_break; regs->monitor_code = lc->monitor_code; + set_pt_regs_flag(regs, PIF_PGM_CHECK); /* Monitor Event? Might be a warning */ if ((regs->int_code & PGM_INT_CODE_MASK) == 0x40) { diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 7f56d5a66de8..e35e0018f372 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -42,7 +42,7 @@ struct pgm_stat { static DEFINE_PER_CPU_SHARED_ALIGNED(struct pgm_stat, pgm_stat); -static inline unsigned long get_trap_ip(struct pt_regs *regs) +unsigned long get_trap_ip(struct pt_regs *regs) { static const unsigned long nullified[2] = { PGM_CHECKS_NULLIFIED }; psw_t psw = regs->psw; @@ -56,6 +56,7 @@ static inline unsigned long get_trap_ip(struct pt_regs *regs) psw.addr = __rewind_psw(psw, regs->int_code >> 16); return psw.addr; } +NOKPROBE_SYMBOL(get_trap_ip); #ifdef CONFIG_GENERIC_BUG int is_valid_bugaddr(unsigned long addr) @@ -353,6 +354,7 @@ void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags) regs->int_code = lc->pgm_int_code; regs->int_parm_long = teid.val; regs->monitor_code = lc->monitor_code; + set_pt_regs_flag(regs, PIF_PGM_CHECK); trapnr = regs->int_code & PGM_INT_CODE_MASK; stat = this_cpu_ptr(&pgm_stat); From 70fdd89e316af48c037a3b9b0c267335158a669d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:24 +0200 Subject: [PATCH 062/102] s390/extable: Rework exception handling logic Add EX_TABLE_INSN macros which will replace the existing EX_TABLE macros. The macros come with new semantics for the whole exception handling: with the new exception handling the exception table entries always point to the instruction which may cause an exception. This is regardless of exception type, and whether an exception is nullifying, suppressing, completing, or terminating. This improves the current situation where exception table entries are required which point to the instruction (nullifying) and/or behind the instruction (suppressing, completing, or terminating), since depending on usage an instruction can generate different exceptions. The new EX_TABLE_INSN macros are aliases to the existing EX_TABLE macros. The exception handling logic is changed to first perform a lookup in the exception tables with get_trap_ip() / boot_get_trap_ip(). If that finds nothing, it falls back to looking up with instruction_pointer(). This is a temporary solution until all EX_TABLE usages have been converted to EX_TABLE_INSN. When that work is complete EX_TABLE will be removed, exception tables will only be searched by using the return value of get_trap_ip() / boot_get_trap_ip(). Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/boot/pgm_check.c | 35 ++++++++++++++++++++++++----- arch/s390/include/asm/asm-extable.h | 10 +++++++++ arch/s390/mm/extable.c | 4 +++- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/arch/s390/boot/pgm_check.c b/arch/s390/boot/pgm_check.c index fa621fa5bc02..56d0be7e2823 100644 --- a/arch/s390/boot/pgm_check.c +++ b/arch/s390/boot/pgm_check.c @@ -3,9 +3,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -40,19 +42,40 @@ static inline unsigned long extable_insn(const struct exception_table_entry *x) return (unsigned long)&x->insn + x->insn; } -static bool ex_handler(struct pt_regs *regs) +static unsigned long boot_get_trap_ip(struct pt_regs *regs) +{ + static const unsigned long nullified[2] = { PGM_CHECKS_NULLIFIED }; + + if (test_bit(regs->int_code & PGM_INT_CODE_MASK, nullified)) + return regs->psw.addr; + return __rewind_psw(regs->psw, regs->int_code >> 16); +} + +static const struct exception_table_entry *search_boot_extable(unsigned long ip) { const struct exception_table_entry *ex; for (ex = __start___ex_table; ex < __stop___ex_table; ex++) { - if (extable_insn(ex) != regs->psw.addr) + if (extable_insn(ex) != ip) continue; if (ex->type != EX_TYPE_FIXUP) - return false; - regs->psw.addr = extable_fixup(ex); - return true; + return NULL; + return ex; } - return false; + return NULL; +} + +static bool ex_handler(struct pt_regs *regs) +{ + const struct exception_table_entry *ex; + + ex = search_boot_extable(boot_get_trap_ip(regs)); + if (!ex) + ex = search_boot_extable(instruction_pointer(regs)); + if (!ex) + return false; + regs->psw.addr = extable_fixup(ex); + return true; } void do_pgm_check(struct pt_regs *regs) diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index d23ea0c94e4e..488bd10452b3 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -92,4 +92,14 @@ #define EX_TABLE_UA_MVCOS_FROM(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_MVCOS_FROM, __stringify(%%r0), __stringify(%%r0), 0) +#define EX_TABLE_INSN EX_TABLE +#define EX_TABLE_INSN_AMODE31 EX_TABLE_AMODE31 +#define EX_TABLE_INSN_UA_FAULT EX_TABLE_UA_FAULT +#define EX_TABLE_INSN_UA_LOAD_REG EX_TABLE_UA_LOAD_REG +#define EX_TABLE_INSN_UA_LOAD_REGPAIR EX_TABLE_UA_LOAD_REGPAIR +#define EX_TABLE_INSN_ZEROPAD EX_TABLE_ZEROPAD +#define EX_TABLE_INSN_FPC EX_TABLE_FPC +#define EX_TABLE_INSN_UA_MVCOS_TO EX_TABLE_UA_MVCOS_TO +#define EX_TABLE_INSN_UA_MVCOS_FROM EX_TABLE_UA_MVCOS_FROM + #endif /* __ASM_EXTABLE_H */ diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index 848855da9a70..3d2ad6e853d1 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -120,7 +120,9 @@ bool fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *ex; - ex = s390_search_extables(instruction_pointer(regs)); + ex = s390_search_extables(get_trap_ip(regs)); + if (!ex) + ex = s390_search_extables(instruction_pointer(regs)); if (!ex) return false; switch (ex->type) { From 7d32c0db67980550af06409e7eaee3a9ea311f88 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:25 +0200 Subject: [PATCH 063/102] s390/diag: Convert amode31 exception tables to EX_TABLE_INSN Convert all amode31 exception table entries to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so named labels are replaced with numeric labels. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/kernel/text_amode31.S | 34 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/arch/s390/kernel/text_amode31.S b/arch/s390/kernel/text_amode31.S index f007d892d0c5..7761de491443 100644 --- a/arch/s390/kernel/text_amode31.S +++ b/arch/s390/kernel/text_amode31.S @@ -19,9 +19,9 @@ * affects a few functions that are not performance-relevant. */ .macro BR_EX_AMODE31_r14 - exrl 0,0f + exrl 0,555f j . -0: br %r14 +555: br %r14 .endm /* @@ -33,15 +33,13 @@ SYM_TYPED_FUNC_START(_diag14_amode31) lgr %r3,%r4 lhi %r5,-EIO sam31 - diag %r1,%r2,0x14 -.Ldiag14_ex: +0: diag %r1,%r2,0x14 ipm %r5 srl %r5,28 -.Ldiag14_fault: - sam64 +1: sam64 lgfr %r2,%r5 BR_EX_AMODE31_r14 - EX_TABLE_AMODE31(.Ldiag14_ex, .Ldiag14_fault) + EX_TABLE_INSN_AMODE31(0b, 1b) SYM_FUNC_END(_diag14_amode31) /* @@ -51,15 +49,13 @@ SYM_TYPED_FUNC_START(_diag210_amode31) lgr %r1,%r2 lhi %r2,-1 sam31 - diag %r1,%r0,0x210 -.Ldiag210_ex: +0: diag %r1,%r0,0x210 ipm %r2 srl %r2,28 -.Ldiag210_fault: - sam64 +1: sam64 lgfr %r2,%r2 BR_EX_AMODE31_r14 - EX_TABLE_AMODE31(.Ldiag210_ex, .Ldiag210_fault) + EX_TABLE_INSN_AMODE31(0b, 1b) SYM_FUNC_END(_diag210_amode31) /* @@ -68,12 +64,11 @@ SYM_FUNC_END(_diag210_amode31) SYM_TYPED_FUNC_START(_diag8c_amode31) llgf %r3,0(%r3) sam31 - diag %r2,%r4,0x8c -.Ldiag8c_ex: - sam64 +0: diag %r2,%r4,0x8c +1: sam64 lgfr %r2,%r3 BR_EX_AMODE31_r14 - EX_TABLE_AMODE31(.Ldiag8c_ex, .Ldiag8c_ex) + EX_TABLE_INSN_AMODE31(0b, 1b) SYM_FUNC_END(_diag8c_amode31) /* * int _diag26c_amode31(void *req, void *resp, enum diag26c_sc subcode) @@ -81,12 +76,11 @@ SYM_FUNC_END(_diag8c_amode31) SYM_TYPED_FUNC_START(_diag26c_amode31) lghi %r5,-EOPNOTSUPP sam31 - diag %r2,%r4,0x26c -.Ldiag26c_ex: - sam64 +0: diag %r2,%r4,0x26c +1: sam64 lgfr %r2,%r5 BR_EX_AMODE31_r14 - EX_TABLE_AMODE31(.Ldiag26c_ex, .Ldiag26c_ex) + EX_TABLE_INSN_AMODE31(0b, 1b) SYM_FUNC_END(_diag26c_amode31) /* From cd19c5f9f9f6f04331edf2eb986c3c47783c02ba Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:26 +0200 Subject: [PATCH 064/102] s390/uaccess: Convert inline assembly to EX_TABLE_INSN Convert all uaccess exception table entries to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so duplicate entries and nopr padding are no longer needed and removed. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/include/asm/uaccess.h | 45 +++++++++++---------------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index a9f32c53f699..ac99b32c6ceb 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -43,10 +43,8 @@ raw_copy_from_user(void *to, const void __user *from, unsigned long size) asm_inline volatile( " lhi %%r0,%[spec]\n" "0: mvcos %[to],%[from],%[size]\n" - "1: nopr %%r7\n" CC_IPM(cc) - EX_TABLE_UA_MVCOS_FROM(0b, 0b) - EX_TABLE_UA_MVCOS_FROM(1b, 0b) + EX_TABLE_INSN_UA_MVCOS_FROM(0b, 0b) : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to) : [spec] "I" (0x81), [from] "Q" (*(const char __user *)from) : CC_CLOBBER_LIST("memory", "0")); @@ -71,10 +69,8 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long size) asm_inline volatile( " llilh %%r0,%[spec]\n" "0: mvcos %[to],%[from],%[size]\n" - "1: nopr %%r7\n" CC_IPM(cc) - EX_TABLE_UA_MVCOS_TO(0b, 0b) - EX_TABLE_UA_MVCOS_TO(1b, 0b) + EX_TABLE_INSN_UA_MVCOS_TO(0b, 0b) : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to) : [spec] "I" (0x81), [from] "Q" (*(const char *)from) : CC_CLOBBER_LIST("memory", "0")); @@ -123,9 +119,7 @@ __put_user_##type##_noinstr(unsigned type __user *to, \ asm goto( \ " llilh %%r0,%[spec]\n" \ "0: mvcos %[to],%[from],%[size]\n" \ - "1: nopr %%r7\n" \ - EX_TABLE(0b, %l[Efault]) \ - EX_TABLE(1b, %l[Efault]) \ + EX_TABLE_INSN(0b, %l[Efault]) \ : [to] "+Q" (*to) \ : [size] "d" (size), [from] "Q" (*from), \ [spec] "I" (0x81) \ @@ -150,10 +144,9 @@ __put_user_##type##_noinstr(unsigned type __user *to, \ asm_inline volatile( \ " llilh %%r0,%[spec]\n" \ "0: mvcos %[to],%[from],%[size]\n" \ - "1: lhi %[rc],0\n" \ - "2:\n" \ - EX_TABLE_UA_FAULT(0b, 2b, %[rc]) \ - EX_TABLE_UA_FAULT(1b, 2b, %[rc]) \ + " lhi %[rc],0\n" \ + "1:\n" \ + EX_TABLE_INSN_UA_FAULT(0b, 1b, %[rc]) \ : [rc] "=d" (rc), [to] "+Q" (*to) \ : [size] "d" (size), [from] "Q" (*from), \ [spec] "I" (0x81) \ @@ -238,9 +231,7 @@ __get_user_##type##_noinstr(unsigned type *to, \ asm goto( \ " lhi %%r0,%[spec]\n" \ "0: mvcos %[to],%[from],%[size]\n" \ - "1: nopr %%r7\n" \ - EX_TABLE(0b, %l[Efault]) \ - EX_TABLE(1b, %l[Efault]) \ + EX_TABLE_INSN(0b, %l[Efault]) \ : [to] "=Q" (*to) \ : [size] "d" (size), [from] "Q" (*from), \ [spec] "I" (0x81) \ @@ -266,10 +257,9 @@ __get_user_##type##_noinstr(unsigned type *to, \ asm_inline volatile( \ " lhi %%r0,%[spec]\n" \ "0: mvcos %[to],%[from],%[size]\n" \ - "1: lhi %[rc],0\n" \ - "2:\n" \ - EX_TABLE_UA_FAULT(0b, 2b, %[rc]) \ - EX_TABLE_UA_FAULT(1b, 2b, %[rc]) \ + " lhi %[rc],0\n" \ + "1:\n" \ + EX_TABLE_INSN_UA_FAULT(0b, 1b, %[rc]) \ : [rc] "=d" (rc), [to] "=Q" (*to) \ : [size] "d" (size), [from] "Q" (*from), \ [spec] "I" (0x81) \ @@ -374,10 +364,8 @@ __clear_user(void __user *to, unsigned long size) asm_inline volatile( " llilh %%r0,%[spec]\n" "0: mvcos %[to],%[from],%[size]\n" - "1: nopr %%r7\n" CC_IPM(cc) - EX_TABLE_UA_MVCOS_TO(0b, 0b) - EX_TABLE_UA_MVCOS_TO(1b, 0b) + EX_TABLE_INSN_UA_MVCOS_TO(0b, 0b) : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to) : [spec] "I" (0x81), [from] "Q" (*(const char *)empty_zero_page) : CC_CLOBBER_LIST("memory", "0")); @@ -418,9 +406,7 @@ do { \ case 8: \ asm goto( \ "0: mvc %O[_dst](%[_len],%R[_dst]),%[_src]\n" \ - "1: nopr %%r7\n" \ - EX_TABLE(0b, %l[err_label]) \ - EX_TABLE(1b, %l[err_label]) \ + EX_TABLE_INSN(0b, %l[err_label]) \ : [_dst] "=Q" (*(type *)dst) \ : [_src] "Q" (*(type *)(src)), \ [_len] "I" (sizeof(type)) \ @@ -447,10 +433,9 @@ do { \ case 8: \ asm_inline volatile( \ "0: mvc 0(%[_len],%[_dst]),%[_src]\n" \ - "1: lhi %[_rc],0\n" \ - "2:\n" \ - EX_TABLE_UA_FAULT(0b, 2b, %[_rc]) \ - EX_TABLE_UA_FAULT(1b, 2b, %[_rc]) \ + " lhi %[_rc],0\n" \ + "1:\n" \ + EX_TABLE_INSN_UA_FAULT(0b, 1b, %[_rc]) \ : [_rc] "=d" (__rc), \ "=m" (*__dst) \ : [_src] "Q" (*(type *)(src)), \ From adddc28271c0b10eb7314086aeafb68d9e1005f6 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:27 +0200 Subject: [PATCH 065/102] s390/lib/uaccess: Convert inline assembly to EX_TABLE_INSN Convert all uaccess exception table entries to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so duplicate entries are removed. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/lib/uaccess.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 0ac2f3998b14..d3f462c12bba 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -49,24 +49,22 @@ static nokprobe_inline int __cmpxchg_key_small(void *address, unsigned int *uval "20: spka 0(%[key])\n" " llill %[count],%[max_loops]\n" "0: l %[prev],%[address]\n" - "1: nr %[prev],%[mask]\n" + " nr %[prev],%[mask]\n" " xilf %[mask],0xffffffff\n" " or %[new],%[prev]\n" " or %[prev],%[tmp]\n" - "2: lr %[tmp],%[prev]\n" - "3: cs %[prev],%[new],%[address]\n" - "4: jnl 5f\n" + "1: lr %[tmp],%[prev]\n" + "2: cs %[prev],%[new],%[address]\n" + " jnl 3f\n" " xr %[tmp],%[prev]\n" " xr %[new],%[tmp]\n" " nr %[tmp],%[mask]\n" - " jnz 5f\n" - " brct %[count],2b\n" - "5: spka %[default_key]\n" + " jnz 3f\n" + " brct %[count],1b\n" + "3: spka %[default_key]\n" "21:\n" - EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev]) + EX_TABLE_INSN_UA_LOAD_REG(0b, 3b, %[rc], %[prev]) + EX_TABLE_INSN_UA_LOAD_REG(2b, 3b, %[rc], %[prev]) SKEY_REGION(20b, 21b) : [rc] "+&d" (rc), [prev] "=&d" (prev), @@ -133,8 +131,7 @@ int __kprobes __cmpxchg_key4(void *address, unsigned int *uval, unsigned int old "0: cs %[prev],%[new],%[address]\n" "1: spka %[default_key]\n" "21:\n" - EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) + EX_TABLE_INSN_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) SKEY_REGION(20b, 21b) : [rc] "+&d" (rc), [prev] "+&d" (prev), @@ -160,8 +157,7 @@ int __kprobes __cmpxchg_key8(void *address, unsigned long *uval, unsigned long o "0: csg %[prev],%[new],%[address]\n" "1: spka %[default_key]\n" "21:\n" - EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) + EX_TABLE_INSN_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) SKEY_REGION(20b, 21b) : [rc] "+&d" (rc), [prev] "+&d" (prev), @@ -187,8 +183,7 @@ int __kprobes __cmpxchg_key16(void *address, __uint128_t *uval, __uint128_t old, "0: cdsg %[prev],%[new],%[address]\n" "1: spka %[default_key]\n" "21:\n" - EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) - EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev]) + EX_TABLE_INSN_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) SKEY_REGION(20b, 21b) : [rc] "+&d" (rc), [prev] "+&d" (prev), From e2780ada7bdf0b84d8ad678b890b40e6e4564266 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:28 +0200 Subject: [PATCH 066/102] s390/futex: Convert inline assembly to EX_TABLE_INSN Convert all futex exception table entries to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so duplicate entries are removed. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/include/asm/futex.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h index 942f21c39697..71e8060637be 100644 --- a/arch/s390/include/asm/futex.h +++ b/arch/s390/include/asm/futex.h @@ -23,13 +23,11 @@ __futex_atomic_##name(int oparg, int *old, u32 __user *uaddr) \ "0: l %[old],%[uaddr]\n" \ "1:"insn \ "2: cs %[old],%[new],%[uaddr]\n" \ - "3: jl 1b\n" \ + " jl 1b\n" \ " lhi %[rc],0\n" \ - "4: sacf 768\n" \ - EX_TABLE_UA_FAULT(0b, 4b, %[rc]) \ - EX_TABLE_UA_FAULT(1b, 4b, %[rc]) \ - EX_TABLE_UA_FAULT(2b, 4b, %[rc]) \ - EX_TABLE_UA_FAULT(3b, 4b, %[rc]) \ + "3: sacf 768\n" \ + EX_TABLE_INSN_UA_FAULT(0b, 3b, %[rc]) \ + EX_TABLE_INSN_UA_FAULT(2b, 3b, %[rc]) \ : [rc] "=d" (rc), [old] "=&d" (*old), \ [new] "=&d" (new), [uaddr] "+Q" (*uaddr) \ : [oparg] "d" (oparg) \ @@ -86,10 +84,9 @@ int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 asm_inline volatile( " sacf 256\n" "0: cs %[old],%[new],%[uaddr]\n" - "1: lhi %[rc],0\n" - "2: sacf 768\n" - EX_TABLE_UA_FAULT(0b, 2b, %[rc]) - EX_TABLE_UA_FAULT(1b, 2b, %[rc]) + " lhi %[rc],0\n" + "1: sacf 768\n" + EX_TABLE_INSN_UA_FAULT(0b, 1b, %[rc]) : [rc] "=d" (rc), [old] "+d" (oldval), [uaddr] "+Q" (*uaddr) : [new] "d" (newval) : "cc", "memory"); From 0f748fe1f2c6401672e4d5b0ea371df3da9a6a76 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:29 +0200 Subject: [PATCH 067/102] s390/diag: Convert inline assembly to EX_TABLE_INSN Convert all diag exception table entries to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so duplicate entries and nopr padding are no longer needed and removed. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/include/asm/diag.h | 5 ++--- arch/s390/include/asm/diag288.h | 6 +++--- arch/s390/kernel/diag/diag.c | 12 ++++++------ 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h index 8db8db3b1018..1416ee1ad323 100644 --- a/arch/s390/include/asm/diag.h +++ b/arch/s390/include/asm/diag.h @@ -68,9 +68,8 @@ static inline void diag10_range(unsigned long start_pfn, unsigned long num_pfn) diag_stat_inc(DIAG_STAT_X010); asm_inline volatile( "0: diag %0,%1,0x10\n" - "1: nopr %%r7\n" - EX_TABLE(0b, 1b) - EX_TABLE(1b, 1b) + "1:\n" + EX_TABLE_INSN(0b, 1b) : : "a" (start_addr), "a" (end_addr)); } diff --git a/arch/s390/include/asm/diag288.h b/arch/s390/include/asm/diag288.h index 5e1b43cea9d6..f0a37a80d672 100644 --- a/arch/s390/include/asm/diag288.h +++ b/arch/s390/include/asm/diag288.h @@ -28,10 +28,10 @@ static inline int __diag288(unsigned int func, unsigned int timeout, int rc = -EINVAL; asm volatile( - " diag %[r1],%[r3],0x288\n" - "0: lhi %[rc],0\n" + "0: diag %[r1],%[r3],0x288\n" + " lhi %[rc],0\n" "1:" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [rc] "+d" (rc) : [r1] "d" (r1.pair), [r3] "d" (r3.pair) : "cc", "memory"); diff --git a/arch/s390/kernel/diag/diag.c b/arch/s390/kernel/diag/diag.c index 56b862ba9be8..f23b3e2b070e 100644 --- a/arch/s390/kernel/diag/diag.c +++ b/arch/s390/kernel/diag/diag.c @@ -196,9 +196,9 @@ static inline int __diag204(unsigned long *subcode, unsigned long size, void *ad union register_pair rp = { .even = *subcode, .odd = size }; asm_inline volatile( - " diag %[addr],%[rp],0x204\n" - "0: nopr %%r7\n" - EX_TABLE(0b,0b) + "0: diag %[addr],%[rp],0x204\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [rp] "+&d" (rp.pair) : [addr] "d" (addr) : "memory"); *subcode = rp.even; return rp.odd; @@ -287,10 +287,10 @@ int diag224(void *ptr) diag_stat_inc(DIAG_STAT_X224); asm_inline volatile("\n" - " diag %[type],%[addr],0x224\n" - "0: lhi %[rc],0\n" + "0: diag %[type],%[addr],0x224\n" + " lhi %[rc],0\n" "1:\n" - EX_TABLE(0b,1b) + EX_TABLE_INSN(0b, 1b) : [rc] "+d" (rc) , "=m" (*(struct { char buf[PAGE_SIZE]; } *)ptr) : [type] "d" (0), [addr] "d" (addr)); From 6934b8174347736404f91ae304e11a1c770a6502 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:30 +0200 Subject: [PATCH 068/102] s390/mm/pfault: Convert inline assembly to EX_TABLE_INSN Convert all pfault exception table entries to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so nopr padding is no longer needed and removed. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/mm/pfault.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/mm/pfault.c b/arch/s390/mm/pfault.c index 6ecd6b0a22a8..8f37741471e2 100644 --- a/arch/s390/mm/pfault.c +++ b/arch/s390/mm/pfault.c @@ -58,9 +58,9 @@ int __pfault_init(void) return rc; diag_stat_inc(DIAG_STAT_X258); asm_inline volatile( - " diag %[refbk],%[rc],0x258\n" - "0: nopr %%r7\n" - EX_TABLE(0b, 0b) + "0: diag %[refbk],%[rc],0x258\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [rc] "+d" (rc) : [refbk] "a" (virt_to_phys(&pfault_init_refbk)), "m" (pfault_init_refbk) : "cc"); @@ -80,9 +80,9 @@ void __pfault_fini(void) return; diag_stat_inc(DIAG_STAT_X258); asm_inline volatile( - " diag %[refbk],0,0x258\n" - "0: nopr %%r7\n" - EX_TABLE(0b, 0b) + "0: diag %[refbk],0,0x258\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : : [refbk] "a" (virt_to_phys(&pfault_fini_refbk)), "m" (pfault_fini_refbk) : "cc"); From aafb54c544f58cc306cf91fb7409138e7a5bb57b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:31 +0200 Subject: [PATCH 069/102] s390/cpu_mf: Convert inline assembly to EX_TABLE_INSN Convert all cpu_mf exception table entries to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so duplicate entries are removed. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/include/asm/cpu_mf.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/s390/include/asm/cpu_mf.h b/arch/s390/include/asm/cpu_mf.h index 1798fbd59068..dd2991622e82 100644 --- a/arch/s390/include/asm/cpu_mf.h +++ b/arch/s390/include/asm/cpu_mf.h @@ -173,9 +173,9 @@ static inline int qctri(struct cpumf_ctr_info *info) asm_inline volatile ( "0: qctri %1\n" - "1: lhi %0,0\n" - "2:\n" - EX_TABLE(1b, 2b) + " lhi %0,0\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : "+d" (rc), "=Q" (*info)); return rc; } @@ -257,9 +257,9 @@ static inline int qsi(struct hws_qsi_info_block *info) asm volatile( "0: qsi %1\n" - "1: lhi %0,0\n" - "2:\n" - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) + " lhi %0,0\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : "+d" (cc), "+Q" (*info)); return cc ? -EINVAL : 0; } @@ -272,10 +272,10 @@ static inline int lsctl(struct hws_lsctl_request_block *req) exception = 1; asm volatile( "0: lsctl %[req]\n" - "1: lhi %[exc],0\n" - "2:\n" + " lhi %[exc],0\n" + "1:\n" CC_IPM(cc) - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [exc] "+d" (exception) : [req] "Q" (*req) : CC_CLOBBER); From 3e8b505bfb5d635c04e6cbdc36f5330496b16a94 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:32 +0200 Subject: [PATCH 070/102] s390/cert_store: Convert inline assembly to EX_TABLE_INSN Convert the diag320 exception table entry to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so nopr padding is no longer needed and removed. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/kernel/cert_store.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/cert_store.c b/arch/s390/kernel/cert_store.c index dc1992a675de..998f2da56175 100644 --- a/arch/s390/kernel/cert_store.c +++ b/arch/s390/kernel/cert_store.c @@ -236,9 +236,9 @@ static int __diag320(unsigned long subcode, void *addr) union register_pair rp = { .even = (unsigned long)addr, }; asm_inline volatile( - " diag %[rp],%[subcode],0x320\n" - "0: nopr %%r7\n" - EX_TABLE(0b, 0b) + "0: diag %[rp],%[subcode],0x320\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [rp] "+d" (rp.pair) : [subcode] "d" (subcode) : "cc", "memory"); From d8d1a740a7c80a7bced35f66b4851ea10f391a16 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:33 +0200 Subject: [PATCH 071/102] s390/ipl: Convert inline assembly to EX_TABLE_INSN Convert the diag308 exception table entry to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so nopr padding is no longer needed and removed. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/kernel/ipl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index b1e798f8e1dd..c441b1357492 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -188,9 +188,9 @@ static inline int __diag308(unsigned long subcode, unsigned long addr) r1.even = addr; r1.odd = 0; asm_inline volatile( - " diag %[r1],%[subcode],0x308\n" - "0: nopr %%r7\n" - EX_TABLE(0b,0b) + "0: diag %[r1],%[subcode],0x308\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [r1] "+&d" (r1.pair) : [subcode] "d" (subcode) : "cc", "memory"); From 62d064756e0f0a62988a7fa3c3b952ef5d420d3d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:34 +0200 Subject: [PATCH 072/102] s390/hypfs: Convert inline assembly to EX_TABLE_INSN Convert the diag2fc exception table entry to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so nopr padding is no longer needed and removed. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/hypfs/hypfs_vm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/hypfs/hypfs_vm.c b/arch/s390/hypfs/hypfs_vm.c index 4db2895e4da3..bd47fce59731 100644 --- a/arch/s390/hypfs/hypfs_vm.c +++ b/arch/s390/hypfs/hypfs_vm.c @@ -42,9 +42,9 @@ static int diag2fc(int size, char* query, void *addr) diag_stat_inc(DIAG_STAT_X2FC); asm volatile( - " diag %0,%1,0x2fc\n" - "0: nopr %%r7\n" - EX_TABLE(0b,0b) + "0: diag %0,%1,0x2fc\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : "=d" (residual_cnt), "+d" (rc) : "0" (&parm_list) : "memory"); if ((rc != 0 ) && (rc != -2)) From b8bdc2c77755aff3daf06e488428932748ed8abb Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:35 +0200 Subject: [PATCH 073/102] s390/ap: Convert inline assembly to EX_TABLE_INSN Convert all ap exception table entries to use EX_TABLE_INSN. Reviewed-by: Harald Freudenberger Signed-off-by: Heiko Carstens --- arch/s390/include/asm/ap.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h index c91b6ace199d..b0f8e3946a43 100644 --- a/arch/s390/include/asm/ap.h +++ b/arch/s390/include/asm/ap.h @@ -91,10 +91,10 @@ static inline bool ap_instructions_available(void) " lgr 0,%[reg0]\n" /* qid into gr0 */ " lghi 1,0\n" /* 0 into gr1 */ " lghi 2,0\n" /* 0 into gr2 */ - " .insn rre,0xb2af0000,0,0\n" /* PQAP(TAPQ) */ - "0: la %[reg1],1\n" /* 1 into reg1 */ + "0: .insn rre,0xb2af0000,0,0\n" /* PQAP(TAPQ) */ + " la %[reg1],1\n" /* 1 into reg1 */ "1:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [reg1] "+&d" (reg1) : [reg0] "d" (reg0) : "cc", "0", "1", "2"); @@ -272,10 +272,10 @@ static inline int ap_qci(struct ap_config_info *config) asm volatile( " lgr 0,%[reg0]\n" /* QCI fc into gr0 */ " lgr 2,%[reg2]\n" /* ptr to config into gr2 */ - " .insn rre,0xb2af0000,0,0\n" /* PQAP(QCI) */ - "0: la %[reg1],0\n" /* good case, QCI fc available */ + "0: .insn rre,0xb2af0000,0,0\n" /* PQAP(QCI) */ + " la %[reg1],0\n" /* good case, QCI fc available */ "1:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [reg1] "+&d" (reg1) : [reg0] "d" (reg0), [reg2] "d" (reg2) : "cc", "memory", "0", "2"); From 558b4fe2572115878e4655fde7c1a610fffaba4d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:36 +0200 Subject: [PATCH 074/102] s390/fpu: Convert inline assembly to EX_TABLE_INSN Convert the fpu_lfpc_safe exception table entry to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so nopr padding is no longer needed and removed. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/include/asm/fpu-insn.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index ae8b7033cfd2..eeb0a233d5c7 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -104,9 +104,9 @@ static inline void fpu_lfpc_safe(unsigned int *fpc) { instrument_read(fpc, sizeof(*fpc)); asm_inline volatile( - " lfpc %[fpc]\n" - "0: nopr %%r7\n" - EX_TABLE_FPC(0b, 0b) + "0: lfpc %[fpc]\n" + "1:\n" + EX_TABLE_INSN_FPC(0b, 1b) : : [fpc] "Q" (*fpc) : "memory"); From 4ff587ae7fd92d6794a962f3776762d2de191c6b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:37 +0200 Subject: [PATCH 075/102] s390/traps: Convert inline assembly to EX_TABLE_INSN Convert the monitor call exception table entry to use EX_TABLE_INSN. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/kernel/traps.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index e35e0018f372..83fa4a794807 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -312,10 +312,10 @@ static void __init test_monitor_call(void) if (!IS_ENABLED(CONFIG_BUG)) return; asm_inline volatile( - " mc %[monc](%%r0),0\n" - "0: lhi %[val],0\n" + "0: mc %[monc](%%r0),0\n" + " lhi %[val],0\n" "1:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [val] "+d" (val) : [monc] "i" (MONCODE_BUG)); if (!val) From 345a429724ec783254aa76c10d60bbf588ed5b02 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:38 +0200 Subject: [PATCH 076/102] s390/word-at-a-time: Convert inline assembly to EX_TABLE_INSN Convert the load_unaligned_zeropad exception table entries to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so the duplicate entry and nopr padding are no longer needed and removed. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/include/asm/word-at-a-time.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index eaa19dee7699..0b7779c8d5a4 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -54,9 +54,8 @@ static inline unsigned long load_unaligned_zeropad(const void *addr) asm_inline volatile( "0: lg %[data],0(%[addr])\n" - "1: nopr %%r7\n" - EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) - EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) + "1:\n" + EX_TABLE_INSN_ZEROPAD(0b, 1b, %[data], %[addr]) : [data] "=d" (data) : [addr] "a" (addr), "m" (*(unsigned long *)addr)); return data; From 4b73aa1666f8414d7a09eafc7f1efd3e16c7ce9a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:39 +0200 Subject: [PATCH 077/102] s390/kvm: Convert inline assembly to EX_TABLE_INSN Convert the gaccess mvcos exception table entries to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so duplicate entries are removed. Acked-by: Claudio Imbrenda Signed-off-by: Heiko Carstens --- arch/s390/kvm/s390/gaccess.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/arch/s390/kvm/s390/gaccess.c b/arch/s390/kvm/s390/gaccess.c index e5c064f263df..5184bac7f552 100644 --- a/arch/s390/kvm/s390/gaccess.c +++ b/arch/s390/kvm/s390/gaccess.c @@ -838,12 +838,11 @@ static int mvcos_key(void *to, const void *from, unsigned long size, u8 dst_key, int exception = PGM_PROTECTION; asm_inline volatile( - " lr %%r0,%[spec]\n" - "0: mvcos %[to],%[from],%[size]\n" - "1: lhi %[exc],0\n" - "2:\n" - EX_TABLE(0b, 2b) - EX_TABLE(1b, 2b) + " lr %%r0,%[spec]\n" + "0: mvcos %[to],%[from],%[size]\n" + " lhi %[exc],0\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [size] "+d" (size), [to] "=Q" (*(char *)to), [exc] "+d" (exception) : [spec] "d" (spec.val), [from] "Q" (*(const char *)from) : "memory", "cc", "0"); From cb29b52dc5fb29cd6bb246ce85b43f3ed2eedfcd Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:40 +0200 Subject: [PATCH 078/102] s390/pci: Convert inline assembly to EX_TABLE_INSN Convert all pci exception table entries to use EX_TABLE_INSN. With EX_TABLE_INSN entries always point to the faulting instruction, so duplicate entries are removed. Reviewed-by: Niklas Schnelle Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_clp.c | 12 ++++++------ arch/s390/pci/pci_insn.c | 36 ++++++++++++++++++------------------ arch/s390/pci/pci_mmio.c | 28 +++++++++++++++------------- 3 files changed, 39 insertions(+), 37 deletions(-) diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c index 177aa0214547..2d7eaf4d9d59 100644 --- a/arch/s390/pci/pci_clp.c +++ b/arch/s390/pci/pci_clp.c @@ -55,11 +55,11 @@ static inline int clp_get_ilp(unsigned long *ilp) exception = 1; asm_inline volatile ( - " .insn rrf,0xb9a00000,%[mask],%[cmd],8,0\n" - "0: lhi %[exc],0\n" + "0: .insn rrf,0xb9a00000,%[mask],%[cmd],8,0\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [mask] "=d" (mask), [exc] "+d" (exception) : [cmd] "a" (1) : CC_CLOBBER); @@ -78,11 +78,11 @@ static __always_inline int clp_req(void *data, unsigned int lps) exception = 1; asm_inline volatile ( - " .insn rrf,0xb9a00000,%[ign],%[req],0,%[lps]\n" - "0: lhi %[exc],0\n" + "0: .insn rrf,0xb9a00000,%[ign],%[req],0,%[lps]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [ign] "=d" (ignored), "+m" (*req), [exc] "+d" (exception) : [req] "a" (req), [lps] "i" (lps) : CC_CLOBBER); diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c index 35ceb1bea1c6..ed0def408f18 100644 --- a/arch/s390/pci/pci_insn.c +++ b/arch/s390/pci/pci_insn.c @@ -161,11 +161,11 @@ static inline int ____pcilg(u64 *data, u64 req, u64 offset, u8 *status) exception = 1; asm_inline volatile ( - " .insn rre,0xb9d20000,%[data],%[req_off]\n" - "0: lhi %[exc],0\n" + "0: .insn rre,0xb9d20000,%[data],%[req_off]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [data] "=d" (__data), [req_off] "+d" (req_off.pair), [exc] "+d" (exception) : @@ -230,11 +230,11 @@ static inline int __pcilg_mio(u64 *data, u64 ioaddr, u64 len, u8 *status) exception = 1; asm_inline volatile ( - " .insn rre,0xb9d60000,%[data],%[ioaddr_len]\n" - "0: lhi %[exc],0\n" + "0: .insn rre,0xb9d60000,%[data],%[ioaddr_len]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [data] "=d" (__data), [ioaddr_len] "+d" (ioaddr_len.pair), [exc] "+d" (exception) : @@ -268,11 +268,11 @@ static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status) exception = 1; asm_inline volatile ( - " .insn rre,0xb9d00000,%[data],%[req_off]\n" - "0: lhi %[exc],0\n" + "0: .insn rre,0xb9d00000,%[data],%[req_off]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [req_off] "+d" (req_off.pair), [exc] "+d" (exception) : [data] "d" (data) : CC_CLOBBER); @@ -322,11 +322,11 @@ static inline int __pcistg_mio(u64 data, u64 ioaddr, u64 len, u8 *status) exception = 1; asm_inline volatile ( - " .insn rre,0xb9d40000,%[data],%[ioaddr_len]\n" - "0: lhi %[exc],0\n" + "0: .insn rre,0xb9d40000,%[data],%[ioaddr_len]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [ioaddr_len] "+d" (ioaddr_len.pair), [exc] "+d" (exception) : [data] "d" (data) : CC_CLOBBER_LIST("memory")); @@ -357,11 +357,11 @@ static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status) exception = 1; asm_inline volatile ( - " .insn rsy,0xeb00000000d0,%[req],%[offset],%[data]\n" - "0: lhi %[exc],0\n" + "0: .insn rsy,0xeb00000000d0,%[req],%[offset],%[data]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [req] "+d" (req), [exc] "+d" (exception) : [offset] "d" (offset), [data] "Q" (*data) : CC_CLOBBER); @@ -411,11 +411,11 @@ static inline int __pcistb_mio(const u64 *data, u64 ioaddr, u64 len, u8 *status) exception = 1; asm_inline volatile ( - " .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n" - "0: lhi %[exc],0\n" + "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [len] "+d" (len), [exc] "+d" (exception) : [ioaddr] "d" (ioaddr), [data] "Q" (*data) : CC_CLOBBER); diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index f3f79ba78410..8b49175b0031 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -39,10 +39,10 @@ static inline int __pcistb_mio_inuser( asm_inline volatile ( " sacf 256\n" "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n" - "1: lhi %[exc],0\n" - "2: sacf 768\n" + " lhi %[exc],0\n" + "1: sacf 768\n" CC_IPM(cc) - EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [len] "+d" (len), [exc] "+d" (exception) : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src)) : CC_CLOBBER_LIST("memory")); @@ -72,15 +72,16 @@ static inline int __pcistg_mio_inuser( asm_inline volatile ( " sacf 256\n" "0: llgc %[tmp],0(%[src])\n" - "4: sllg %[val],%[val],8\n" + " sllg %[val],%[val],8\n" " aghi %[src],1\n" " ogr %[val],%[tmp]\n" " brctg %[cnt],0b\n" "1: .insn rre,0xb9d40000,%[val],%[ioaddr_len]\n" - "2: lhi %[exc],0\n" - "3: sacf 768\n" + " lhi %[exc],0\n" + "2: sacf 768\n" CC_IPM(cc) - EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b) + EX_TABLE_INSN(0b, 2b) + EX_TABLE_INSN(1b, 2b) : [src] "+a" (src), [cnt] "+d" (cnt), [val] "+d" (val), [tmp] "=d" (tmp), [exc] "+d" (exception), CC_OUT(cc, cc), [ioaddr_len] "+&d" (ioaddr_len.pair) @@ -227,13 +228,13 @@ static inline int __pcilg_mio_inuser( asm_inline volatile ( " sacf 256\n" "0: .insn rre,0xb9d60000,%[val],%[ioaddr_len]\n" - "1: lhi %[exc],0\n" + " lhi %[exc],0\n" " jne 4f\n" - "2: ahi %[shift],-8\n" + "1: ahi %[shift],-8\n" " srlg %[tmp],%[val],0(%[shift])\n" - "3: stc %[tmp],0(%[dst])\n" - "5: aghi %[dst],1\n" - " brctg %[cnt],2b\n" + "2: stc %[tmp],0(%[dst])\n" + " aghi %[dst],1\n" + " brctg %[cnt],1b\n" /* * Use xr to clear exc and set condition code to zero * to ensure flag output is correct for this branch. @@ -241,7 +242,8 @@ static inline int __pcilg_mio_inuser( " xr %[exc],%[exc]\n" "4: sacf 768\n" CC_IPM(cc) - EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b) + EX_TABLE_INSN(0b, 4b) + EX_TABLE_INSN(2b, 4b) : [ioaddr_len] "+&d" (ioaddr_len.pair), [exc] "+d" (exception), CC_OUT(cc, cc), [val] "=d" (val), [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp), From a1e2c6e8be6cf81a27aa7c5281a1c0c7ce14e8ce Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:41 +0200 Subject: [PATCH 079/102] s390/boot: Convert inline assembly to EX_TABLE_INSN Convert all boot exception table entries to use EX_TABLE_INSN. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/boot/ipl_parm.c | 6 +++--- arch/s390/boot/physmem_info.c | 18 +++++++++--------- arch/s390/boot/startup.c | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index 637cb7b6343f..8bbd102fa50f 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -40,9 +40,9 @@ static inline int __diag308(unsigned long subcode, void *addr) union register_pair r1 = { .even = (unsigned long)addr, .odd = 0 }; asm_inline volatile( - " diag %[r1],%[subcode],0x308\n" - "0:\n" - EX_TABLE(0b, 0b) + "0: diag %[r1],%[subcode],0x308\n" + "1:\n" + EX_TABLE_INSN(0b, 1b) : [r1] "+d" (r1.pair) : [subcode] "d" (subcode) : "cc", "memory"); diff --git a/arch/s390/boot/physmem_info.c b/arch/s390/boot/physmem_info.c index 0ebb2174713f..e5248b637326 100644 --- a/arch/s390/boot/physmem_info.c +++ b/arch/s390/boot/physmem_info.c @@ -68,11 +68,11 @@ static int __diag260(unsigned long rx1, unsigned long rx2) ry = 0x10; /* storage configuration */ exception = 1; asm_inline volatile( - " diag %[rx],%[ry],0x260\n" - "0: lhi %[exc],0\n" + "0: diag %[rx],%[ry],0x260\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [exc] "+d" (exception), [ry] "+d" (ry) : [rx] "d" (rx.pair) : CC_CLOBBER_LIST("memory")); @@ -108,9 +108,9 @@ static int diag500_storage_limit(unsigned long *max_physmem_end) asm_inline volatile( " lghi %%r1,%[subcode]\n" " lghi %%r2,0\n" - " diag %%r2,%%r4,0x500\n" - "0: lgr %[slimit],%%r2\n" - EX_TABLE(0b, 0b) + "0: diag %%r2,%%r4,0x500\n" + "1: lgr %[slimit],%%r2\n" + EX_TABLE_INSN(0b, 1b) : [slimit] "=d" (storage_limit) : [subcode] "i" (DIAG500_SC_STOR_LIMIT) : "memory", "1", "2"); @@ -127,11 +127,11 @@ static int tprot(unsigned long addr) exception = 1; asm_inline volatile( - " tprot 0(%[addr]),0\n" - "0: lhi %[exc],0\n" + "0: tprot 0(%[addr]),0\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [exc] "+d" (exception) : [addr] "a" (addr) : CC_CLOBBER_LIST("memory")); diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index c1dcf8b1b579..5eb7a3e1f9f1 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -89,10 +89,10 @@ static void detect_diag9c(void) cpu = stap(); asm_inline volatile( - " diag %[cpu],%%r0,0x9c\n" - "0: lhi %[rc],0\n" + "0: diag %[cpu],%%r0,0x9c\n" + " lhi %[rc],0\n" "1:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [rc] "+d" (rc) : [cpu] "d" (cpu) : "cc", "memory"); @@ -150,10 +150,10 @@ static int cmma_test_essa(void) /* Test ESSA_GET_STATE */ asm_inline volatile( - " .insn rrf,0xb9ab0000,%[tmp],%[tmp],%[cmd],0\n" - "0: lhi %[rc],0\n" + "0: .insn rrf,0xb9ab0000,%[tmp],%[tmp],%[cmd],0\n" + " lhi %[rc],0\n" "1:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [rc] "+d" (rc), [tmp] "+d" (tmp) : [cmd] "i" (ESSA_GET_STATE) : "cc", "memory"); From 7e27bba542791cd553f2d3a5720092c86e3fc62c Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:42 +0200 Subject: [PATCH 080/102] s390/cio: Convert inline assembly to EX_TABLE_INSN Convert all cio exception table entries to use EX_TABLE_INSN. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- drivers/s390/cio/ioasm.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/s390/cio/ioasm.c b/drivers/s390/cio/ioasm.c index 8b06b234e110..4974e80b0568 100644 --- a/drivers/s390/cio/ioasm.c +++ b/drivers/s390/cio/ioasm.c @@ -24,11 +24,11 @@ static inline int __stsch(struct subchannel_id schid, struct schib *addr) exception = 1; asm_inline volatile( " lgr 1,%[r1]\n" - " stsch %[addr]\n" - "0: lhi %[exc],0\n" + "0: stsch %[addr]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, ccode), [addr] "=Q" (*addr), [exc] "+d" (exception) : [r1] "d" (r1) : CC_CLOBBER_LIST("1")); @@ -54,11 +54,11 @@ static inline int __msch(struct subchannel_id schid, struct schib *addr) exception = 1; asm_inline volatile( " lgr 1,%[r1]\n" - " msch %[addr]\n" - "0: lhi %[exc],0\n" + "0: msch %[addr]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, ccode), [exc] "+d" (exception) : [r1] "d" (r1), [addr] "Q" (*addr) : CC_CLOBBER_LIST("1")); @@ -108,11 +108,11 @@ static inline int __ssch(struct subchannel_id schid, union orb *addr) exception = 1; asm_inline volatile( " lgr 1,%[r1]\n" - " ssch %[addr]\n" - "0: lhi %[exc],0\n" + "0: ssch %[addr]\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, ccode), [exc] "+d" (exception) : [r1] "d" (r1), [addr] "Q" (*addr) : CC_CLOBBER_LIST("memory", "1")); @@ -179,11 +179,11 @@ int chsc(void *chsc_area) exception = 1; asm_inline volatile( - " .insn rre,0xb25f0000,%[chsc_area],0\n" - "0: lhi %[exc],0\n" + "0: .insn rre,0xb25f0000,%[chsc_area],0\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), "+m" (*(addr_type *)chsc_area), [exc] "+d" (exception) : [chsc_area] "d" (chsc_area) : CC_CLOBBER); From 4193d7529b90ea26ad68ea2ecb3c2a0b76ee39cf Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:43 +0200 Subject: [PATCH 081/102] s390/dasd: Convert inline assembly to EX_TABLE_INSN Convert the dasd_diag exception table entry to use EX_TABLE_INSN. Acked-by: Stefan Haberland Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- drivers/s390/block/dasd_diag.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/s390/block/dasd_diag.c b/drivers/s390/block/dasd_diag.c index af68c2ee659d..ce391d35adf9 100644 --- a/drivers/s390/block/dasd_diag.c +++ b/drivers/s390/block/dasd_diag.c @@ -77,11 +77,11 @@ static inline int __dia250(void *iob, int cmd) exception = 1; asm_inline volatile( - " diag %[rx],%[cmd],0x250\n" - "0: lhi %[exc],0\n" + "0: diag %[rx],%[cmd],0x250\n" + " lhi %[exc],0\n" "1:\n" CC_IPM(cc) - EX_TABLE(0b,1b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [rx] "+d" (rx.pair), "+m" (*(addr_type *)iob), [exc] "+d" (exception) : [cmd] "d" (cmd) From 1faf2c15b9df7caa5757ffbce75518cfd35f9b2b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:44 +0200 Subject: [PATCH 082/102] s390/diag_ftp: Convert inline assembly to EX_TABLE_INSN Convert the diag_ftp exception table entry to use EX_TABLE_INSN. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- drivers/s390/char/diag_ftp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/s390/char/diag_ftp.c b/drivers/s390/char/diag_ftp.c index a1e110c96f74..0a506559799e 100644 --- a/drivers/s390/char/diag_ftp.c +++ b/drivers/s390/char/diag_ftp.c @@ -106,11 +106,11 @@ static int diag_ftp_2c4(struct diag_ftp_ldfpl *fpl, diag_stat_inc(DIAG_STAT_X2C4); asm_inline volatile( - " diag %[addr],%[cmd],0x2c4\n" - "0: j 2f\n" + "0: diag %[addr],%[cmd],0x2c4\n" + " j 2f\n" "1: la %[rc],%[err]\n" "2:\n" - EX_TABLE(0b, 1b) + EX_TABLE_INSN(0b, 1b) : [rc] "=d" (rc), "+m" (*fpl) : [cmd] "0" (cmd), [addr] "d" (virt_to_phys(fpl)), [err] "i" (DIAG_FTP_RET_EPERM) From fcee842c0674c988255e906e8201c01659438f37 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:45 +0200 Subject: [PATCH 083/102] s390/sclp: Convert inline assembly to EX_TABLE_INSN Convert the sclp_service_call exception table entries to use EX_TABLE_INSN. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- drivers/s390/char/sclp.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/s390/char/sclp.h b/drivers/s390/char/sclp.h index b31a680e0871..b91d31f8e0ef 100644 --- a/drivers/s390/char/sclp.h +++ b/drivers/s390/char/sclp.h @@ -320,11 +320,10 @@ static inline int sclp_service_call(sclp_cmdw_t command, void *sccb) exception = 1; asm_inline volatile( "0: .insn rre,0xb2200000,%[cmd],%[sccb]\n" /* servc */ - "1: lhi %[exc],0\n" - "2:\n" + " lhi %[exc],0\n" + "1:\n" CC_IPM(cc) - EX_TABLE(0b, 2b) - EX_TABLE(1b, 2b) + EX_TABLE_INSN(0b, 1b) : CC_OUT(cc, cc), [exc] "+d" (exception) : [cmd] "d" (command), [sccb] "a" (__pa(sccb)) : CC_CLOBBER_LIST("memory")); From aeca9e9d71a595b8f716d4a42ab5b0c7272c4862 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:46 +0200 Subject: [PATCH 084/102] s390/bpf: Simplify exception table entry handling With exception table lookup always pointing to the faulting instruction, only a single exception table entry per probe instruction is needed. Remove the second exception table entry, the extra nop instruction emitted between/after probe instructions, and simplify probe handling in the BPF JIT. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/net/bpf_jit_comp.c | 78 ++++++++++-------------------------- 1 file changed, 21 insertions(+), 57 deletions(-) diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index c4b47070bb59..9f1632e068e1 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -723,7 +723,6 @@ bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) */ struct bpf_jit_probe { int prg; /* JITed instruction offset */ - int nop_prg; /* JITed nop offset */ int reg; /* Register to clear on exception */ int arena_reg; /* Register to use for arena addressing */ }; @@ -731,29 +730,10 @@ struct bpf_jit_probe { static void bpf_jit_probe_init(struct bpf_jit_probe *probe) { probe->prg = -1; - probe->nop_prg = -1; probe->reg = -1; probe->arena_reg = REG_0; } -/* - * Handlers of certain exceptions leave psw.addr pointing to the instruction - * directly after the failing one. Therefore, create two exception table - * entries and also add a nop in case two probing instructions come directly - * after each other. - */ -static void bpf_jit_probe_emit_nop(struct bpf_jit *jit, - struct bpf_jit_probe *probe) -{ - if (probe->prg == -1 || probe->nop_prg != -1) - /* The probe is not armed or nop is already emitted. */ - return; - - probe->nop_prg = jit->prg; - /* bcr 0,%0 */ - _EMIT2(0x0700); -} - static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, struct bpf_jit_probe *probe) { @@ -815,47 +795,36 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_jit_probe *probe) { struct exception_table_entry *ex; - int i, prg; s64 delta; - u8 *insn; if (probe->prg == -1) /* The probe is not armed. */ return 0; - bpf_jit_probe_emit_nop(jit, probe); if (!fp->aux->extable) /* Do nothing during early JIT passes. */ return 0; - insn = jit->prg_buf + probe->prg; - if (WARN_ON_ONCE(probe->prg + insn_length(*insn) != probe->nop_prg)) - /* JIT bug - gap between probe and nop instructions. */ + if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries)) + /* Verifier bug - not enough entries. */ return -1; - for (i = 0; i < 2; i++) { - if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries)) - /* Verifier bug - not enough entries. */ - return -1; - ex = &fp->aux->extable[jit->excnt]; - /* Add extable entries for probe and nop instructions. */ - prg = i == 0 ? probe->prg : probe->nop_prg; - delta = jit->prg_buf + prg - (u8 *)&ex->insn; - if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) - /* JIT bug - code and extable must be close. */ - return -1; - ex->insn = delta; - /* - * Land on the current instruction. Note that the extable - * infrastructure ignores the fixup field; it is handled by - * ex_handler_bpf(). - */ - delta = jit->prg_buf + jit->prg - (u8 *)&ex->fixup; - if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) - /* JIT bug - landing pad and extable must be close. */ - return -1; - ex->fixup = delta; - ex->type = EX_TYPE_BPF; - ex->data = probe->reg; - jit->excnt++; - } + ex = &fp->aux->extable[jit->excnt]; + delta = jit->prg_buf + probe->prg - (u8 *)&ex->insn; + if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) + /* JIT bug - code and extable must be close. */ + return -1; + ex->insn = delta; + /* + * Land on the current instruction. Note that the extable + * infrastructure ignores the fixup field; it is handled by + * ex_handler_bpf(). + */ + delta = jit->prg_buf + jit->prg - (u8 *)&ex->fixup; + if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX)) + /* JIT bug - landing pad and extable must be close. */ + return -1; + ex->fixup = delta; + ex->type = EX_TYPE_BPF; + ex->data = probe->reg; + jit->excnt++; return 0; } @@ -1705,7 +1674,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, EMIT6_DISP_LH(0xe3000000, is32 ? 0x0058 : 0x0004, REG_W0, REG_0, load_probe.arena_reg, off); - bpf_jit_probe_emit_nop(jit, &load_probe); /* Reuse {ly|lg}'s arena_reg for {csy|csg}. */ if (load_probe.prg != -1) { probe.prg = jit->prg; @@ -1716,7 +1684,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* 0: {csy|csg} %w0,%src,off(%arena) */ EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, REG_W0, src_reg, probe.arena_reg, off); - bpf_jit_probe_emit_nop(jit, &probe); /* brc 4,0b */ EMIT4_PCREL_RIC(0xa7040000, 4, loop_start); /* {llgfr|lgr} %src,%w0 */ @@ -2364,9 +2331,6 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, */ fp->aux->num_exentries += 1; } - /* We need two entries per insn. */ - fp->aux->num_exentries *= 2; - code_size = roundup(jit->size, __alignof__(struct exception_table_entry)); extable_size = fp->aux->num_exentries * From 6d86f714aed7287e15178dee949d4b64bf5d7ed2 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 1 Oct 2026 12:32:47 +0200 Subject: [PATCH 085/102] s390/extable: Remove EX_TABLE macros and fallback extable search Now that all exception table entries point to the faulting instruction, rename all EX_TABLE* macros to EX_TABLE_INSN* and remove the EX_TABLE_INSN* aliases. Also remove the fallback extable search via instruction_pointer() which was only needed to handle the old EX_TABLE entries that pointed to the instruction after the faulting instruction. Add a comment above the EX_TABLE_INSN macros that documents the usage rule and describe the difference to the previous EX_TABLE macros. This might be helpful in case of backporting to old kernels. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/boot/pgm_check.c | 23 +++++------------ arch/s390/include/asm/asm-extable.h | 38 ++++++++++++++--------------- arch/s390/include/asm/extable.h | 25 ++++++------------- arch/s390/mm/extable.c | 2 -- 4 files changed, 32 insertions(+), 56 deletions(-) diff --git a/arch/s390/boot/pgm_check.c b/arch/s390/boot/pgm_check.c index 56d0be7e2823..e5a454b21669 100644 --- a/arch/s390/boot/pgm_check.c +++ b/arch/s390/boot/pgm_check.c @@ -51,31 +51,20 @@ static unsigned long boot_get_trap_ip(struct pt_regs *regs) return __rewind_psw(regs->psw, regs->int_code >> 16); } -static const struct exception_table_entry *search_boot_extable(unsigned long ip) +static bool ex_handler(struct pt_regs *regs) { const struct exception_table_entry *ex; + unsigned long ip = boot_get_trap_ip(regs); for (ex = __start___ex_table; ex < __stop___ex_table; ex++) { if (extable_insn(ex) != ip) continue; if (ex->type != EX_TYPE_FIXUP) - return NULL; - return ex; + return false; + regs->psw.addr = extable_fixup(ex); + return true; } - return NULL; -} - -static bool ex_handler(struct pt_regs *regs) -{ - const struct exception_table_entry *ex; - - ex = search_boot_extable(boot_get_trap_ip(regs)); - if (!ex) - ex = search_boot_extable(instruction_pointer(regs)); - if (!ex) - return false; - regs->psw.addr = extable_fixup(ex); - return true; + return false; } void do_pgm_check(struct pt_regs *regs) diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index 488bd10452b3..fd43ecd3e491 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -65,41 +65,41 @@ stringify_in_c(.purgem extable_reg;) \ stringify_in_c(.previous) -#define EX_TABLE(_fault, _target) \ +/* + * EX_TABLE_INSN usage rules: + * + * _fault must point to the faulting instruction, never behind it. + * + * This differs from the former EX_TABLE convention, where usually two entries + * were needed. One that pointed to the faulting instruction (for nullifying + * exceptions), and a second one that pointed behind the instruction (for + * suppressing, completing, and terminating exceptions). + */ +#define EX_TABLE_INSN(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP, __stringify(%%r0), __stringify(%%r0), 0) -#define EX_TABLE_AMODE31(_fault, _target) \ +#define EX_TABLE_INSN_AMODE31(_fault, _target) \ __EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP, __stringify(%%r0), __stringify(%%r0), 0) -#define EX_TABLE_UA_FAULT(_fault, _target, _regerr) \ +#define EX_TABLE_INSN_UA_FAULT(_fault, _target, _regerr) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_FAULT, _regerr, _regerr, 0) -#define EX_TABLE_UA_LOAD_REG(_fault, _target, _regerr, _regzero) \ +#define EX_TABLE_INSN_UA_LOAD_REG(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REG, _regerr, _regzero, 0) -#define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ +#define EX_TABLE_INSN_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) -#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ +#define EX_TABLE_INSN_ZEROPAD(_fault, _target, _regdata, _regaddr) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) -#define EX_TABLE_FPC(_fault, _target) \ +#define EX_TABLE_INSN_FPC(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0) -#define EX_TABLE_UA_MVCOS_TO(_fault, _target) \ +#define EX_TABLE_INSN_UA_MVCOS_TO(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_MVCOS_TO, __stringify(%%r0), __stringify(%%r0), 0) -#define EX_TABLE_UA_MVCOS_FROM(_fault, _target) \ +#define EX_TABLE_INSN_UA_MVCOS_FROM(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_MVCOS_FROM, __stringify(%%r0), __stringify(%%r0), 0) -#define EX_TABLE_INSN EX_TABLE -#define EX_TABLE_INSN_AMODE31 EX_TABLE_AMODE31 -#define EX_TABLE_INSN_UA_FAULT EX_TABLE_UA_FAULT -#define EX_TABLE_INSN_UA_LOAD_REG EX_TABLE_UA_LOAD_REG -#define EX_TABLE_INSN_UA_LOAD_REGPAIR EX_TABLE_UA_LOAD_REGPAIR -#define EX_TABLE_INSN_ZEROPAD EX_TABLE_ZEROPAD -#define EX_TABLE_INSN_FPC EX_TABLE_FPC -#define EX_TABLE_INSN_UA_MVCOS_TO EX_TABLE_UA_MVCOS_TO -#define EX_TABLE_INSN_UA_MVCOS_FROM EX_TABLE_UA_MVCOS_FROM - #endif /* __ASM_EXTABLE_H */ diff --git a/arch/s390/include/asm/extable.h b/arch/s390/include/asm/extable.h index af6ba52743e9..5032c648781b 100644 --- a/arch/s390/include/asm/extable.h +++ b/arch/s390/include/asm/extable.h @@ -5,25 +5,14 @@ #include #include -/* - * The exception table consists of three addresses: - * - * - Address of an instruction that is allowed to fault. - * - Address at which the program should continue. - * - Optional address of handler that takes pt_regs * argument and runs in - * interrupt context. - * - * No registers are modified, so it is entirely up to the continuation code - * to figure out what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. +/** + * struct exception_table_entry - entry in exception table + * @insn: Relative pointer to faulting instruction + * @fixup: Relative pointer to instruction where execution resumes + * @type: Exception type + * @data: handler-specific data */ - -struct exception_table_entry -{ +struct exception_table_entry { int insn, fixup; short type, data; }; diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index 3d2ad6e853d1..d0a1ded21520 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -121,8 +121,6 @@ bool fixup_exception(struct pt_regs *regs) const struct exception_table_entry *ex; ex = s390_search_extables(get_trap_ip(regs)); - if (!ex) - ex = s390_search_extables(instruction_pointer(regs)); if (!ex) return false; switch (ex->type) { From 10f9cf286ec3e6f99569e5ce3e99afbeb77b962f Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 12 Oct 2026 13:42:46 +0200 Subject: [PATCH 086/102] s390/percpu: Use asm_inline for arch_this_cpu_add() When converting to preemptible percpu operations the arch_this_cpu_add() inline assemblies were not converted to use asm_inline instead of a plain asm statement. With all the macro usage the inline assemblies grow significantly, and the compiler may decide to not inline even small functions. This can be observed with gcc16 and a defconfig build where many calls to __srcu_read_lock_fast() are not inlined anymore, even though it is basically only a this_cpu_inc(). Use asm_inline to address this. Fixes: 18ec6c5d7ec6 ("s390/percpu: Use new percpu code section for arch_this_cpu_add()") Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index c3995bd8d1de..dfab41b43b8b 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -193,7 +193,7 @@ do { \ ptr__ = PERCPU_PTR(&(pcp)); \ if (__builtin_constant_p(val__) && \ ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ - asm volatile( \ + asm_inline volatile( \ __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ "%[pair__]","%N[pair__]") \ op2 " 0(%N[pair__]),%[val__]\n" \ @@ -206,7 +206,7 @@ do { \ "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ } else { \ - asm volatile( \ + asm_inline volatile( \ __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ "%[pair__]","%N[pair__]") \ op1 " %[old__],%[val__],0(%N[pair__])\n" \ From 3681abf4bfa29983ff6ff6a0f6b6e411d58337d4 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 12 Oct 2026 13:42:47 +0200 Subject: [PATCH 087/102] s390/percpu: Drop unused output register from arch_this_cpu_add() The laa/laag instructions return the old memory content, but arch_this_cpu_add() does not need it. Instead of allocating a dedicated output register old__ just to discard the returned value, change the val__ constraint from "d" to "+&d" so it is used as input/output register. This reduces register pressure. Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index dfab41b43b8b..fd0a3a9951ce 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -188,7 +188,7 @@ do { \ typedef typeof(pcp) pcp_op_T__; \ union register_pair rp__; \ pcp_op_T__ val__ = (val); \ - pcp_op_T__ old__, *ptr__; \ + pcp_op_T__ *ptr__; \ \ ptr__ = PERCPU_PTR(&(pcp)); \ if (__builtin_constant_p(val__) && \ @@ -209,12 +209,12 @@ do { \ asm_inline volatile( \ __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ "%[pair__]","%N[pair__]") \ - op1 " %[old__],%[val__],0(%N[pair__])\n" \ + op1 " %[val__],%[val__],0(%N[pair__])\n" \ __PCPU_END("%[lcreg]") \ - : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \ + : [val__] "+&d" (val__), [pair__] "=&a" (rp__.pair), \ "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "d" (val__), [ptr__] "a" (ptr__), \ + : [ptr__] "a" (ptr__), \ [lcreg] "i" (LC_PERCPU_REGISTER), \ [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ From 5af4c564803cd080361e9b99593c9cd6552cb73e Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 12 Oct 2026 13:42:48 +0200 Subject: [PATCH 088/102] s390/percpu: Drop unused output register from arch_this_cpu_to_op() The lan/lang/lao/laog instructions return the old memory content, but arch_this_cpu_to_op() does not need it. Instead of allocating a dedicated output register old__ just to discard the returned value, change the val__ constraint from "d" to "+&d" so it is used as input/output register. This reduces register pressure. Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index fd0a3a9951ce..7d87e0a262c3 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -257,18 +257,18 @@ do { \ typedef typeof(pcp) pcp_op_T__; \ union register_pair rp__; \ pcp_op_T__ val__ = (val); \ - pcp_op_T__ old__, *ptr__; \ + pcp_op_T__ *ptr__; \ \ ptr__ = PERCPU_PTR(&(pcp)); \ asm_inline volatile( \ __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ "%[pair__]","%N[pair__]") \ - op " %[old__],%[val__],0(%N[pair__])\n" \ + op " %[val__],%[val__],0(%N[pair__])\n" \ __PCPU_END("%[lcreg]") \ - : [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair), \ + : [val__] "+&d" (val__), [pair__] "=&a" (rp__.pair), \ "+m" (*ptr__), \ "=m" (((struct lowcore *)0)->percpu_register) \ - : [val__] "d" (val__), [ptr__] "a" (ptr__), \ + : [ptr__] "a" (ptr__), \ [lcreg] "i" (LC_PERCPU_REGISTER), \ [lcoff] "i" (LC_PERCPU_OFFSET), \ "m" (((struct lowcore *)0)->percpu_offset) \ From 8f4778cf7f76e0da395b8015a25953f3ce05ab16 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 12 Oct 2026 13:42:49 +0200 Subject: [PATCH 089/102] s390/percpu: Implement this_cpu_try_cmpxchg() Provide an s390 specific implementation of this_cpu_try_cmpxchg() for the four and eight byte cases, if the compiler supports flag output operands. This variant uses the condition code set by cs/csg, which avoids a compare instruction required by the generic variant and generates slightly better code. Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/include/asm/percpu.h | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 7d87e0a262c3..7b7a8cb51daa 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -5,6 +5,7 @@ #include #include #include +#include /* * s390 uses its own implementation for per cpu data, the offset of @@ -404,6 +405,44 @@ do { \ old__; \ }) +#ifdef __HAVE_ASM_FLAG_OUTPUTS__ + +#define arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, op) \ +({ \ + typedef typeof(pcp) pcp_op_T__; \ + typeof(ovalp) ovalp__ = (ovalp); \ + pcp_op_T__ old__ = *ovalp__; \ + pcp_op_T__ new__ = (nval); \ + union register_pair rp__; \ + pcp_op_T__ *ptr__; \ + int cc__; \ + \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + __PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]", \ + "%[pair__]","%N[pair__]") \ + op " %[old__],%[new__],0(%N[pair__])\n" \ + __PCPU_END("%[lcreg]") \ + : [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair), \ + "+m" (*ptr__), "=@cc" (cc__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [new__] "d" (new__), [ptr__] "a" (ptr__), \ + [lcreg] "i" (LC_PERCPU_REGISTER), \ + [lcoff] "i" (LC_PERCPU_OFFSET), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "memory"); \ + if (unlikely(cc__)) \ + *ovalp__ = old__; \ + likely(cc__ == 0); \ +}) + +#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, "cs") +#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) arch_this_cpu_try_cmpxchg(pcp, ovalp, nval, "csg") + +#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) this_cpu_try_cmpxchg_8(pcp, ovalp, nval) + +#endif /* __HAVE_ASM_FLAG_OUTPUTS__ */ + #define arch_this_cpu_xchg_simple(pcp, nval) \ ({ \ typeof(pcp) *ptr__; \ From 737f5b4eaaa54edeb537687225ef14af2bde2589 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 2 Oct 2026 17:20:54 +0200 Subject: [PATCH 090/102] s390/entry: Last breaking event vs invalid kernel stack In case an invalid kernel stack is detected the last breaking event address is copied to the orig_gpr2 member of the generated pt_regs structure. Correct location would be last_break. Fix this. This adds help information to the printed messages: Before: Kernel stack pointer invalid ... Last Breaking-Event-Address: [<0000000000000000>] 0x0 After: Kernel stack pointer invalid ... Last Breaking-Event-Address: [<000001b802c1ae1c>] proc_reg_write+0x5c/0xd0 Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/kernel/entry.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index f51ecd8e7d21..459de56fb608 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -623,7 +623,7 @@ SYM_CODE_START(stack_invalid) mvc __PT_R8(64,%r11),0(%r14) xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) GET_LC %r2 - mvc __PT_ORIG_GPR2(8,%r11),__LC_PGM_LAST_BREAK(%r2) + mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK(%r2) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) lgr %r2,%r11 # pass pointer to pt_regs jg kernel_stack_invalid From 5cc8fe84f91175af10825d08c9b6fc88426d83ae Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 2 Oct 2026 17:20:55 +0200 Subject: [PATCH 091/102] s390/entry: Save lowcore on invalid kernel stack In case of an invalid kernel stack copy the current CPU's lowcore to stack_invalid_lowcore, so it is available to debug tools after a crash. The copy is helpful since the lowcore is modified e.g., when printing messages before a dump is taken. Keep it as unmodified as possible and as close to the event that caused register corruption. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/kernel/entry.S | 14 ++++++++++++++ arch/s390/kernel/entry.h | 4 ++++ arch/s390/kernel/traps.c | 6 ++++++ 3 files changed, 24 insertions(+) diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 459de56fb608..4242cd7ab2fe 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -613,6 +613,8 @@ SYM_CODE_END(early_pgm_check_handler) * The synchronous or the asynchronous stack pointer is invalid. We are dead. * No need to properly save the registers, we are going to panic anyway. * Setup a pt_regs so that show_trace can provide a good call trace. + * Copy lowcore of the current CPU to stack_invalid_lowcore before it + * is modified by printing debug messages and crash handling. */ SYM_CODE_START(stack_invalid) GET_LC %r15 @@ -624,6 +626,18 @@ SYM_CODE_START(stack_invalid) xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) GET_LC %r2 mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK(%r2) + /* copy lowcore; skip if used by another CPU */ + larl %r4,stack_invalid_lowcore_used + lhi %r1,0 + lhi %r3,1 + cs %r1,%r3,0(%r4) + jne .Lskip + larl %r0,stack_invalid_lowcore + lghi %r1,LC_PAGES * PAGE_SIZE + lgr %r4,%r2 + lgr %r5,%r1 + mvcl %r0,%r4 +.Lskip: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) lgr %r2,%r11 # pass pointer to pt_regs jg kernel_stack_invalid diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index dc84ab497251..5d6685e5bf5f 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -64,6 +65,9 @@ long sys_s390_sthyi(unsigned long function_code, void __user *buffer, u64 __user unsigned long stack_alloc(void); void stack_free(unsigned long stack); +extern int stack_invalid_lowcore_used; +extern struct lowcore stack_invalid_lowcore; + extern char kprobes_insn_page[]; extern char _samode31[], _eamode31[]; diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 83fa4a794807..6b083b9584eb 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -290,6 +290,12 @@ static void monitor_event_exception(struct pt_regs *regs) } } +/* + * Content of the faulting CPU's lowcore, copied by stack_invalid(). + */ +struct lowcore stack_invalid_lowcore; +int stack_invalid_lowcore_used; + void kernel_stack_invalid(struct pt_regs *regs) { /* From cfadcb9c0fb6025950c2891b808b0e23efa827e6 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Tue, 13 Oct 2026 00:04:02 +0900 Subject: [PATCH 092/102] s390/sclp: Remove redundant memset after memblock_alloc_low() memblock_alloc_low() returns zeroed memory, so the memset after the allocation in sclp_early_get_core_info() is redundant. So remove the memset. Signed-off-by: Sang-Heon Jeon Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- drivers/s390/char/sclp_early.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index 22dd797e6229..821b61727d1f 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -138,7 +138,6 @@ int __init sclp_early_get_core_info(struct sclp_core_info *info) if (!sccb) return -ENOMEM; - memset(sccb, 0, length); sccb->header.length = length; sccb->header.control_mask[2] = 0x80; if (sclp_early_cmd(SCLP_CMDW_READ_CPU_INFO, sccb)) { From 9bfcede4926054b8ee154e92e30dfabecf238ca4 Mon Sep 17 00:00:00 2001 From: Gerd Bayer Date: Mon, 12 Oct 2026 16:21:49 +0200 Subject: [PATCH 093/102] s390/pci: Adjust alignment in insn trace Currently we have 64bit wide fields start at odd byte-offsets in the insn traces for pci_error in the s390dbf debug area, which are hard to decode in hex dumps. At the same time, trace entries are padded at the end. Drop the packed attribute on the struct definition to exploit the compiler's natural alignment, introduce explicit padding before and after the 64bit fields for humans, and add a static assertion to avoid accidental overruns of 2 lines in the s390dbf trace. Signed-off-by: Gerd Bayer Reviewed-by: Benjamin Block Reviewed-by: Matthew Rosato Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_insn.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c index ed0def408f18..6afabc5b68aa 100644 --- a/arch/s390/pci/pci_insn.c +++ b/arch/s390/pci/pci_insn.c @@ -18,11 +18,13 @@ #include #define ZPCI_INSN_BUSY_DELAY 1 /* 1 microsecond */ +#define ZPCI_ERR_INSN_DATA_SIZE 32 struct zpci_err_insn_data { u8 insn; u8 cc; u8 status; + u8 pad0[5]; union { struct { u64 req; @@ -33,7 +35,10 @@ struct zpci_err_insn_data { u64 len; }; }; -} __packed; + u64 pad1; +}; + +static_assert(sizeof(struct zpci_err_insn_data) <= ZPCI_ERR_INSN_DATA_SIZE); static inline void zpci_err_insn_req(int lvl, u8 insn, u8 cc, u8 status, u64 req, u64 offset) From 75c1c3b702910be7b15ad1145a97180e060b042c Mon Sep 17 00:00:00 2001 From: Gerd Bayer Date: Mon, 12 Oct 2026 16:21:50 +0200 Subject: [PATCH 094/102] s390/pci: Use 32-bit fh outside of inline asm zpci_refresh_trans() declared the PCI function handle as a u64 type just because the actual RPCIT machine instruction required that in a 64bit register, while the function handle is architected as a 32bit type and defined as such in the zpci_dev struct. Push the extension from u32 to u64 down in the call chain to just before calling the RPCIT wrapper function. Since the struct of the typical req's matches the format expected by RPCIT, too, create one of those to pass along. Signed-off-by: Gerd Bayer Reviewed-by: Niklas Schnelle Reviewed-by: Matthew Rosato Signed-off-by: Heiko Carstens --- arch/s390/include/asm/pci_insn.h | 2 +- arch/s390/pci/pci_insn.c | 5 +++-- drivers/iommu/s390-iommu.c | 8 +++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/arch/s390/include/asm/pci_insn.h b/arch/s390/include/asm/pci_insn.h index 025c6dcbf893..e6613b037f93 100644 --- a/arch/s390/include/asm/pci_insn.h +++ b/arch/s390/include/asm/pci_insn.h @@ -147,7 +147,7 @@ union zpci_sic_iib { DECLARE_STATIC_KEY_FALSE(have_mio); u8 zpci_mod_fc(u64 req, struct zpci_fib *fib, u8 *status); -int zpci_refresh_trans(u64 fn, u64 addr, u64 range); +int zpci_refresh_trans(u32 fh, u64 addr, u64 range); int __zpci_load(u64 *data, u64 req, u64 offset); int zpci_load(u64 *data, const volatile void __iomem *addr, unsigned long len); int __zpci_store(u64 data, u64 req, u64 offset); diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c index 6afabc5b68aa..b65135a94642 100644 --- a/arch/s390/pci/pci_insn.c +++ b/arch/s390/pci/pci_insn.c @@ -116,13 +116,14 @@ static inline u8 __rpcit(u64 fn, u64 addr, u64 range, u8 *status) return CC_TRANSFORM(cc); } -int zpci_refresh_trans(u64 fn, u64 addr, u64 range) +int zpci_refresh_trans(u32 fh, u64 addr, u64 range) { + u64 req = ZPCI_CREATE_REQ(fh, 0, 0); bool retried = false; u8 cc, status; do { - cc = __rpcit(fn, addr, range, &status); + cc = __rpcit(req, addr, range, &status); if (cc == 2) { udelay(ZPCI_INSN_BUSY_DELAY); if (!retried) { diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c index f148f559ac56..77c195fc752f 100644 --- a/drivers/iommu/s390-iommu.c +++ b/drivers/iommu/s390-iommu.c @@ -788,7 +788,7 @@ static struct iommu_device *s390_iommu_probe_device(struct device *dev) static int zpci_refresh_all(struct zpci_dev *zdev) { - return zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma, + return zpci_refresh_trans(zdev->fh, zdev->start_dma, zdev->end_dma - zdev->start_dma + 1); } @@ -819,8 +819,7 @@ static void s390_iommu_iotlb_sync(struct iommu_domain *domain, rcu_read_lock(); list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) { atomic64_inc(&s390_domain->ctrs.sync_rpcits); - zpci_refresh_trans((u64)zdev->fh << 32, gather->start, - size); + zpci_refresh_trans(zdev->fh, gather->start, size); } rcu_read_unlock(); } @@ -837,8 +836,7 @@ static int s390_iommu_iotlb_sync_map(struct iommu_domain *domain, if (!zdev->tlb_refresh) continue; atomic64_inc(&s390_domain->ctrs.sync_map_rpcits); - ret = zpci_refresh_trans((u64)zdev->fh << 32, - iova, size); + ret = zpci_refresh_trans(zdev->fh, iova, size); /* * let the hypervisor discover invalidated entries * allowing it to free IOVAs and unpin pages From 824c652322ae659cc1c24778fa3604831575de8f Mon Sep 17 00:00:00 2001 From: Gerd Bayer Date: Mon, 12 Oct 2026 16:21:51 +0200 Subject: [PATCH 095/102] s390/pci: Add function handle to RPCIT insn trace In certain debug situations it would be helpful, if the insn trace for a failing RPCIT would reveal the PCI function on which this was attempted. Introduce a variant of zpci_err_insn_req() called zpci_err_insn_rpcit() that accepts an IO virtual address and a range parameter together with the req conveying the function handle. Switch zpci_refresh_trans() to use that new error trace. Signed-off-by: Gerd Bayer Reviewed-by: Niklas Schnelle Reviewed-by: Matthew Rosato Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_insn.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c index b65135a94642..b5b3ea11ea42 100644 --- a/arch/s390/pci/pci_insn.c +++ b/arch/s390/pci/pci_insn.c @@ -28,18 +28,40 @@ struct zpci_err_insn_data { union { struct { u64 req; - u64 offset; + union { + /* PCILG/PCISTG/MPCIFC */ + struct { + u64 offset; + u64 pad1; + }; + /* RPCIT */ + struct { + u64 iova; + u64 range; + }; + }; }; + /* PCILGI/PCISTGI */ struct { u64 addr; u64 len; + u64 pad2; }; }; - u64 pad1; }; static_assert(sizeof(struct zpci_err_insn_data) <= ZPCI_ERR_INSN_DATA_SIZE); +static inline void zpci_err_insn_rpcit(int lvl, u8 insn, u8 cc, u8 status, + u64 req, u64 iova, u64 range) +{ + struct zpci_err_insn_data data = { + .insn = insn, .cc = cc, .status = status, + .req = req, .iova = iova, .range = range}; + + zpci_err_hex_level(lvl, &data, sizeof(data)); +} + static inline void zpci_err_insn_req(int lvl, u8 insn, u8 cc, u8 status, u64 req, u64 offset) { @@ -127,16 +149,16 @@ int zpci_refresh_trans(u32 fh, u64 addr, u64 range) if (cc == 2) { udelay(ZPCI_INSN_BUSY_DELAY); if (!retried) { - zpci_err_insn_addr(1, 'R', cc, status, addr, range); + zpci_err_insn_rpcit(1, 'R', cc, status, req, addr, range); retried = true; } } } while (cc == 2); if (cc) - zpci_err_insn_addr(0, 'R', cc, status, addr, range); + zpci_err_insn_rpcit(0, 'R', cc, status, req, addr, range); else if (retried) - zpci_err_insn_addr(1, 'R', cc, status, addr, range); + zpci_err_insn_rpcit(1, 'R', cc, status, req, addr, range); if (cc == 1 && (status == 4 || status == 16)) return -ENOMEM; From d61903049923242744cfe7009ed60bb7d3708630 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Wed, 7 Oct 2026 13:41:53 +0200 Subject: [PATCH 096/102] s390/mm: Cleanup pXXp_flush_lazy() routines Avoid unnecessary pgtable entry read in ptep_flush_lazy() and pmdp_flush_lazy() routines. Use the already obtained value of a pgtable entry instead of dereferencing it again. It might appear that the optimized out second dereferencing happens under mm_context_t::flush_count lock and therefore a race condition is possible. However, both routines write invalid entry bits to the pgtable entries and therefore no reader is allowed to interpret the rest of the value afterwards. Signed-off-by: Alexander Gordeev Reviewed-by: Gerald Schaefer Signed-off-by: Heiko Carstens --- arch/s390/mm/pgtable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 4acd8b140c4b..da076928b63b 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -107,7 +107,7 @@ static inline pte_t ptep_flush_lazy(struct mm_struct *mm, atomic_inc(&mm->context.flush_count); if (cpumask_equal(&mm->context.cpu_attach_mask, cpumask_of(smp_processor_id()))) { - set_pte(ptep, set_pte_bit(*ptep, __pgprot(_PAGE_INVALID))); + set_pte(ptep, set_pte_bit(old, __pgprot(_PAGE_INVALID))); mm->context.flush_mm = 1; } else ptep_ipte_global(mm, addr, ptep, nodat); @@ -227,7 +227,7 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, atomic_inc(&mm->context.flush_count); if (cpumask_equal(&mm->context.cpu_attach_mask, cpumask_of(smp_processor_id()))) { - set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_INVALID))); + set_pmd(pmdp, set_pmd_bit(old, __pgprot(_SEGMENT_ENTRY_INVALID))); mm->context.flush_mm = 1; } else { pmdp_idte_global(mm, addr, pmdp); From 89fb88215875248f6981a5b168c50538d7f9c751 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 14 Oct 2026 13:50:06 +0200 Subject: [PATCH 097/102] s390: Select ARCH_MEMORY_ORDER_TSO s390 is a Total Store Order (TSO) architecture. Therefore select ARCH_MEMORY_ORDER_TSO which was introduced with [1]. As of now this has only a small effect on futex_robust_unlock(), but that might change in the future. [1] commit 6149fc36c09b ("uaccess: Provide unsafe_atomic_store_release_user()") Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 4b51bc6e8948..ebdf9c8049af 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -145,6 +145,7 @@ config S390 select ARCH_INLINE_WRITE_UNLOCK_BH select ARCH_INLINE_WRITE_UNLOCK_IRQ select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE + select ARCH_MEMORY_ORDER_TSO select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW From d7ccab11a13475e656b9d2277900422c1bfc12c5 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 12 Oct 2026 10:36:27 +0200 Subject: [PATCH 098/102] s390/probes: Fix wrong opcode for cgrb fixup The last byte of the cgrb instruction is 0xe4, however the probes code compares against 0xe6. Fix it. Fixes: 6b169ac9b434 ("s390/kprobes: add support for compare and branch instructions") Cc: stable@vger.kernel.org Signed-off-by: Heiko Carstens --- arch/s390/lib/probes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/lib/probes.c b/arch/s390/lib/probes.c index 1e184a03442d..049300b68ddd 100644 --- a/arch/s390/lib/probes.c +++ b/arch/s390/lib/probes.c @@ -99,8 +99,8 @@ int probe_get_fixup_type(u16 *insn) break; case 0xec: switch (insn[2] & 0xff) { + case 0xe4: /* cgrb */ case 0xe5: /* clgrb */ - case 0xe6: /* cgrb */ case 0xf6: /* crb */ case 0xf7: /* clrb */ case 0xfc: /* cgib */ From ebac476520afb06028c5e539f47badb2415cd85f Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 12 Oct 2026 10:36:28 +0200 Subject: [PATCH 099/102] s390/probes: Add missing fixup for bctgr and bic For the bctgr and bic instructions the probes fixup is missing. For both instructions the instruction address must only be fixed up if the branch is not taken. If the branch is taken the instruction address is already correct. Return FIXUP_BRANCH_NOT_TAKEN instead of the default FIXUP_PSW_NORMAL to reflect that. Signed-off-by: Heiko Carstens --- arch/s390/lib/probes.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/arch/s390/lib/probes.c b/arch/s390/lib/probes.c index 049300b68ddd..3e7654223b05 100644 --- a/arch/s390/lib/probes.c +++ b/arch/s390/lib/probes.c @@ -77,6 +77,10 @@ int probe_get_fixup_type(u16 *insn) if ((insn[0] & 0xff) == 0xb2) fixup = FIXUP_NOT_REQUIRED; break; + case 0xb9: /* bctgr */ + if ((insn[0] & 0xff) == 0x46) + fixup = FIXUP_BRANCH_NOT_TAKEN; + break; case 0xa7: /* bras */ if ((insn[0] & 0x0f) == 0x05) fixup |= FIXUP_RETURN_REGISTER; @@ -85,6 +89,14 @@ int probe_get_fixup_type(u16 *insn) if ((insn[0] & 0x0f) == 0x05) /* brasl */ fixup |= FIXUP_RETURN_REGISTER; break; + case 0xe3: + switch (insn[2] & 0xff) { + case 0x46: /* bctg */ + case 0x47: /* bic */ + fixup = FIXUP_BRANCH_NOT_TAKEN; + break; + } + break; case 0xeb: switch (insn[2] & 0xff) { case 0x44: /* bxhg */ @@ -93,10 +105,6 @@ int probe_get_fixup_type(u16 *insn) break; } break; - case 0xe3: /* bctg */ - if ((insn[2] & 0xff) == 0x46) - fixup = FIXUP_BRANCH_NOT_TAKEN; - break; case 0xec: switch (insn[2] & 0xff) { case 0xe4: /* cgrb */ From 450c172f0e59c58a2e623974dae69cda51493720 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 12 Oct 2026 10:36:29 +0200 Subject: [PATCH 100/102] s390/probes: Fix wrong mnemonic comment for clgib There is no cglib instruction; the correct mnemonic is clgib. Fix the comment. Fixes: 6b169ac9b434 ("s390/kprobes: add support for compare and branch instructions") Signed-off-by: Heiko Carstens --- arch/s390/lib/probes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/lib/probes.c b/arch/s390/lib/probes.c index 3e7654223b05..66221ec16822 100644 --- a/arch/s390/lib/probes.c +++ b/arch/s390/lib/probes.c @@ -112,7 +112,7 @@ int probe_get_fixup_type(u16 *insn) case 0xf6: /* crb */ case 0xf7: /* clrb */ case 0xfc: /* cgib */ - case 0xfd: /* cglib */ + case 0xfd: /* clgib */ case 0xfe: /* cib */ case 0xff: /* clib */ fixup = FIXUP_BRANCH_NOT_TAKEN; From adc9e872f7350e3ac29c1818d8c68b0760b4ba23 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 13 Oct 2026 15:34:52 +0200 Subject: [PATCH 101/102] s390/signal: Fix incorrect return address in signal frame exit_to_user_mode_loop() can call arch_do_signal_or_restart() multiple times when signals are pending. If the first call set up a vDSO system call restart by saving the real user return address and adjusting psw.addr so it points to the vDSO stub, a second call builds the signal frame with the wrong return address. Restore psw.addr from arch_data, and set the return code to -EINTR before building the signal frame. This requires restart_block.arch_data to be properly initialized for all tasks. Move the initialization in copy_thread() in front of the kernel thread check, so that it also covers kernel threads and their children (PID 1, usermode helpers). Only PID 0 is left uninitialized, since it never returns to user space. Fixes: df29a7440c4b ("s390/signal: switch to using vdso for sigreturn and syscall restart") Cc: stable@vger.kernel.org Signed-off-by: Heiko Carstens --- arch/s390/kernel/process.c | 12 ++++++------ arch/s390/kernel/signal.c | 6 +++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 4c73d2528fcb..3092ea0347a1 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -130,6 +130,12 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) p->thread.hardirq_timer = 0; p->thread.softirq_timer = 0; p->thread.last_break = 1; + /* + * s390 stores the svc return address in arch_data when calling + * sigreturn()/restart_syscall() via vdso. 1 means no valid address + * stored. + */ + p->restart_block.arch_data = 1; frame->sf.back_chain = 0; frame->sf.gprs[11 - 6] = (unsigned long)&frame->childregs; @@ -167,12 +173,6 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) p->thread.acrs[0] = (unsigned int)(tls >> 32); p->thread.acrs[1] = (unsigned int)tls; } - /* - * s390 stores the svc return address in arch_data when calling - * sigreturn()/restart_syscall() via vdso. 1 means no valid address - * stored. - */ - p->restart_block.arch_data = 1; return 0; } diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index 4874de5edea0..b59b0c49a1aa 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -475,7 +475,11 @@ void arch_do_signal_or_restart(struct pt_regs *regs) } /* No longer in a system call */ clear_pt_regs_flag(regs, PIF_SYSCALL); - + if (!(current->restart_block.arch_data & 1)) { + regs->psw.addr = current->restart_block.arch_data; + current->restart_block.arch_data = 1; + regs->gprs[2] = -EINTR; + } rseq_signal_deliver(&ksig, regs); handle_signal(&ksig, oldset, regs); return; From 3e30b96fab780110f62156d5a8cd5240152b3c59 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 13 Oct 2026 15:34:53 +0200 Subject: [PATCH 102/102] s390/signal: Cleanup coding style Only whitespace, comment, indentation and other trivial changes to adopt signal.c to current coding style. No functional change. Reviewed-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens --- arch/s390/kernel/signal.c | 224 +++++++++++++++----------------------- 1 file changed, 90 insertions(+), 134 deletions(-) diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index b59b0c49a1aa..deeef053758c 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -27,8 +27,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -61,8 +61,7 @@ * have a fixed position and moves if gprs_high or vxrs exist. * Future extensions will be added to _sigregs_ext. */ -struct sigframe -{ +struct sigframe { __u8 callee_used_stack[__SIGNAL_FRAMESIZE]; struct sigcontext sc; _sigregs sregs; @@ -96,22 +95,21 @@ struct sigframe * ----------------------------------------- * Future extensions will be added to _sigregs_ext. */ -struct rt_sigframe -{ +struct rt_sigframe { __u8 callee_used_stack[__SIGNAL_FRAMESIZE]; __u16 svc_insn; struct siginfo info; struct ucontext_extended uc; }; -/* Store registers needed to create the signal frame */ +/* Store registers needed to create the signal frame. */ static void store_sigregs(void) { save_access_regs(current->thread.acrs); save_user_fpu_regs(); } -/* Load registers after signal return */ +/* Load registers after signal return. */ static void load_sigregs(void) { restore_access_regs(current->thread.acrs); @@ -122,14 +120,14 @@ static int save_sigregs(struct pt_regs *regs, _sigregs __user *sregs) { _sigregs user_sregs; - /* Copy a 'clean' PSW mask to the user to avoid leaking - information about whether PER is currently on. */ - user_sregs.regs.psw.mask = PSW_USER_BITS | - (regs->psw.mask & (PSW_MASK_USER | PSW_MASK_RI)); + /* + * Copy a 'clean' PSW mask to the user to avoid leaking + * information about whether PER is currently on. + */ + user_sregs.regs.psw.mask = PSW_USER_BITS | (regs->psw.mask & (PSW_MASK_USER | PSW_MASK_RI)); user_sregs.regs.psw.addr = regs->psw.addr; memcpy(&user_sregs.regs.gprs, ®s->gprs, sizeof(sregs->regs.gprs)); - memcpy(&user_sregs.regs.acrs, current->thread.acrs, - sizeof(user_sregs.regs.acrs)); + memcpy(&user_sregs.regs.acrs, current->thread.acrs, sizeof(user_sregs.regs.acrs)); fpregs_store(&user_sregs.fpregs, ¤t->thread.ufpu); if (__copy_to_user(sregs, &user_sregs, sizeof(_sigregs))) return -EFAULT; @@ -140,44 +138,37 @@ static int restore_sigregs(struct pt_regs *regs, _sigregs __user *sregs) { _sigregs user_sregs; - /* Always make any pending restarted system call return -EINTR */ + /* Always make any pending restarted system call return -EINTR. */ current->restart_block.fn = do_no_restart_syscall; - if (__copy_from_user(&user_sregs, sregs, sizeof(user_sregs))) return -EFAULT; - if (!is_ri_task(current) && (user_sregs.regs.psw.mask & PSW_MASK_RI)) return -EINVAL; - /* Use regs->psw.mask instead of PSW_USER_BITS to preserve PER bit. */ - regs->psw.mask = (regs->psw.mask & ~(PSW_MASK_USER | PSW_MASK_RI)) | - (user_sregs.regs.psw.mask & (PSW_MASK_USER | PSW_MASK_RI)); + regs->psw.mask &= ~(PSW_MASK_USER | PSW_MASK_RI); + regs->psw.mask |= user_sregs.regs.psw.mask & (PSW_MASK_USER | PSW_MASK_RI); /* Check for invalid user address space control. */ if ((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_HOME) - regs->psw.mask = PSW_ASC_PRIMARY | - (regs->psw.mask & ~PSW_MASK_ASC); - /* Check for invalid amode */ + regs->psw.mask = PSW_ASC_PRIMARY | (regs->psw.mask & ~PSW_MASK_ASC); + /* Check for invalid amode. */ if (regs->psw.mask & PSW_MASK_EA) regs->psw.mask |= PSW_MASK_BA; regs->psw.addr = user_sregs.regs.psw.addr; memcpy(®s->gprs, &user_sregs.regs.gprs, sizeof(sregs->regs.gprs)); - memcpy(¤t->thread.acrs, &user_sregs.regs.acrs, - sizeof(current->thread.acrs)); - + memcpy(¤t->thread.acrs, &user_sregs.regs.acrs, sizeof(current->thread.acrs)); fpregs_load(&user_sregs.fpregs, ¤t->thread.ufpu); - - clear_pt_regs_flag(regs, PIF_SYSCALL); /* No longer in a system call */ + /* No longer in a system call. */ + clear_pt_regs_flag(regs, PIF_SYSCALL); return 0; } /* Returns non-zero on fault. */ -static int save_sigregs_ext(struct pt_regs *regs, - _sigregs_ext __user *sregs_ext) +static int save_sigregs_ext(struct pt_regs *regs, _sigregs_ext __user *sregs_ext) { __u64 vxrs[__NUM_VXRS_LOW]; int i; - /* Save vector registers to signal stack */ + /* Save vector registers to signal stack. */ if (cpu_has_vx()) { for (i = 0; i < __NUM_VXRS_LOW; i++) vxrs[i] = current->thread.ufpu.vxrs[i].low; @@ -191,13 +182,12 @@ static int save_sigregs_ext(struct pt_regs *regs, return 0; } -static int restore_sigregs_ext(struct pt_regs *regs, - _sigregs_ext __user *sregs_ext) +static int restore_sigregs_ext(struct pt_regs *regs, _sigregs_ext __user *sregs_ext) { __u64 vxrs[__NUM_VXRS_LOW]; int i; - /* Restore vector registers from signal stack */ + /* Restore vector registers from signal stack. */ if (cpu_has_vx()) { if (__copy_from_user(vxrs, &sregs_ext->vxrs_low, sizeof(sregs_ext->vxrs_low)) || @@ -213,11 +203,12 @@ static int restore_sigregs_ext(struct pt_regs *regs, SYSCALL_DEFINE0(sigreturn) { - struct pt_regs *regs = task_pt_regs(current); - struct sigframe __user *frame = - (struct sigframe __user *) regs->gprs[15]; + struct sigframe __user *frame; + struct pt_regs *regs; sigset_t set; + regs = task_pt_regs(current); + frame = (struct sigframe __user *)regs->gprs[15]; if (__copy_from_user(&set.sig, &frame->sc.oldmask, _SIGMASK_COPY_SIZE)) goto badframe; set_current_blocked(&set); @@ -235,11 +226,12 @@ SYSCALL_DEFINE0(sigreturn) SYSCALL_DEFINE0(rt_sigreturn) { - struct pt_regs *regs = task_pt_regs(current); - struct rt_sigframe __user *frame = - (struct rt_sigframe __user *)regs->gprs[15]; + struct rt_sigframe __user *frame; + struct pt_regs *regs; sigset_t set; + regs = task_pt_regs(current); + frame = (struct rt_sigframe __user *)regs->gprs[15]; if (__copy_from_user(&set.sig, &frame->uc.uc_sigmask, sizeof(set))) goto badframe; set_current_blocked(&set); @@ -257,94 +249,76 @@ SYSCALL_DEFINE0(rt_sigreturn) return 0; } -/* - * Determine which stack to use.. - */ -static inline void __user * -get_sigframe(struct k_sigaction *ka, struct pt_regs * regs, size_t frame_size) +/* Determine which stack to use. */ +static inline void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size) { unsigned long sp; - /* Default to using normal stack */ + /* Default to using normal stack. */ sp = regs->gprs[15]; - /* Overflow on alternate signal stack gives SIGSEGV. */ if (on_sig_stack(sp) && !on_sig_stack((sp - frame_size) & -8UL)) - return (void __user *) -1UL; - - /* This is the X/Open sanctioned signal stack switching. */ + return (void __user *)-1UL; + /* This is the X/Open sanctioned signal stack switching. */ if (ka->sa.sa_flags & SA_ONSTACK) { - if (! sas_ss_flags(sp)) + if (!sas_ss_flags(sp)) sp = current->sas_ss_sp + current->sas_ss_size; } - - return (void __user *)((sp - frame_size) & -8ul); + return (void __user *)((sp - frame_size) & -8UL); } -static int setup_frame(int sig, struct k_sigaction *ka, - sigset_t *set, struct pt_regs * regs) +static int setup_frame(int sig, struct k_sigaction *ka, sigset_t *set, struct pt_regs *regs) { struct sigframe __user *frame; - struct sigcontext sc; unsigned long restorer; + struct sigcontext sc; size_t frame_size; frame_size = sizeof(*frame) - sizeof(frame->sregs_ext); if (cpu_has_vx()) frame_size += sizeof(frame->sregs_ext); frame = get_sigframe(ka, regs, frame_size); - if (frame == (void __user *) -1UL) + if (frame == (void __user *)-1UL) return -EFAULT; - /* Set up backchain. */ - if (__put_user(regs->gprs[15], (addr_t __user *) frame)) + if (__put_user(regs->gprs[15], (addr_t __user *)frame)) return -EFAULT; - - /* Create struct sigcontext on the signal stack */ + /* Create struct sigcontext on the signal stack. */ memcpy(&sc.oldmask, &set->sig, _SIGMASK_COPY_SIZE); - sc.sregs = (_sigregs __user __force *) &frame->sregs; + sc.sregs = (_sigregs __user __force *)&frame->sregs; if (__copy_to_user(&frame->sc, &sc, sizeof(frame->sc))) return -EFAULT; - - /* Store registers needed to create the signal frame */ + /* Store registers needed to create the signal frame. */ store_sigregs(); - - /* Create _sigregs on the signal stack */ + /* Create _sigregs on the signal stack. */ if (save_sigregs(regs, &frame->sregs)) return -EFAULT; - - /* Place signal number on stack to allow backtrace from handler. */ - if (__put_user(regs->gprs[2], (int __user *) &frame->signo)) + /* Place signal number on stack to allow backtrace from handler. */ + if (__put_user(regs->gprs[2], (int __user *)&frame->signo)) return -EFAULT; - - /* Create _sigregs_ext on the signal stack */ + /* Create _sigregs_ext on the signal stack. */ if (save_sigregs_ext(regs, &frame->sregs_ext)) return -EFAULT; - - /* Set up to return from userspace. If provided, use a stub - already in userspace. */ + /* + * Set up to return from userspace. If provided, use a stub + * already in userspace. + */ if (ka->sa.sa_flags & SA_RESTORER) - restorer = (unsigned long) ka->sa.sa_restorer; + restorer = (unsigned long)ka->sa.sa_restorer; else restorer = VDSO_SYMBOL(current, sigreturn); - - /* Set up registers for signal handler */ + /* Set up registers for signal handler. */ regs->gprs[14] = restorer; - regs->gprs[15] = (unsigned long) frame; + regs->gprs[15] = (unsigned long)frame; /* Force default amode and default user address space control. */ - regs->psw.mask = PSW_MASK_EA | PSW_MASK_BA | - (PSW_USER_BITS & PSW_MASK_ASC) | - (regs->psw.mask & ~PSW_MASK_ASC); - regs->psw.addr = (unsigned long) ka->sa.sa_handler; - + regs->psw.mask &= ~PSW_MASK_ASC; + regs->psw.mask |= (PSW_USER_BITS & PSW_MASK_ASC) | PSW_MASK_EA | PSW_MASK_BA; + regs->psw.addr = (unsigned long)ka->sa.sa_handler; regs->gprs[2] = sig; - regs->gprs[3] = (unsigned long) &frame->sc; - - /* We forgot to include these in the sigcontext. - To avoid breaking binary compatibility, they are passed as args. */ - if (sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || - sig == SIGTRAP || sig == SIGFPE) { - /* set extra registers only for synchronous signals */ + regs->gprs[3] = (unsigned long)&frame->sc; + /* Missing in sigcontext. Keep binary compatibility by passing as args. */ + if (sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP || sig == SIGFPE) { + /* Set extra registers only for synchronous signals. */ regs->gprs[4] = regs->int_code & 127; regs->gprs[5] = regs->int_parm_long; regs->gprs[6] = current->thread.last_break; @@ -352,8 +326,7 @@ static int setup_frame(int sig, struct k_sigaction *ka, return 0; } -static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, - struct pt_regs *regs) +static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { struct rt_sigframe __user *frame; unsigned long uc_flags, restorer; @@ -366,27 +339,24 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, uc_flags |= UC_VXRS; } frame = get_sigframe(&ksig->ka, regs, frame_size); - if (frame == (void __user *) -1UL) + if (frame == (void __user *)-1UL) return -EFAULT; - /* Set up backchain. */ - if (__put_user(regs->gprs[15], (addr_t __user *) frame)) + if (__put_user(regs->gprs[15], (addr_t __user *)frame)) return -EFAULT; - - /* Set up to return from userspace. If provided, use a stub - already in userspace. */ + /* + * Set up to return from userspace. If provided, use a stub + * already in userspace. + */ if (ksig->ka.sa.sa_flags & SA_RESTORER) - restorer = (unsigned long) ksig->ka.sa.sa_restorer; + restorer = (unsigned long)ksig->ka.sa.sa_restorer; else restorer = VDSO_SYMBOL(current, rt_sigreturn); - - /* Create siginfo on the signal stack */ + /* Create siginfo on the signal stack. */ if (copy_siginfo_to_user(&frame->info, &ksig->info)) return -EFAULT; - - /* Store registers needed to create the signal frame */ + /* Store registers needed to create the signal frame. */ store_sigregs(); - /* Create ucontext on the signal stack. */ if (__put_user(uc_flags, &frame->uc.uc_flags) || __put_user(NULL, &frame->uc.uc_link) || @@ -395,34 +365,29 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)) || save_sigregs_ext(regs, &frame->uc.uc_mcontext_ext)) return -EFAULT; - - /* Set up registers for signal handler */ + /* Set up registers for signal handler. */ regs->gprs[14] = restorer; - regs->gprs[15] = (unsigned long) frame; + regs->gprs[15] = (unsigned long)frame; /* Force default amode and default user address space control. */ - regs->psw.mask = PSW_MASK_EA | PSW_MASK_BA | - (PSW_USER_BITS & PSW_MASK_ASC) | - (regs->psw.mask & ~PSW_MASK_ASC); - regs->psw.addr = (unsigned long) ksig->ka.sa.sa_handler; - + regs->psw.mask &= ~PSW_MASK_ASC; + regs->psw.mask |= (PSW_USER_BITS & PSW_MASK_ASC) | PSW_MASK_EA | PSW_MASK_BA; + regs->psw.addr = (unsigned long)ksig->ka.sa.sa_handler; regs->gprs[2] = ksig->sig; - regs->gprs[3] = (unsigned long) &frame->info; - regs->gprs[4] = (unsigned long) &frame->uc; + regs->gprs[3] = (unsigned long)&frame->info; + regs->gprs[4] = (unsigned long)&frame->uc; regs->gprs[5] = current->thread.last_break; return 0; } -static void handle_signal(struct ksignal *ksig, sigset_t *oldset, - struct pt_regs *regs) +static void handle_signal(struct ksignal *ksig, sigset_t *oldset, struct pt_regs *regs) { int ret; - /* Set up the stack frame */ + /* Set up the stack frame. */ if (ksig->ka.sa.sa_flags & SA_SIGINFO) ret = setup_rt_frame(ksig, oldset, regs); else ret = setup_frame(ksig->sig, &ksig->ka, oldset, regs); - signal_setup_done(ret, ksig, test_thread_flag(TIF_SINGLE_STEP)); } @@ -435,22 +400,19 @@ static void handle_signal(struct ksignal *ksig, sigset_t *oldset, * the kernel can handle, and then we build all the user-level signal handling * stack-frames in one go after that. */ - void arch_do_signal_or_restart(struct pt_regs *regs) { - struct ksignal ksig; sigset_t *oldset = sigmask_to_save(); + struct ksignal ksig; /* * Get signal to deliver. When running under ptrace, at this point * the debugger may change all our registers, including the system * call information. */ - current->thread.system_call = - test_pt_regs_flag(regs, PIF_SYSCALL) ? regs->int_code : 0; - + current->thread.system_call = test_pt_regs_flag(regs, PIF_SYSCALL) ? regs->int_code : 0; if (get_signal(&ksig)) { - /* Whee! Actually deliver the signal. */ + /* Whee! Actually deliver the signal. */ if (current->thread.system_call) { regs->int_code = current->thread.system_call; /* Check for system call restarting. */ @@ -467,13 +429,11 @@ void arch_do_signal_or_restart(struct pt_regs *regs) fallthrough; case -ERESTARTNOINTR: regs->gprs[2] = regs->orig_gpr2; - regs->psw.addr = - __rewind_psw(regs->psw, - regs->int_code >> 16); + regs->psw.addr = __rewind_psw(regs->psw, regs->int_code >> 16); break; } } - /* No longer in a system call */ + /* No longer in a system call. */ clear_pt_regs_flag(regs, PIF_SYSCALL); if (!(current->restart_block.arch_data & 1)) { regs->psw.addr = current->restart_block.arch_data; @@ -484,14 +444,13 @@ void arch_do_signal_or_restart(struct pt_regs *regs) handle_signal(&ksig, oldset, regs); return; } - - /* No handlers present - check for system call restart */ + /* No handlers present - check for system call restart. */ clear_pt_regs_flag(regs, PIF_SYSCALL); if (current->thread.system_call) { regs->int_code = current->thread.system_call; switch (regs->gprs[2]) { case -ERESTART_RESTARTBLOCK: - /* Restart with sys_restart_syscall */ + /* Restart with sys_restart_syscall. */ regs->gprs[2] = regs->orig_gpr2; current->restart_block.arch_data = regs->psw.addr; regs->psw.addr = VDSO_SYMBOL(current, restart_syscall); @@ -508,9 +467,6 @@ void arch_do_signal_or_restart(struct pt_regs *regs) break; } } - - /* - * If there's no signal to deliver, we just put the saved sigmask back. - */ + /* If there's no signal to deliver, put the saved sigmask back. */ restore_saved_sigmask(); }