From 4eef4ab3aa3a32725e5bc79032c722f9f4a90172 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Tue, 1 Sep 2026 14:33:18 +0200 Subject: [PATCH 01/18] s390/pai: Use PAI PMU index as parameter replacing event Use PAI PMU index value as function argument instead of pointer to struct perf_event. Only that index value is used inside functions pai_alloc_cpu() and pai_event_destroy_cpu(). No functional change. Signed-off-by: Thomas Richter Reviewed-by: Sumanth Korikkar Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_pai.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/arch/s390/kernel/perf_pai.c b/arch/s390/kernel/perf_pai.c index 5c18c8b82ab7..109b15227d8f 100644 --- a/arch/s390/kernel/perf_pai.c +++ b/arch/s390/kernel/perf_pai.c @@ -140,16 +140,14 @@ static void pai_free(struct pai_mapptr *mp) /* Adjust usage counters and remove allocated memory when all users are * gone. */ -static void pai_event_destroy_cpu(struct perf_event *event, int cpu) +static void pai_event_destroy_cpu(int idx, int cpu) { - int idx = PAI_PMU_IDX(event); struct pai_mapptr *mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); struct pai_map *cpump = mp->mapptr; mutex_lock(&pai_reserve_mutex); - debug_sprintf_event(paidbg, 5, "%s event %#llx idx %d cpu %d users %d " - "refcnt %u\n", __func__, event->attr.config, idx, - event->cpu, cpump->active_events, + debug_sprintf_event(paidbg, 5, "%s users %d refcnt %u\n", + __func__, cpump->active_events, refcount_read(&cpump->refcnt)); if (refcount_dec_and_test(&cpump->refcnt)) pai_free(mp); @@ -159,17 +157,17 @@ static void pai_event_destroy_cpu(struct perf_event *event, int cpu) static void pai_event_destroy(struct perf_event *event) { - int cpu; + int cpu = 0, idx = PAI_PMU_IDX(event); free_page(PAI_SAVE_AREA(event)); if (event->cpu == -1) { struct cpumask *mask = PAI_CPU_MASK(event); for_each_cpu(cpu, mask) - pai_event_destroy_cpu(event, cpu); + pai_event_destroy_cpu(idx, cpu); kfree(mask); } else { - pai_event_destroy_cpu(event, event->cpu); + pai_event_destroy_cpu(idx, event->cpu); } } @@ -241,12 +239,12 @@ static u64 paicrypt_getall(struct perf_event *event) * * Allocate the memory for the event. */ -static int pai_alloc_cpu(struct perf_event *event, int cpu) +static int pai_alloc_cpu(int idx, int cpu) { - int rc, idx = PAI_PMU_IDX(event); struct pai_map *cpump = NULL; bool need_paiext_cb = false; struct pai_mapptr *mp; + int rc; mutex_lock(&pai_reserve_mutex); /* Allocate root node */ @@ -318,6 +316,7 @@ static int pai_alloc_cpu(struct perf_event *event, int cpu) static int pai_alloc(struct perf_event *event) { + int idx = PAI_PMU_IDX(event); struct cpumask *maskptr; int cpu, rc = -ENOMEM; @@ -326,10 +325,10 @@ static int pai_alloc(struct perf_event *event) goto out; for_each_online_cpu(cpu) { - rc = pai_alloc_cpu(event, cpu); + rc = pai_alloc_cpu(idx, cpu); if (rc) { for_each_cpu(cpu, maskptr) - pai_event_destroy_cpu(event, cpu); + pai_event_destroy_cpu(idx, cpu); kfree(maskptr); goto out; } @@ -392,7 +391,7 @@ static int pai_event_init(struct perf_event *event, int idx) } if (event->cpu >= 0) - rc = pai_alloc_cpu(event, event->cpu); + rc = pai_alloc_cpu(idx, event->cpu); else rc = pai_alloc(event); if (rc) { From e8df39dacb7d98d2b2aea431ca652d9fadf5efa3 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Tue, 1 Sep 2026 14:33:19 +0200 Subject: [PATCH 02/18] s390/pai: Move locking to event init and delete Move mutex locking from per CPU allocation to event allocation. No functional change. Signed-off-by: Thomas Richter Reviewed-by: Sumanth Korikkar Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_pai.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/arch/s390/kernel/perf_pai.c b/arch/s390/kernel/perf_pai.c index 109b15227d8f..c333158cc945 100644 --- a/arch/s390/kernel/perf_pai.c +++ b/arch/s390/kernel/perf_pai.c @@ -138,21 +138,19 @@ static void pai_free(struct pai_mapptr *mp) } /* Adjust usage counters and remove allocated memory when all users are - * gone. + * gone. Called under mutex_lock. */ static void pai_event_destroy_cpu(int idx, int cpu) { struct pai_mapptr *mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); struct pai_map *cpump = mp->mapptr; - mutex_lock(&pai_reserve_mutex); debug_sprintf_event(paidbg, 5, "%s users %d refcnt %u\n", __func__, cpump->active_events, refcount_read(&cpump->refcnt)); if (refcount_dec_and_test(&cpump->refcnt)) pai_free(mp); pai_root_free(idx); - mutex_unlock(&pai_reserve_mutex); } static void pai_event_destroy(struct perf_event *event) @@ -160,6 +158,7 @@ static void pai_event_destroy(struct perf_event *event) int cpu = 0, idx = PAI_PMU_IDX(event); free_page(PAI_SAVE_AREA(event)); + mutex_lock(&pai_reserve_mutex); if (event->cpu == -1) { struct cpumask *mask = PAI_CPU_MASK(event); @@ -169,6 +168,7 @@ static void pai_event_destroy(struct perf_event *event) } else { pai_event_destroy_cpu(idx, event->cpu); } + mutex_unlock(&pai_reserve_mutex); } static void paicrypt_event_destroy(struct perf_event *event) @@ -232,12 +232,10 @@ static u64 paicrypt_getall(struct perf_event *event) return sum; } -/* Check concurrent access of counting and sampling for crypto events. - * This function is called in process context and it is save to block. - * When the event initialization functions fails, no other call back will - * be invoked. - * - * Allocate the memory for the event. +/* Allocate all per-CPU data structures. This function is called in + * process context and can block. In case of error all partly allocated + * memory is released and the reference counters adjusted correctly. + * Called under mutex_lock. */ static int pai_alloc_cpu(int idx, int cpu) { @@ -246,11 +244,10 @@ static int pai_alloc_cpu(int idx, int cpu) struct pai_mapptr *mp; int rc; - mutex_lock(&pai_reserve_mutex); /* Allocate root node */ rc = pai_root_alloc(idx); if (rc) - goto unlock; + goto out; /* Allocate node for this event */ mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); @@ -308,12 +305,12 @@ static int pai_alloc_cpu(int idx, int cpu) */ pai_root_free(idx); } -unlock: - mutex_unlock(&pai_reserve_mutex); +out: /* If rc is non-zero, no increment of counter/sampler was done. */ return rc; } +/* Called under mutex_lock */ static int pai_alloc(struct perf_event *event) { int idx = PAI_PMU_IDX(event); @@ -390,10 +387,12 @@ static int pai_event_init(struct perf_event *event, int idx) } } + mutex_lock(&pai_reserve_mutex); if (event->cpu >= 0) rc = pai_alloc_cpu(idx, event->cpu); else rc = pai_alloc(event); + mutex_unlock(&pai_reserve_mutex); if (rc) { free_page(PAI_SAVE_AREA(event)); goto out; From 9ecc4d033879f7761f2df07e20cd2fbec00fd90b Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Tue, 1 Sep 2026 14:33:20 +0200 Subject: [PATCH 03/18] s390/pai: Support CPU hotplug for PMU PAI The command 'perf stat -e pai_crypto/CRYPTO_ALL/ -- ' crashes the kernel when CPUs are hotplug added during that run. Root cause is the missing allocation of per-CPU data structures for that new CPU. The allocation is dynamic and the first event that has task context creates such a structure for each online CPU. This is not sufficient. CPUs may be offline during event creation and can be set online during the perf run time. For example commands # echo 0 > /sys/devices/system/cpu/cpu1/online # perf stat -e cycles -i -- stress-ng -t10s --matrix X # sleep 1 # echo 1 > /sys/devices/system/cpu/cpu1/online Currently without a CPU hotplug handler, that new CPU has no per-CPU data infrastructure. The scheduler runs PMU call back function pai_add() to install the PMU support for that CPU before the task is being scheduled on that new CPU. In pai_add() instructions mp = this_cpu_ptr(pai_root[idx].mapptr); cpump = mp->mapptr; return a NULL pointer and the result is a kernel panic as variable cpump is used inside that function. Add CPU hotplug support for CPU add and delete and create the necessary per-CPU data infrastructure during CPU hotplug add processing. Same for CPU hotplug remove. This is done when the CPU is offline to ensure the data structures are available when CPU is made online and tasks are scheduled on it. [hca@linux.ibm.com: fixup error path in pai_init()] Cc: stable@vger.kernel.org # v6.19 Fixes: 582cc1b28e8c ("s390/pai_ext: Enable per-task and system-wide sampling event") Fixes: 9f66572f2889 ("s390/pai_crypto: Enable per-task and system-wide sampling event") Signed-off-by: Thomas Richter Reviewed-by: Jan Polensky Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/pai.h | 1 - arch/s390/kernel/perf_pai.c | 178 ++++++++++++++++++++++++++---------- 2 files changed, 128 insertions(+), 51 deletions(-) diff --git a/arch/s390/include/asm/pai.h b/arch/s390/include/asm/pai.h index 534d0320e2aa..a3456a36aaa7 100644 --- a/arch/s390/include/asm/pai.h +++ b/arch/s390/include/asm/pai.h @@ -76,7 +76,6 @@ static __always_inline void pai_kernel_exit(struct pt_regs *regs) } #define PAI_SAVE_AREA(x) ((x)->hw.event_base) -#define PAI_CPU_MASK(x) ((x)->hw.addr_filters) #define PAI_PMU_IDX(x) ((x)->hw.last_tag) #define PAI_SWLIST(x) (&(x)->hw.tp_list) diff --git a/arch/s390/kernel/perf_pai.c b/arch/s390/kernel/perf_pai.c index c333158cc945..013c3dae21ec 100644 --- a/arch/s390/kernel/perf_pai.c +++ b/arch/s390/kernel/perf_pai.c @@ -67,6 +67,7 @@ struct pai_mapptr { static struct pai_root { /* Anchor to per CPU data */ refcount_t refcnt; /* Overall active events */ + atomic_t tskctx; /* Overall per-task events */ struct pai_mapptr __percpu *mapptr; } pai_root[PAI_PMU_MAX]; @@ -93,14 +94,15 @@ struct pai_pmu { /* Define PAI PMU characteristics */ static struct pai_pmu pai_pmu[]; /* Forward declaration */ /* Free per CPU data when the last event is removed. */ -static void pai_root_free(int idx) +static void pai_root_free(int idx, int tasks) { - if (refcount_dec_and_test(&pai_root[idx].refcnt)) { + if (refcount_sub_and_test(tasks, &pai_root[idx].refcnt)) { free_percpu(pai_root[idx].mapptr); pai_root[idx].mapptr = NULL; } - debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d\n", __func__, - idx, refcount_read(&pai_root[idx].refcnt)); + debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d tskctx %d\n", + __func__, idx, refcount_read(&pai_root[idx].refcnt), + atomic_read(&pai_root[idx].tskctx)); } /* @@ -137,20 +139,36 @@ static void pai_free(struct pai_mapptr *mp) mp->mapptr = NULL; } -/* Adjust usage counters and remove allocated memory when all users are - * gone. Called under mutex_lock. - */ -static void pai_event_destroy_cpu(int idx, int cpu) +/* Called under mutex_lock */ +static void pai_event_destroy_cpu(int idx, int cpu, bool hotplug) { - struct pai_mapptr *mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); - struct pai_map *cpump = mp->mapptr; + struct pai_mapptr *mp; + struct pai_map *cpump; + int tasks = 1; - debug_sprintf_event(paidbg, 5, "%s users %d refcnt %u\n", - __func__, cpump->active_events, - refcount_read(&cpump->refcnt)); - if (refcount_dec_and_test(&cpump->refcnt)) + /* Check reference count and return when all gone. + * 1. An event is installed on online CPU X. + * 2. CPU x is offlined and the per-CPU data is removed. + * 3. Event is destroyed via close system call. + */ + if (!refcount_read(&pai_root[idx].refcnt)) + return; /* No events at all */ + mp = per_cpu_ptr(pai_root[idx].mapptr, cpu); + if (!mp || !mp->mapptr) /* No events on that CPU */ + return; + + /* When hotplug is true, invocation is from CPU hotplug callback. + * Delete per-CPU resource and adjust refcnt when per-task events + * are currently active. This can be more than one. + * In this case adjust counters. + */ + if (hotplug) + tasks = atomic_read(&pai_root[idx].tskctx); + + cpump = mp->mapptr; + if (refcount_sub_and_test(tasks, &cpump->refcnt)) pai_free(mp); - pai_root_free(idx); + pai_root_free(idx, tasks); } static void pai_event_destroy(struct perf_event *event) @@ -158,17 +176,17 @@ static void pai_event_destroy(struct perf_event *event) int cpu = 0, idx = PAI_PMU_IDX(event); free_page(PAI_SAVE_AREA(event)); + cpus_read_lock(); mutex_lock(&pai_reserve_mutex); if (event->cpu == -1) { - struct cpumask *mask = PAI_CPU_MASK(event); - - for_each_cpu(cpu, mask) - pai_event_destroy_cpu(idx, cpu); - kfree(mask); + atomic_dec(&pai_root[idx].tskctx); + for_each_online_cpu(cpu) + pai_event_destroy_cpu(idx, cpu, false); } else { - pai_event_destroy_cpu(idx, event->cpu); + pai_event_destroy_cpu(idx, event->cpu, false); } mutex_unlock(&pai_reserve_mutex); + cpus_read_unlock(); } static void paicrypt_event_destroy(struct perf_event *event) @@ -232,17 +250,25 @@ static u64 paicrypt_getall(struct perf_event *event) return sum; } -/* Allocate all per-CPU data structures. This function is called in - * process context and can block. In case of error all partly allocated - * memory is released and the reference counters adjusted correctly. - * Called under mutex_lock. - */ -static int pai_alloc_cpu(int idx, int cpu) +/* Called under mutex_lock */ +static int pai_alloc_cpu(int idx, int cpu, bool hotplug) { struct pai_map *cpump = NULL; bool need_paiext_cb = false; struct pai_mapptr *mp; - int rc; + int tasks = 1, rc = 0; + + /* When hotplug is true, invocation is from CPU hotplug callback. + * Allocate per-CPU resource when per-task events are currently active. + * This can be more than one. In this case adjust all reference + * counters. Otherwise return, this ensures memory is only allocated + * when needed. + */ + if (hotplug) { + tasks = atomic_read(&pai_root[idx].tskctx); + if (!tasks) + goto out; + } /* Allocate root node */ rc = pai_root_alloc(idx); @@ -291,26 +317,42 @@ static int pai_alloc_cpu(int idx, int cpu) goto undo; } INIT_LIST_HEAD(&cpump->syswide_list); - refcount_set(&cpump->refcnt, 1); + refcount_set(&cpump->refcnt, tasks); rc = 0; } else { - refcount_inc(&cpump->refcnt); + refcount_add(tasks, &cpump->refcnt); } + /* If tasks is greater than 1, we are called from CPU hotplug path + * and need to adjust the pai_root[idx].refcnt by the number of + * per-process events. Function pai_root_alloc(idx) already + * incremented by one. Adjust for the rest. + */ + if (tasks > 1) + refcount_add(tasks - 1, &pai_root[idx].refcnt); undo: if (rc) { /* Error in allocation of event, decrement anchor. Since * the event in not created, its destroy() function is never * invoked. Adjust the reference counter for the anchor. + * The failure happened in the case of variable + * cpump == NULL branch above. The pai_root[XXX].refcnt has + * been incremented by one. Then the per-CPU allocation + * failed, so decrement it by one, regardless of tasks. */ - pai_root_free(idx); + pai_root_free(idx, 1); } out: /* If rc is non-zero, no increment of counter/sampler was done. */ return rc; } -/* Called under mutex_lock */ +/* Check concurrent access of counting and sampling for PAI events. + * This function is called in process context and it is safe to block. + * When the event initialization functions fails, no other call back will + * be invoked. + * Called under mutex_lock. + */ static int pai_alloc(struct perf_event *event) { int idx = PAI_PMU_IDX(event); @@ -322,24 +364,20 @@ static int pai_alloc(struct perf_event *event) goto out; for_each_online_cpu(cpu) { - rc = pai_alloc_cpu(idx, cpu); + rc = pai_alloc_cpu(idx, cpu, false); if (rc) { for_each_cpu(cpu, maskptr) - pai_event_destroy_cpu(idx, cpu); - kfree(maskptr); - goto out; + pai_event_destroy_cpu(idx, cpu, false); + goto undo; } cpumask_set_cpu(cpu, maskptr); } - /* - * On error all cpumask are freed and all events have been destroyed. - * Save of which CPUs data structures have been allocated for. - * Release them in pai_event_destroy call back function - * for this event. - */ - PAI_CPU_MASK(event) = maskptr; rc = 0; + /* Trace per-task events for CPU hotplug. */ + atomic_inc(&pai_root[idx].tskctx); +undo: + kfree(maskptr); out: return rc; } @@ -387,12 +425,14 @@ static int pai_event_init(struct perf_event *event, int idx) } } + cpus_read_lock(); mutex_lock(&pai_reserve_mutex); if (event->cpu >= 0) - rc = pai_alloc_cpu(idx, event->cpu); + rc = pai_alloc_cpu(idx, event->cpu, false); else rc = pai_alloc(event); mutex_unlock(&pai_reserve_mutex); + cpus_read_unlock(); if (rc) { free_page(PAI_SAVE_AREA(event)); goto out; @@ -1237,8 +1277,35 @@ static int __init paipmu_setup(void) return install_ok; } +static int pai_online_cpu(unsigned int cpu) +{ + int rc; + + mutex_lock(&pai_reserve_mutex); + rc = pai_alloc_cpu(PAI_PMU_CRYPTO, cpu, true); + if (rc) + goto out; + rc = pai_alloc_cpu(PAI_PMU_EXT, cpu, true); + if (rc) + pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true); +out: + mutex_unlock(&pai_reserve_mutex); + return rc; +} + +static int pai_offline_cpu(unsigned int cpu) +{ + mutex_lock(&pai_reserve_mutex); + pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true); + pai_event_destroy_cpu(PAI_PMU_EXT, cpu, true); + mutex_unlock(&pai_reserve_mutex); + return 0; +} + static int __init pai_init(void) { + int state, rc; + /* Setup s390dbf facility */ paidbg = debug_register("pai", 1, 1, 128); if (!paidbg) { @@ -1247,13 +1314,24 @@ static int __init pai_init(void) } debug_register_view(paidbg, &debug_sprintf_view); - if (!paipmu_setup()) { - /* No PMU registration, no need for debug buffer */ - debug_unregister_view(paidbg, &debug_sprintf_view); - debug_unregister(paidbg); - return -ENODEV; - } + /* CPUHP_BP_PREPARE_DYN --> before CPU is brought online */ + state = cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "perf/pai:prepare", + pai_online_cpu, pai_offline_cpu); + rc = state < 0 ? state : 0; + if (rc < 0) + goto out_debug; + + rc = -ENODEV; + if (!paipmu_setup()) + goto out_cpuhp; return 0; + +out_cpuhp: + cpuhp_remove_state(state); +out_debug: + debug_unregister_view(paidbg, &debug_sprintf_view); + debug_unregister(paidbg); + return rc; } device_initcall(pai_init); From b1eb31d533cdfcae1011ed53850d52f36afe5774 Mon Sep 17 00:00:00 2001 From: Mikhail Zaslonko Date: Thu, 3 Sep 2026 15:07:31 +0200 Subject: [PATCH 04/18] s390/debug: Fix NULL pointer dereference in debug_set_level() Commit a2cec6863709 ("s390/debug: Add s390dbf kernel parameter") incorrectly removed a null-id check from debug_set_level(), introducing a possible NULL pointer dereference for debug-API users that put debug_register() results unchecked into debug_set_level(). Fix this by moving the check from the internal _debug_set_level() variant back to the external debug_set_level() wrapper. Fixes: a2cec6863709 ("s390/debug: Add s390dbf kernel parameter") Signed-off-by: Mikhail Zaslonko Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/debug.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index 14d2b58ad093..e06abf1dbc21 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -1074,9 +1074,6 @@ static void _debug_set_level(debug_info_t *id, int new_level) { unsigned long flags; - if (!id) - return; - if (new_level == DEBUG_OFF_LEVEL) { pr_info("%s: switched off\n", id->name); } else if ((new_level > DEBUG_MAX_LEVEL) || (new_level < 0)) { @@ -1101,6 +1098,9 @@ static void _debug_set_level(debug_info_t *id, int new_level) */ void debug_set_level(debug_info_t *id, int new_level) { + if (!id) + return; + /* Level specified via kernel parameter takes precedence */ debug_get_param(id->name, &new_level, NULL); From 22d4210bf988047bd30803cd6ef5177f113e4004 Mon Sep 17 00:00:00 2001 From: Mikhail Zaslonko Date: Thu, 3 Sep 2026 15:07:32 +0200 Subject: [PATCH 05/18] s390/debug: Do not repeat parameter override notice on debug_set_level() Commit a2cec6863709 ("s390/debug: Add s390dbf kernel parameter") calls debug_get_param() from both debug_info_create() and debug_set_level(). Since debug_get_param() emits the override notice unconditionally, and drivers typically call debug_set_level() right after debug_register(), the same line is printed twice per debug area: s390dbf: 0.0.1234: override level to 6 s390dbf: 0.0.1234: override level to 6 For areas registered per device this is multiplied by the device count. With 's390dbf=0.0.*:6' a system with many DASDs emits a large number of redundant lines during boot. Add a quiet parameter to debug_get_param() and pass quiet=true from debug_set_level(), where the override has already been announced during registration. The remaining callers keep printing the notice. Signed-off-by: Mikhail Zaslonko Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/debug.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index e06abf1dbc21..cf411f203571 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -182,7 +182,7 @@ static struct debug_param_t { static int debug_param_num; /* functions */ -static void debug_get_param(const char *name, int *level, int *pages) +static void debug_get_param(const char *name, int *level, int *pages, bool quiet) { struct debug_param_t *p; int i; @@ -192,11 +192,13 @@ static void debug_get_param(const char *name, int *level, int *pages) if (!glob_match(p->name, name)) continue; if (level && p->level != PARAM_UNSET) { - pr_info("%s: override level to %d\n", name, p->level); + if (!quiet) + pr_info("%s: override level to %d\n", name, p->level); *level = p->level; } if (pages && p->pages != PARAM_UNSET) { - pr_info("%s: override pages to %d\n", name, p->pages); + if (!quiet) + pr_info("%s: override pages to %d\n", name, p->pages); *pages = p->pages; } } @@ -251,7 +253,7 @@ static int __init s390dbf_parse(char *arg) * regular memory allocations are possible. */ for (i = 0, id = __s390dbf_info; &id[i] < __s390dbf_info_end; i++) - debug_get_param(id[i]->name, &id[i]->level, NULL); + debug_get_param(id[i]->name, &id[i]->level, NULL, false); return rc; } @@ -395,7 +397,7 @@ static debug_info_t *debug_info_create(const char *name, int pages_per_area, int level = DEBUG_DEFAULT_LEVEL; debug_info_t *rc; - debug_get_param(name, &level, &pages_per_area); + debug_get_param(name, &level, &pages_per_area, false); rc = debug_info_alloc(name, pages_per_area, nr_areas, buf_size, level, ALL_AREAS); if (!rc) goto out; @@ -960,7 +962,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) return; } - debug_get_param(id->name, &id->level, &pages_per_area); + debug_get_param(id->name, &id->level, &pages_per_area, false); copy = debug_info_alloc("", pages_per_area, nr_areas, id->buf_size, id->level, ALL_AREAS); if (!copy) { @@ -1101,8 +1103,11 @@ void debug_set_level(debug_info_t *id, int new_level) if (!id) return; - /* Level specified via kernel parameter takes precedence */ - debug_get_param(id->name, &new_level, NULL); + /* + * Level specified via kernel parameter takes precedence. The override + * was already announced during registration, so stay quiet here. + */ + debug_get_param(id->name, &new_level, NULL, true); _debug_set_level(id, new_level); } From 0945285e6cd67ee87e9313fb10221aba5bd69c6a Mon Sep 17 00:00:00 2001 From: Mikhail Zaslonko Date: Thu, 3 Sep 2026 15:07:33 +0200 Subject: [PATCH 06/18] s390/debug: Fix race between debug area resize and event logging Trace functions check for non-NULL id->areas without lock to minimize overhead. This opens a race window where a NULL pointer dereference occurs if id->areas is set to NULL (e.g. via echo 0 > ../pages) after the check and before id->lock is taken. Fix this by rechecking id->areas under lock. Signed-off-by: Mikhail Zaslonko Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/debug.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index cf411f203571..b5bf8284dbfc 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -1283,7 +1283,7 @@ void debug_set_critical(void) debug_entry_t *debug_event_common(debug_info_t *id, int level, const void *buf, int len) { - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; if (!debug_active || !id->areas) @@ -1294,6 +1294,8 @@ debug_entry_t *debug_event_common(debug_info_t *id, int level, const void *buf, } else { raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; do { active = get_active_entry(id); memcpy(DEBUG_DATA(active), buf, min(len, id->buf_size)); @@ -1303,7 +1305,7 @@ debug_entry_t *debug_event_common(debug_info_t *id, int level, const void *buf, len -= id->buf_size; buf += id->buf_size; } while (len > 0); - +out: raw_spin_unlock_irqrestore(&id->lock, flags); return active; } @@ -1316,7 +1318,7 @@ EXPORT_SYMBOL(debug_event_common); debug_entry_t *debug_exception_common(debug_info_t *id, int level, const void *buf, int len) { - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; if (!debug_active || !id->areas) @@ -1327,6 +1329,8 @@ debug_entry_t *debug_exception_common(debug_info_t *id, int level, } else { raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; do { active = get_active_entry(id); memcpy(DEBUG_DATA(active), buf, min(len, id->buf_size)); @@ -1336,7 +1340,7 @@ debug_entry_t *debug_exception_common(debug_info_t *id, int level, len -= id->buf_size; buf += id->buf_size; } while (len > 0); - +out: raw_spin_unlock_irqrestore(&id->lock, flags); return active; } @@ -1362,7 +1366,7 @@ static inline int debug_count_numargs(char *string) debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string, ...) { debug_sprintf_entry_t *curr_event; - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; int numargs, idx; va_list ap; @@ -1377,6 +1381,8 @@ debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string, } else { raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; active = get_active_entry(id); curr_event = (debug_sprintf_entry_t *) DEBUG_DATA(active); va_start(ap, string); @@ -1385,6 +1391,7 @@ debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string, curr_event->args[idx] = va_arg(ap, long); va_end(ap); debug_finish_entry(id, active, level, 0); +out: raw_spin_unlock_irqrestore(&id->lock, flags); return active; @@ -1397,7 +1404,7 @@ EXPORT_SYMBOL(__debug_sprintf_event); debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *string, ...) { debug_sprintf_entry_t *curr_event; - debug_entry_t *active; + debug_entry_t *active = NULL; unsigned long flags; int numargs, idx; va_list ap; @@ -1413,6 +1420,8 @@ debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *stri } else { raw_spin_lock_irqsave(&id->lock, flags); } + if (!id->areas) + goto out; active = get_active_entry(id); curr_event = (debug_sprintf_entry_t *)DEBUG_DATA(active); va_start(ap, string); @@ -1421,6 +1430,7 @@ debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *stri curr_event->args[idx] = va_arg(ap, long); va_end(ap); debug_finish_entry(id, active, level, 1); +out: raw_spin_unlock_irqrestore(&id->lock, flags); return active; @@ -1663,9 +1673,11 @@ static void debug_flush(debug_info_t *id, int area) unsigned long flags; int i, j; - if (!id || !id->areas) + if (!id) return; raw_spin_lock_irqsave(&id->lock, flags); + if (!id->areas) + goto out; if (area == DEBUG_FLUSH_ALL) { id->active_area = 0; memset(id->active_entries, 0, id->nr_areas * sizeof(int)); @@ -1680,6 +1692,7 @@ static void debug_flush(debug_info_t *id, int area) for (i = 0; i < id->pages_per_area; i++) memset(id->areas[area][i], 0, PAGE_SIZE); } +out: raw_spin_unlock_irqrestore(&id->lock, flags); } From 15fa028589c3a2545f0bff355eff06d5844196bf Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2026 10:37:33 +0200 Subject: [PATCH 07/18] s390/crypto: Fix skcipher_walk return code handling in aes_s390 The return codes from skcipher_walk_virt() were not properly checked before entering the processing loops in ecb_aes_crypt() and ctr_aes_crypt(). If skcipher_walk_virt() fails, the walk structure may be in an undefined state, and attempting to process data could lead to incorrect behavior or accessing uninitialized memory. Add proper return code checking to ensure correct handling of the walk initialization and walk advance and eventually return to the caller with that return code. Fixes: 7988fb2c03c8 ("crypto: s390/aes - convert to skcipher API") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Cc: stable@vger.kernel.org # 5.5+ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/aes_s390.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 62edc66d5478..366ce22d3623 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -129,7 +129,7 @@ static int ecb_aes_crypt(struct skcipher_request *req, unsigned long modifier) return fallback_skcipher_crypt(sctx, req, modifier); ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(sctx->fc | modifier, sctx->key, @@ -233,7 +233,7 @@ static int cbc_aes_crypt(struct skcipher_request *req, unsigned long modifier) return ret; memcpy(param.iv, walk.iv, AES_BLOCK_SIZE); memcpy(param.key, sctx->key, sctx->key_len); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_kmc(sctx->fc | modifier, ¶m, @@ -359,7 +359,7 @@ static int xts_aes_crypt(struct skcipher_request *req, unsigned long modifier) memcpy(xts_param.key + offset, xts_ctx->key, xts_ctx->key_len); memcpy(xts_param.init, pcc_param.xts, 16); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(xts_ctx->fc | modifier, xts_param.key + offset, @@ -487,7 +487,7 @@ static int fullxts_aes_crypt(struct skcipher_request *req, unsigned long modifi memcpy(fxts_param.tweak, req->iv, AES_BLOCK_SIZE); fxts_param.nap[0] = 0x01; /* initial alpha power (1, little-endian) */ - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(xts_ctx->fc | modifier, fxts_param.key + offset, @@ -577,7 +577,7 @@ static int ctr_aes_crypt(struct skcipher_request *req) locked = mutex_trylock(&ctrblk_lock); ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { + while (!ret && ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE)) { n = AES_BLOCK_SIZE; if (nbytes >= 2*AES_BLOCK_SIZE && locked) @@ -596,7 +596,7 @@ static int ctr_aes_crypt(struct skcipher_request *req) /* * final block may be < AES_BLOCK_SIZE, copy only nbytes */ - if (nbytes) { + if (!ret && nbytes) { memset(buf, 0, AES_BLOCK_SIZE); memcpy(buf, walk.src.virt.addr, nbytes); cpacf_kmctr(sctx->fc, sctx->key, buf, buf, From 8b7c3b6914f19caf648d05726a86af6326d3c2c6 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2026 10:37:34 +0200 Subject: [PATCH 08/18] s390/crypto: Fix missing scrub of temp buffers with AES ctr and gcm algorithm In function ctr_aes_crypt() there is a buffer used to process remaining bytes < AES_BLOCK_SIZE. This buffer was not scrubbed and thus could lead to expose of unwanted data. When the buffer is used explicitly scrub it at the end of the code block to avoid exposure of maybe sensitive data. In a similar way the function gcm_aes_crypt() hat an error path where the CPACF param block was not scrubbed. Instead of return early now these error paths go to end of function where explicit scrubbing is done. Similar with the buffers which are part of the gcm_sg_walk structs from the variables gw_in and gw_out. Fixes: d07f951903fa ("crypto: s390/aes - Fix buffer overread in CTR mode") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Cc: stable@vger.kernel.org # 6.8+ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/aes_s390.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 366ce22d3623..10561aa687c7 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -604,6 +604,7 @@ static int ctr_aes_crypt(struct skcipher_request *req) memcpy(walk.dst.virt.addr, buf, nbytes); crypto_inc(walk.iv, AES_BLOCK_SIZE); ret = skcipher_walk_done(&walk, 0); + memzero_explicit(buf, sizeof(buf)); } return ret; @@ -895,10 +896,14 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags) gw_in.ptr, aad_bytes); n = aad_bytes + pc_bytes; - if (gcm_in_walk_done(&gw_in, n) != n) - return -ENOMEM; - if (gcm_out_walk_done(&gw_out, n) != n) - return -ENOMEM; + if (gcm_in_walk_done(&gw_in, n) != n) { + ret = -ENOMEM; + goto out; + } + if (gcm_out_walk_done(&gw_out, n) != n) { + ret = -ENOMEM; + goto out; + } aadlen -= aad_bytes; pclen -= pc_bytes; } while (aadlen + pclen > 0); @@ -910,7 +915,10 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags) } else scatterwalk_map_and_copy(param.t, req->dst, len, taglen, 1); +out: memzero_explicit(¶m, sizeof(param)); + memzero_explicit(gw_in.buf, sizeof(gw_in.buf)); + memzero_explicit(gw_out.buf, sizeof(gw_out.buf)); return ret; } From d1c44a7d085473173bb360b7218a43699c3f56c7 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2026 10:37:35 +0200 Subject: [PATCH 09/18] s390/crypto: Fix use of mutex in atomic context The AES CTR implementation used a mutex to lock one page of exclusive memory for fast CTR processing. Unfortunately a mutex is not save to use in atomic or interrupt context. So use a binary semaphore instead which is save to use in such environments. Furthermore rework the code to get rid of conditional locking. So restructure the AES CRT code by extracting the main loop into a separate function and just give in information about the (locked) page can be used or not (is not locked). Fixes: 7988fb2c03c8 ("crypto: s390/aes - convert to skcipher API") Suggested-by: Heiko Carstens Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Cc: stable@vger.kernel.org # 5.5+ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/aes_s390.c | 63 +++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 10561aa687c7..0be6fa779d2c 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -26,14 +26,14 @@ #include #include #include -#include #include +#include #include #include #include static u8 *ctrblk; -static DEFINE_MUTEX(ctrblk_lock); +static DEFINE_SEMAPHORE(ctrblk_sem, 1); static cpacf_mask_t km_functions, kmc_functions, kmctr_functions, kma_functions; @@ -562,46 +562,61 @@ static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) return n; } +static int __ctr_aes_crypt(struct s390_aes_ctx *sctx, + struct skcipher_walk *walk, bool locked) +{ + unsigned int n, nbytes; + int ret = 0; + u8 *ctrptr; + + while (!ret && ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE)) { + n = AES_BLOCK_SIZE; + if (nbytes >= 2 * AES_BLOCK_SIZE && locked) + n = __ctrblk_init(ctrblk, walk->iv, nbytes); + ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; + cpacf_kmctr(sctx->fc, sctx->key, walk->dst.virt.addr, + walk->src.virt.addr, n, ctrptr); + if (ctrptr == ctrblk) + memcpy(walk->iv, ctrptr + n - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + crypto_inc(walk->iv, AES_BLOCK_SIZE); + ret = skcipher_walk_done(walk, nbytes - n); + } + + return ret; +} + static int ctr_aes_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm); - u8 buf[AES_BLOCK_SIZE], *ctrptr; struct skcipher_walk walk; - unsigned int n, nbytes; - int ret, locked; + u8 buf[AES_BLOCK_SIZE]; + int ret; if (unlikely(!sctx->fc)) return fallback_skcipher_crypt(sctx, req, 0); - locked = mutex_trylock(&ctrblk_lock); - ret = skcipher_walk_virt(&walk, req, false); - while (!ret && ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE)) { - n = AES_BLOCK_SIZE; + if (ret) + return ret; - if (nbytes >= 2*AES_BLOCK_SIZE && locked) - n = __ctrblk_init(ctrblk, walk.iv, nbytes); - ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv; - cpacf_kmctr(sctx->fc, sctx->key, walk.dst.virt.addr, - walk.src.virt.addr, n, ctrptr); - if (ctrptr == ctrblk) - memcpy(walk.iv, ctrptr + n - AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - crypto_inc(walk.iv, AES_BLOCK_SIZE); - ret = skcipher_walk_done(&walk, nbytes - n); + if (down_trylock(&ctrblk_sem) == 0) { + ret = __ctr_aes_crypt(sctx, &walk, true); + up(&ctrblk_sem); + } else { + ret = __ctr_aes_crypt(sctx, &walk, false); } - if (locked) - mutex_unlock(&ctrblk_lock); + /* * final block may be < AES_BLOCK_SIZE, copy only nbytes */ - if (!ret && nbytes) { + if (!ret && walk.nbytes > 0) { memset(buf, 0, AES_BLOCK_SIZE); - memcpy(buf, walk.src.virt.addr, nbytes); + memcpy(buf, walk.src.virt.addr, walk.nbytes); cpacf_kmctr(sctx->fc, sctx->key, buf, buf, AES_BLOCK_SIZE, walk.iv); - memcpy(walk.dst.virt.addr, buf, nbytes); + memcpy(walk.dst.virt.addr, buf, walk.nbytes); crypto_inc(walk.iv, AES_BLOCK_SIZE); ret = skcipher_walk_done(&walk, 0); memzero_explicit(buf, sizeof(buf)); From 403648373816ae8eb76fe4a393836b0d65fa8f85 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2026 10:38:30 +0200 Subject: [PATCH 10/18] s390/crypto: Fix return code handling at skcipher_walk_done in PAES algorithms All the 4 PAES cipher processing loops were not checking the return value of skcipher_walk_done() immediately after calling it. This could lead to error masking when both the walk operation failed and a subsequent key conversion was needed (k < n condition). Add immediate error checks after skcipher_walk_done() in all main processing loops (ECB, CBC, CTR, XTS modes) to ensure walk errors are properly propagated and not masked by subsequent operations. With that comes a slight rework around the skcipher_walk_done() invocation. It is now necessary to check if the walk has already been finalized (walk->nbytes is then 0) or not to avoid double de-allocation of resources held by the walk. Fixes: 6cd87cb5ef6c ("s390/crypto: Rework protected key AES for true asynch support") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Cc: stable@vger.kernel.org # 6.16+ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/paes_s390.c | 38 +++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 973436592318..89785ab95e6b 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -432,8 +432,11 @@ static int ecb_paes_do_crypt(struct s390_paes_ctx *ctx, n = nbytes & ~(AES_BLOCK_SIZE - 1); k = cpacf_km(ctx->fc | req_ctx->modifier, param, walk->dst.virt.addr, walk->src.virt.addr, n); - if (k) + if (k) { rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; + } if (k < n) { if (!maysleep) { rc = -EKEYEXPIRED; @@ -495,7 +498,7 @@ static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier) atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && walk->nbytes) skcipher_walk_done(walk, rc); out: @@ -549,7 +552,7 @@ static int ecb_paes_do_one_request(struct crypto_engine *engine, void *areq) rc = ecb_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { return pkey_handle_expired(); - } else if (rc) { + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } @@ -690,6 +693,8 @@ static int cbc_paes_do_crypt(struct s390_paes_ctx *ctx, if (k) { memcpy(walk->iv, param->iv, AES_BLOCK_SIZE); rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; } if (k < n) { if (!maysleep) { @@ -752,7 +757,7 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && walk->nbytes) skcipher_walk_done(walk, rc); out: @@ -806,7 +811,7 @@ static int cbc_paes_do_one_request(struct crypto_engine *engine, void *areq) rc = cbc_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { return pkey_handle_expired(); - } else if (rc) { + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } @@ -968,6 +973,11 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, AES_BLOCK_SIZE); crypto_inc(walk->iv, AES_BLOCK_SIZE); rc = skcipher_walk_done(walk, nbytes - k); + if (rc) { + if (locked) + mutex_unlock(&ctrblk_lock); + goto out; + } } if (k < n) { if (!maysleep) { @@ -1061,7 +1071,7 @@ static int ctr_paes_crypt(struct skcipher_request *req) atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && walk->nbytes) skcipher_walk_done(walk, rc); out: @@ -1105,7 +1115,7 @@ static int ctr_paes_do_one_request(struct crypto_engine *engine, void *areq) rc = ctr_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { return pkey_handle_expired(); - } else if (rc) { + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } @@ -1283,8 +1293,11 @@ static int xts_paes_do_crypt_fullkey(struct s390_pxts_ctx *ctx, n = nbytes & ~(AES_BLOCK_SIZE - 1); k = cpacf_km(ctx->fc | req_ctx->modifier, param->key + offset, walk->dst.virt.addr, walk->src.virt.addr, n); - if (k) + if (k) { rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; + } if (k < n) { if (!maysleep) { rc = -EKEYEXPIRED; @@ -1377,8 +1390,11 @@ static int xts_paes_do_crypt_2keys(struct s390_pxts_ctx *ctx, n = nbytes & ~(AES_BLOCK_SIZE - 1); k = cpacf_km(ctx->fc | req_ctx->modifier, param->key + offset, walk->dst.virt.addr, walk->src.virt.addr, n); - if (k) + if (k) { rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; + } if (k < n) { if (!maysleep) { rc = -EKEYEXPIRED; @@ -1485,7 +1501,7 @@ static inline int xts_paes_crypt(struct skcipher_request *req, unsigned long mod atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (rc != -EINPROGRESS && walk->nbytes) skcipher_walk_done(walk, rc); out: @@ -1539,7 +1555,7 @@ static int xts_paes_do_one_request(struct crypto_engine *engine, void *areq) rc = xts_paes_do_crypt(ctx, req_ctx, tested, true); if (rc == -EKEYEXPIRED) { return pkey_handle_expired(); - } else if (rc) { + } else if (rc && walk->nbytes) { skcipher_walk_done(walk, rc); } From 19a218b46b2471d370c11fb52040f36ac8d05d23 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2026 10:38:31 +0200 Subject: [PATCH 11/18] s390/crypto: Fix missing scrub of temp buffers with PAES algorithm In function ctr_paes_do_crypt() there is a buffer used to process remaining bytes < AES_BLOCK_SIZE. This buffer was not scrubbed and thus could lead to expose of unwanted data. Rework the code to explicitly scrub the buffer at the end of the function to avoid exposure of maybe sensitive data. In function __xts_2keys_prep_param() change the existing scrub to clean the whole param block instead of just the key field. Fixes: 6cd87cb5ef6c ("s390/crypto: Rework protected key AES for true asynch support") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Cc: stable@vger.kernel.org # 6.16+ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/paes_s390.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 89785ab95e6b..9c8f9e2570f2 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -1026,6 +1026,7 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, } out: + memzero_explicit(buf, sizeof(buf)); pr_debug("rc=%d\n", rc); return rc; } @@ -1350,7 +1351,7 @@ static inline int __xts_2keys_prep_param(struct s390_pxts_ctx *ctx, memcpy(param->init, pcc_param.xts, 16); } - memzero_explicit(pcc_param.key, sizeof(pcc_param.key)); + memzero_explicit(&pcc_param, sizeof(pcc_param)); return rc; } From 749990db95d45cc3d96aab999ac7f110259b200c Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2026 10:38:32 +0200 Subject: [PATCH 12/18] s390/crypto: Fix use of mutex in atomic context in PAES The PAES CTR implementation used a mutex to lock one page of exclusive memory for fast CTR processing. Unfortunately a mutex is not save to use in atomic or interrupt context. So use a binary semaphore instead which is save to use in such environments. Furthermore rework the code to get rid of conditional locking. So restructure the PAES CRT code by extracting the main loop into a separate function and just give in information about the (locked) page can be used or not (is not locked). Fixes: 6cd87cb5ef6c ("s390/crypto: Rework protected key AES for true asynch support") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Cc: stable@vger.kernel.org # 6.16+ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/paes_s390.c | 113 +++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 53 deletions(-) diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 9c8f9e2570f2..991a26766d40 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include @@ -45,7 +45,7 @@ module_param_named(clrkey, pkey_clrkey_allowed, bool, 0444); MODULE_PARM_DESC(clrkey, "Allow clear key material (default N)"); static u8 *ctrblk; -static DEFINE_MUTEX(ctrblk_lock); +static DEFINE_SEMAPHORE(ctrblk_sem, 1); static cpacf_mask_t km_functions, kmc_functions, kmctr_functions; @@ -919,15 +919,62 @@ static inline unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes return n; } +static int __ctr_paes_do_crypt(struct s390_paes_ctx *ctx, + struct ctr_param *param, + struct skcipher_walk *walk, + bool tested, bool maysleep, bool locked) +{ + unsigned int nbytes, n, k; + u8 *ctrptr; + int rc = 0; + + /* + * Note that in case of partial processing or failure the walk + * is NOT unmapped here. So a follow up task may reuse the walk + * or in case of unrecoverable failure needs to unmap it. + */ + while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { + n = AES_BLOCK_SIZE; + if (nbytes >= 2 * AES_BLOCK_SIZE && locked) + n = __ctrblk_init(ctrblk, walk->iv, nbytes); + ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; + k = cpacf_kmctr(ctx->fc, param, walk->dst.virt.addr, + walk->src.virt.addr, n, ctrptr); + if (k) { + if (ctrptr == ctrblk) + memcpy(walk->iv, ctrptr + k - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + crypto_inc(walk->iv, AES_BLOCK_SIZE); + rc = skcipher_walk_done(walk, nbytes - k); + if (rc) + goto out; + } + if (k < n) { + if (!maysleep) { + rc = -EKEYEXPIRED; + goto out; + } + rc = paes_convert_key(ctx, tested); + if (rc) + goto out; + spin_lock_bh(&ctx->pk_lock); + memcpy(param->key, ctx->pk.protkey, sizeof(param->key)); + spin_unlock_bh(&ctx->pk_lock); + } + } + +out: + return rc; +} + static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, struct s390_pctr_req_ctx *req_ctx, bool tested, bool maysleep) { struct ctr_param *param = &req_ctx->param; struct skcipher_walk *walk = &req_ctx->walk; - u8 buf[AES_BLOCK_SIZE], *ctrptr; - unsigned int nbytes, n, k; - int pk_state, locked, rc = 0; + u8 buf[AES_BLOCK_SIZE]; + int pk_state, rc = 0; if (!req_ctx->param_init_done) { /* fetch and check protected key state */ @@ -953,57 +1000,17 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, if (rc) goto out; - locked = mutex_trylock(&ctrblk_lock); - - /* - * Note that in case of partial processing or failure the walk - * is NOT unmapped here. So a follow up task may reuse the walk - * or in case of unrecoverable failure needs to unmap it. - */ - while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) { - n = AES_BLOCK_SIZE; - if (nbytes >= 2 * AES_BLOCK_SIZE && locked) - n = __ctrblk_init(ctrblk, walk->iv, nbytes); - ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; - k = cpacf_kmctr(ctx->fc, param, walk->dst.virt.addr, - walk->src.virt.addr, n, ctrptr); - if (k) { - if (ctrptr == ctrblk) - memcpy(walk->iv, ctrptr + k - AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - crypto_inc(walk->iv, AES_BLOCK_SIZE); - rc = skcipher_walk_done(walk, nbytes - k); - if (rc) { - if (locked) - mutex_unlock(&ctrblk_lock); - goto out; - } - } - if (k < n) { - if (!maysleep) { - if (locked) - mutex_unlock(&ctrblk_lock); - rc = -EKEYEXPIRED; - goto out; - } - rc = paes_convert_key(ctx, tested); - if (rc) { - if (locked) - mutex_unlock(&ctrblk_lock); - goto out; - } - spin_lock_bh(&ctx->pk_lock); - memcpy(param->key, ctx->pk.protkey, sizeof(param->key)); - spin_unlock_bh(&ctx->pk_lock); - } + if (down_trylock(&ctrblk_sem) == 0) { + rc = __ctr_paes_do_crypt(ctx, param, walk, tested, maysleep, true); + up(&ctrblk_sem); + } else { + rc = __ctr_paes_do_crypt(ctx, param, walk, tested, maysleep, false); } - if (locked) - mutex_unlock(&ctrblk_lock); /* final block may be < AES_BLOCK_SIZE, copy only nbytes */ - if (nbytes) { + if (!rc && walk->nbytes > 0) { memset(buf, 0, AES_BLOCK_SIZE); - memcpy(buf, walk->src.virt.addr, nbytes); + memcpy(buf, walk->src.virt.addr, walk->nbytes); while (1) { if (cpacf_kmctr(ctx->fc, param, buf, buf, AES_BLOCK_SIZE, @@ -1020,7 +1027,7 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, memcpy(param->key, ctx->pk.protkey, sizeof(param->key)); spin_unlock_bh(&ctx->pk_lock); } - memcpy(walk->dst.virt.addr, buf, nbytes); + memcpy(walk->dst.virt.addr, buf, walk->nbytes); crypto_inc(walk->iv, AES_BLOCK_SIZE); rc = skcipher_walk_done(walk, 0); } From 5b97b969030c099333d973be420edef0d6452e0f Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2026 10:38:33 +0200 Subject: [PATCH 13/18] s390/crypto: Fix missing cra_flags in paes_s390 The 4 algorithms implemented in paes_s390 never had any cra_flags set. So add code which sets the cra_flag to CRYPTO_ALG_ASYNC and CRYPTO_ALG_NO_FALLBACK. Fixes: 4ccd065a69df ("crypto: ahash - Add support for drivers with no fallback") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Cc: stable@vger.kernel.org # 6.17+ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/paes_s390.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 991a26766d40..5b7664031ea3 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -570,6 +570,7 @@ static struct skcipher_engine_alg ecb_paes_alg = { .base.cra_name = "ecb(paes)", .base.cra_driver_name = "ecb-paes-s390", .base.cra_priority = 401, /* combo: aes + ecb + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -829,6 +830,7 @@ static struct skcipher_engine_alg cbc_paes_alg = { .base.cra_name = "cbc(paes)", .base.cra_driver_name = "cbc-paes-s390", .base.cra_priority = 402, /* cbc-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -1141,6 +1143,7 @@ static struct skcipher_engine_alg ctr_paes_alg = { .base.cra_name = "ctr(paes)", .base.cra_driver_name = "ctr-paes-s390", .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -1581,6 +1584,7 @@ static struct skcipher_engine_alg xts_paes_alg = { .base.cra_name = "xts(paes)", .base.cra_driver_name = "xts-paes-s390", .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_pxts_ctx), .base.cra_module = THIS_MODULE, From 3fec882c33d9b61983ee31456777f623dccd1a35 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2026 10:38:34 +0200 Subject: [PATCH 14/18] s390/crypto: Fix handling of EBUSY in PAES when req is pushed to crypto engine When a request is transferred to the engine via crypto_transfer_skcipher_request_to_engine() there are two return codes signaling a successful transfer: EINPROGRESS and EBUSY. However the correct handling of EBUSY was missing and has been added as a return code indicating a successful transfer to the crypto engine. Fixes: 6cd87cb5ef6c ("s390/crypto: Rework protected key AES for true asynch support") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Cc: stable@vger.kernel.org # 6.16+ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/paes_s390.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 5b7664031ea3..93e0e54ba2e8 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -463,6 +463,7 @@ static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier) struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -494,15 +495,17 @@ static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier) if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS && walk->nbytes) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; @@ -723,6 +726,7 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -754,15 +758,17 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier) if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS && walk->nbytes) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; @@ -1047,6 +1053,7 @@ static int ctr_paes_crypt(struct skcipher_request *req) struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -1077,15 +1084,17 @@ static int ctr_paes_crypt(struct skcipher_request *req) if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS && walk->nbytes) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; @@ -1477,6 +1486,7 @@ static inline int xts_paes_crypt(struct skcipher_request *req, unsigned long mod struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); struct skcipher_walk *walk = &req_ctx->walk; bool tested = crypto_skcipher_tested(tfm); + bool cleanup = true; int rc; /* @@ -1508,15 +1518,17 @@ static inline int xts_paes_crypt(struct skcipher_request *req, unsigned long mod if (rc == 0 || rc == -EKEYEXPIRED) { atomic_inc(&ctx->via_engine_ctr); rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&ctx->via_engine_ctr); } - if (rc != -EINPROGRESS && walk->nbytes) + if (cleanup && walk->nbytes) skcipher_walk_done(walk, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(&req_ctx->param, sizeof(req_ctx->param)); pr_debug("rc=%d\n", rc); return rc; From 330148371401de474b656eaf521861f12ec1a1ce Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2026 10:38:35 +0200 Subject: [PATCH 15/18] s390/crypto: Fix handling of EBUSY in PHMAC when req is pushed to crypto engine When a request is transferred to the engine via crypto_transfer_hash_request_to_engine() there are two return codes signaling a successful transfer: EINPROGRESS and EBUSY. However the correct handling of EBUSY was missing and has been added as a return code indicating a successful transfer to the crypto engine. Fixes: cbbc675506cc ("crypto: s390 - New s390 specific protected key hash phmac") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Cc: stable@vger.kernel.org # 6.17+ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/phmac_s390.c | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/arch/s390/crypto/phmac_s390.c b/arch/s390/crypto/phmac_s390.c index 020a1beb2e22..532fe6c4e89c 100644 --- a/arch/s390/crypto/phmac_s390.c +++ b/arch/s390/crypto/phmac_s390.c @@ -62,8 +62,10 @@ static inline int hwh_prepare(struct ahash_request *req, */ static inline int hwh_advance(struct hash_walk_helper *hwh, int n) { - if (n < 0) + if (n < 0) { + hwh->walkbytes = n; return crypto_hash_walk_done(&hwh->walk, n); + } hwh->walkbytes -= n; hwh->walkaddr += n; @@ -606,6 +608,7 @@ static int phmac_update(struct ahash_request *req) struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; struct hash_walk_helper *hwh = &req_ctx->hwh; + bool cleanup = true; int rc; /* prep the walk in the request context */ @@ -629,12 +632,15 @@ static int phmac_update(struct ahash_request *req) req_ctx->async_op = OP_UPDATE; atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) { - hwh_advance(hwh, rc); + if (cleanup) { + if (hwh->walkbytes > 0) + hwh_advance(hwh, rc); memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); } @@ -649,6 +655,7 @@ static int phmac_final(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; + bool cleanup = true; int rc = 0; /* Try synchronous operation if no active engine usage */ @@ -667,12 +674,14 @@ static int phmac_final(struct ahash_request *req) req_ctx->async_op = OP_FINAL; atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); pr_debug("rc=%d\n", rc); return rc; @@ -685,6 +694,7 @@ static int phmac_finup(struct ahash_request *req) struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; struct hash_walk_helper *hwh = &req_ctx->hwh; + bool cleanup = true; int rc; /* prep the walk in the request context */ @@ -716,15 +726,17 @@ static int phmac_finup(struct ahash_request *req) /* req->async_op has been set to either OP_FINUP or OP_FINAL */ atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && hwh->walkbytes > 0) hwh_advance(hwh, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); pr_debug("rc=%d\n", rc); return rc; From ac1481320110b803ab9b79ab4d2ca11a74fc05f2 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2026 10:38:36 +0200 Subject: [PATCH 16/18] s390/crypto: Fix wrong return code to engine in asynch callbacks When crypto_finalize_hash_request() or crypto_finalize_skcipher_request() explicitly completes a request, the do_one_request callback must return 0 to indicate successful handling. Returning a negative error code causes the crypto engine to assume the driver failed to take ownership and triggers a second completion via crypto_request_complete(), resulting in a double completion. This pattern occurs in paes_s390.c 4 times and once in phmac_s390.c. Fixed in phmac_do_one_request() and all four paes do_one_request callbacks (ecb, cbc, ctr, xts) by returning 0 after explicit finalization instead of propagating the error code. Fixes: 6cd87cb5ef6c ("s390/crypto: Rework protected key AES for true asynch support") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Cc: stable@vger.kernel.org # 6.16+ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/paes_s390.c | 8 ++++---- arch/s390/crypto/phmac_s390.c | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 93e0e54ba2e8..a4b972459f52 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -565,7 +565,7 @@ static int ecb_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg ecb_paes_alg = { @@ -828,7 +828,7 @@ static int cbc_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg cbc_paes_alg = { @@ -1144,7 +1144,7 @@ static int ctr_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg ctr_paes_alg = { @@ -1588,7 +1588,7 @@ static int xts_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg xts_paes_alg = { diff --git a/arch/s390/crypto/phmac_s390.c b/arch/s390/crypto/phmac_s390.c index 532fe6c4e89c..283a00754a06 100644 --- a/arch/s390/crypto/phmac_s390.c +++ b/arch/s390/crypto/phmac_s390.c @@ -926,7 +926,7 @@ static int phmac_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&tfm_ctx->via_engine_ctr); crypto_finalize_hash_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } #define S390_ASYNC_PHMAC_ALG(x) \ From 7a08507ea5b4d06ad8d269287913573f34467565 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2026 10:38:37 +0200 Subject: [PATCH 17/18] s390/crypto: Map EBUSY to EIO when key conversion fails repeatedly When hardware persistently returns -EBUSY after exhausting retries, the error propagates to crypto_finalize_*_request(). The crypto API's completion wrapper treats -EBUSY as a queueing status and swallows it, preventing the completion callback from firing. This causes callers using crypto_wait_req() to block indefinitely. Translate persistent -EBUSY to -EIO after retry exhaustion to ensure proper error propagation and callback invocation. Fixes: 6cd87cb5ef6c ("s390/crypto: Rework protected key AES for true asynch support") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Cc: stable@vger.kernel.org # 6.16+ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/paes_s390.c | 4 ++++ arch/s390/crypto/phmac_s390.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index a4b972459f52..f987bcbe8f35 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -220,6 +220,10 @@ static inline int convert_key(const u8 *key, unsigned int keylen, xflags); } + /* But finally map -EBUSY to -EIO to indicate an IO failure */ + if (rc == -EBUSY) + rc = -EIO; + out: pr_debug("rc=%d\n", rc); return rc; diff --git a/arch/s390/crypto/phmac_s390.c b/arch/s390/crypto/phmac_s390.c index 283a00754a06..bbf8a6809ecb 100644 --- a/arch/s390/crypto/phmac_s390.c +++ b/arch/s390/crypto/phmac_s390.c @@ -341,6 +341,10 @@ static inline int convert_key(const u8 *key, unsigned int keylen, xflags); } + /* But finally map -EBUSY to -EIO to indicate an IO failure */ + if (rc == -EBUSY) + rc = -EIO; + out: pr_debug("rc=%d\n", rc); return rc; From dc2136341be9835e70ba7c6b36904cf3683fd029 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 31 Aug 2026 10:38:38 +0200 Subject: [PATCH 18/18] s390/crypto: Enable CONTEXT_ANALYSIS Enable CONTEXT_ANALYSIS since s390's crypto code compiles now without warnings. Reviewed-by: Harald Freudenberger Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/crypto/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 48aeb0c0ffbd..1d6420813935 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -3,6 +3,8 @@ # Cryptographic API # +CONTEXT_ANALYSIS := y + obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o obj-$(CONFIG_CRYPTO_PAES_S390) += paes_s390.o obj-$(CONFIG_S390_PRNG) += prng.o