From 99269799bf2448aebccee164df56c22a7b85b02c Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Tue, 5 May 2026 12:34:33 +0200 Subject: [PATCH] s390/pai: Fix missing PAI counter increments under heavy load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Machines with a larger number of CPUs and under heavy load sometimes loose PAI counter increments during recording using events -e CRYPTO_ÂLL or -e NNPA_ALL. Counting is not affected. This happens when several PAI crypto counters are incremented during the same cryptographic operation. During schedule out the functions paiXXX_sched_task() (with XXX either crypt or ext) +--> pai_have_samples() +--> pai_have_sample() +--> pai_copy() +--> pai_push_sample() are called to read out PAI counter values. In pai_copy() the current values of PAI counters are read from the PMU memory mapped page and compared to the values read during last schedule out operation, which have been saved in a backup page named PAI_SAVE_AREA(event). For each PAI counter a delta is calculated and when the delta is positive, that PAI counter was incremented by hardware. This positve delta is reported as raw data record attached to a sample. After all deltas have been calculated, the new PAI counter values are saved in the backup page PAI_SAVE_AREA(event). However this is done in pai_push_sample(), leaving a small window for missing hardware triggered updates. Here is one scenario: PAI counter idx: 0 1 2 3 4 5 6 7 .... N +---+---+---+---+---+---+---+---+ +---+ PAI counter page:| | | X | | | | | |....| Y | +---+---+---+---+---+---+---+---+ +---+ In pai_copy() each PAI counter value is read and compared to its old value. This is done in a loop. When PAI counter indexed N is read, the hardware might increment PAI counter indexed 2 again, updating its value from X to X+1. Later pai_push_sample() simply mem-copies the complete PAI counter page to a backup page and the increment of X+1 is lost, because the backup page now contains the new value. Read each PAI counter and save this value in the backup page when there is a positive delta. This omits any time window between read and store. This also reduced the work load as only modified PAI counters are saved. Cc: stable@vger.kernel.org Fixes: fe861b0c8d06 ("s390/pai: save PAI counter value page in event structure") Signed-off-by: Thomas Richter Reviewed-by: Sumanth Korikkar Signed-off-by: Alexander Gordeev --- arch/s390/kernel/perf_pai.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/arch/s390/kernel/perf_pai.c b/arch/s390/kernel/perf_pai.c index f13c5c5fbea6..cdb8006220ca 100644 --- a/arch/s390/kernel/perf_pai.c +++ b/arch/s390/kernel/perf_pai.c @@ -186,6 +186,13 @@ static u64 pai_getctr(unsigned long *page, int nr, unsigned long offset) return page[nr]; } +static void pai_setctr(unsigned long *page, int nr, unsigned long offset, u64 v) +{ + if (offset) + nr += offset / sizeof(*page); + page[nr] = v; +} + /* Read the counter values. Return value from location in CMP. For base * event xxx_ALL sum up all events. Returns counter value. */ @@ -551,6 +558,8 @@ static void paicrypt_del(struct perf_event *event, int flags) /* Create raw data and save it in buffer. Calculate the delta for each * counter between this invocation and the last invocation. * Returns number of bytes copied. + * After reading from PAI counter page, save the read value to the old + * page to calculate PAI counter deltas. * Saves only entries with positive counter difference of the form * 2 bytes: Number of counter * 8 bytes: Value of counter @@ -562,16 +571,22 @@ static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page, int i, outidx = 0; for (i = 1; i <= pp->num_avail; i++) { - u64 val = 0, val_old = 0; + u64 val = 0, val_old = 0, val_k = 0, val_old_k = 0; if (!exclude_kernel) { - val += pai_getctr(page, i, pp->kernel_offset); - val_old += pai_getctr(page_old, i, pp->kernel_offset); + val_k = pai_getctr(page, i, pp->kernel_offset); + val_old_k = pai_getctr(page_old, i, pp->kernel_offset); + if (val_k != val_old_k) + pai_setctr(page_old, i, pp->kernel_offset, val_k); } if (!exclude_user) { - val += pai_getctr(page, i, 0); - val_old += pai_getctr(page_old, i, 0); + val = pai_getctr(page, i, 0); + val_old = pai_getctr(page_old, i, 0); + if (val != val_old) + pai_setctr(page_old, i, 0, val); } + val += val_k; + val_old += val_old_k; if (val >= val_old) val -= val_old; else @@ -602,8 +617,6 @@ static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page, static int pai_push_sample(size_t rawsize, struct pai_map *cpump, struct perf_event *event) { - int idx = PAI_PMU_IDX(event); - struct pai_pmu *pp = &pai_pmu[idx]; struct perf_sample_data data; struct perf_raw_record raw; struct pt_regs regs; @@ -634,8 +647,6 @@ static int pai_push_sample(size_t rawsize, struct pai_map *cpump, overflow = perf_event_overflow(event, &data, ®s); perf_event_update_userpage(event); - /* Save crypto counter lowcore page after reading event data. */ - memcpy((void *)PAI_SAVE_AREA(event), cpump->area, pp->area_size); return overflow; }