github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-14 08:01:12 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

s390 updates for 7.3-rc3

Browse Source 
- Fix NULL pointer dereferences in s390dbf when setting debug levels or
   resizing debug areas while logging events. Remove duplicate messages
   about kernel parameter overrides
 
 - Fix PAI perf crashes when per task events move to newly onlined CPUs.
   Add CPU hotplug callbacks to allocate and free the per-CPU data
 
 - Fix mutex use in atomic context in AES and PAES CTR code by using
   semaphore trylocks instead. Remove conditional locking and enable
   Clang CONTEXT_ANALYSIS for the crypto code
 
 - Fix scatterlist walk error handling in AES and PAES and avoid freeing
   PAES walk resources twice
 
 - Fix missing scrubbing of temporary AES and PAES buffers, including AES
   GCM error paths
 
 - Set missing CRYPTO_ALG_ASYNC and CRYPTO_ALG_NO_FALLBACK flags for PAES
 
 - Fix -EBUSY handling in PAES and PHMAC to avoid cleaning up requests
   already queued to the crypto engine
 
 - Fix PAES and PHMAC requests being completed twice on errors
 
 - Fix PAES and PHMAC hangs when key conversion keeps returning -EBUSY
   by returning -EIO after the last retry
 -----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCgAdFiEE3QHqV+H2a8xAv27vjYWKoQLXFBgFAmqlzRcACgkQjYWKoQLX
 FBg65gf9H1AoBZnwcgcjhGzJL93sqj8nm9BRquCIdHi85FxypAYFX4nM8o7ESUWj
 HaIuleZA6OncKWBdHSEkBOj8fak+6RixjK7j1chUXzg0+J/bmTSWWE3j9zw8ZWUj
 TGY7yuvDtZ/XRefO3yxirh9Nr7OyS98FH7rxcwnlmKYz6AKQmLayLrMfw+E6BP+a
 Juw1aWPHRMvd+9JqakqzTOIfJNmNz34HT3yL0phqiHojU5mNsgGPUPjlD6jNkFCf
 6TdRVcnkHMuhTEx/HD+/8nHTlYDVFJkLbpe9WYk94ubVY6kLaLxuBWE1ycCk7bv8
 GOQygNWL5mZpL5c0MTtlm49ybIELuw==
 =cYX9
 -----END PGP SIGNATURE-----

Merge tag 's390-7.3-3' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux

Pull s390 fixes from Vasily Gorbik:

 - Fix NULL pointer dereferences in s390dbf when setting debug levels or
   resizing debug areas while logging events. Remove duplicate messages
   about kernel parameter overrides

 - Fix PAI perf crashes when per task events move to newly onlined CPUs.
   Add CPU hotplug callbacks to allocate and free the per-CPU data

 - Fix mutex use in atomic context in AES and PAES CTR code by using
   semaphore trylocks instead. Remove conditional locking and enable
   Clang CONTEXT_ANALYSIS for the crypto code

 - Fix scatterlist walk error handling in AES and PAES and avoid freeing
   PAES walk resources twice

 - Fix missing scrubbing of temporary AES and PAES buffers, including
   AES GCM error paths

 - Set missing CRYPTO_ALG_ASYNC and CRYPTO_ALG_NO_FALLBACK flags for
   PAES

 - Fix -EBUSY handling in PAES and PHMAC to avoid cleaning up requests
   already queued to the crypto engine

 - Fix PAES and PHMAC requests being completed twice on errors

 - Fix PAES and PHMAC hangs when key conversion keeps returning -EBUSY
   by returning -EIO after the last retry

* tag 's390-7.3-3' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux:
  s390/crypto: Enable CONTEXT_ANALYSIS
  s390/crypto: Map EBUSY to EIO when key conversion fails repeatedly
  s390/crypto: Fix wrong return code to engine in asynch callbacks
  s390/crypto: Fix handling of EBUSY in PHMAC when req is pushed to crypto engine
  s390/crypto: Fix handling of EBUSY in PAES when req is pushed to crypto engine
  s390/crypto: Fix missing cra_flags in paes_s390
  s390/crypto: Fix use of mutex in atomic context in PAES
  s390/crypto: Fix missing scrub of temp buffers with PAES algorithm
  s390/crypto: Fix return code handling at skcipher_walk_done in PAES algorithms
  s390/crypto: Fix use of mutex in atomic context
  s390/crypto: Fix missing scrub of temp buffers with AES ctr and gcm algorithm
  s390/crypto: Fix skcipher_walk return code handling in aes_s390
  s390/debug: Fix race between debug area resize and event logging
  s390/debug: Do not repeat parameter override notice on debug_set_level()
  s390/debug: Fix NULL pointer dereference in debug_set_level()
  s390/pai: Support CPU hotplug for PMU PAI
  s390/pai: Move locking to event init and delete
  s390/pai: Use PAI PMU index as parameter replacing event
This commit is contained in:
Linus Torvalds 2026-09-12 16:22:25 -07:00
commit 2f0c1cf72f
7 changed files with 371 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/s390/crypto/Makefile
View File

@ -3,6 +3,8 @@
# Cryptographic API
#
CONTEXT_ANALYSIS := y
obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o
obj-$(CONFIG_CRYPTO_PAES_S390) += paes_s390.o
obj-$(CONFIG_S390_PRNG) += prng.o

87
arch/s390/crypto/aes_s390.c
View File

@ -26,14 +26,14 @@
#include <linux/module.h>
#include <linux/cpufeature.h>
#include <linux/init.h>
#include <linux/mutex.h>
#include <linux/fips.h>
#include <linux/semaphore.h>
#include <linux/string.h>
#include <crypto/xts.h>
#include <asm/cpacf.h>
static u8 *ctrblk;
static DEFINE_MUTEX(ctrblk_lock);
static DEFINE_SEMAPHORE(ctrblk_sem, 1);
static cpacf_mask_t km_functions, kmc_functions, kmctr_functions,
kma_functions;
@ -129,7 +129,7 @@ static int ecb_aes_crypt(struct skcipher_request *req, unsigned long modifier)
return fallback_skcipher_crypt(sctx, req, modifier);
ret = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes) != 0) {
while (!ret && ((nbytes = walk.nbytes) != 0)) {
/* only use complete blocks */
n = nbytes & ~(AES_BLOCK_SIZE - 1);
cpacf_km(sctx->fc | modifier, sctx->key,
@ -233,7 +233,7 @@ static int cbc_aes_crypt(struct skcipher_request *req, unsigned long modifier)
return ret;
memcpy(param.iv, walk.iv, AES_BLOCK_SIZE);
memcpy(param.key, sctx->key, sctx->key_len);
while ((nbytes = walk.nbytes) != 0) {
while (!ret && ((nbytes = walk.nbytes) != 0)) {
/* only use complete blocks */
n = nbytes & ~(AES_BLOCK_SIZE - 1);
cpacf_kmc(sctx->fc | modifier, &param,
@ -359,7 +359,7 @@ static int xts_aes_crypt(struct skcipher_request *req, unsigned long modifier)
memcpy(xts_param.key + offset, xts_ctx->key, xts_ctx->key_len);
memcpy(xts_param.init, pcc_param.xts, 16);
while ((nbytes = walk.nbytes) != 0) {
while (!ret && ((nbytes = walk.nbytes) != 0)) {
/* only use complete blocks */
n = nbytes & ~(AES_BLOCK_SIZE - 1);
cpacf_km(xts_ctx->fc | modifier, xts_param.key + offset,
@ -487,7 +487,7 @@ static int fullxts_aes_crypt(struct skcipher_request *req, unsigned long modifi
memcpy(fxts_param.tweak, req->iv, AES_BLOCK_SIZE);
fxts_param.nap[0] = 0x01; /* initial alpha power (1, little-endian) */
while ((nbytes = walk.nbytes) != 0) {
while (!ret && ((nbytes = walk.nbytes) != 0)) {
/* only use complete blocks */
n = nbytes & ~(AES_BLOCK_SIZE - 1);
cpacf_km(xts_ctx->fc | modifier, fxts_param.key + offset,
@ -562,48 +562,64 @@ static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes)
return n;
}
static int __ctr_aes_crypt(struct s390_aes_ctx *sctx,
struct skcipher_walk *walk, bool locked)
{
unsigned int n, nbytes;
int ret = 0;
u8 *ctrptr;
while (!ret && ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE)) {
n = AES_BLOCK_SIZE;
if (nbytes >= 2 * AES_BLOCK_SIZE && locked)
n = __ctrblk_init(ctrblk, walk->iv, nbytes);
ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv;
cpacf_kmctr(sctx->fc, sctx->key, walk->dst.virt.addr,
walk->src.virt.addr, n, ctrptr);
if (ctrptr == ctrblk)
memcpy(walk->iv, ctrptr + n - AES_BLOCK_SIZE,
AES_BLOCK_SIZE);
crypto_inc(walk->iv, AES_BLOCK_SIZE);
ret = skcipher_walk_done(walk, nbytes - n);
}
return ret;
}
static int ctr_aes_crypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm);
u8 buf[AES_BLOCK_SIZE], *ctrptr;
struct skcipher_walk walk;
unsigned int n, nbytes;
int ret, locked;
u8 buf[AES_BLOCK_SIZE];
int ret;
if (unlikely(!sctx->fc))
return fallback_skcipher_crypt(sctx, req, 0);
locked = mutex_trylock(&ctrblk_lock);
ret = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
n = AES_BLOCK_SIZE;
if (ret)
return ret;
if (nbytes >= 2*AES_BLOCK_SIZE && locked)
n = __ctrblk_init(ctrblk, walk.iv, nbytes);
ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv;
cpacf_kmctr(sctx->fc, sctx->key, walk.dst.virt.addr,
walk.src.virt.addr, n, ctrptr);
if (ctrptr == ctrblk)
memcpy(walk.iv, ctrptr + n - AES_BLOCK_SIZE,
AES_BLOCK_SIZE);
crypto_inc(walk.iv, AES_BLOCK_SIZE);
ret = skcipher_walk_done(&walk, nbytes - n);
if (down_trylock(&ctrblk_sem) == 0) {
ret = __ctr_aes_crypt(sctx, &walk, true);
up(&ctrblk_sem);
} else {
ret = __ctr_aes_crypt(sctx, &walk, false);
}
if (locked)
mutex_unlock(&ctrblk_lock);
/*
* final block may be < AES_BLOCK_SIZE, copy only nbytes
*/
if (nbytes) {
if (!ret && walk.nbytes > 0) {
memset(buf, 0, AES_BLOCK_SIZE);
memcpy(buf, walk.src.virt.addr, nbytes);
memcpy(buf, walk.src.virt.addr, walk.nbytes);
cpacf_kmctr(sctx->fc, sctx->key, buf, buf,
AES_BLOCK_SIZE, walk.iv);
memcpy(walk.dst.virt.addr, buf, nbytes);
memcpy(walk.dst.virt.addr, buf, walk.nbytes);
crypto_inc(walk.iv, AES_BLOCK_SIZE);
ret = skcipher_walk_done(&walk, 0);
memzero_explicit(buf, sizeof(buf));
}
return ret;
@ -895,10 +911,14 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags)
gw_in.ptr, aad_bytes);
n = aad_bytes + pc_bytes;
if (gcm_in_walk_done(&gw_in, n) != n)
return -ENOMEM;
if (gcm_out_walk_done(&gw_out, n) != n)
return -ENOMEM;
if (gcm_in_walk_done(&gw_in, n) != n) {
ret = -ENOMEM;
goto out;
}
if (gcm_out_walk_done(&gw_out, n) != n) {
ret = -ENOMEM;
goto out;
}
aadlen -= aad_bytes;
pclen -= pc_bytes;
} while (aadlen + pclen > 0);
@ -910,7 +930,10 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags)
} else
scatterwalk_map_and_copy(param.t, req->dst, len, taglen, 1);
out:
memzero_explicit(&param, sizeof(param));
memzero_explicit(gw_in.buf, sizeof(gw_in.buf));
memzero_explicit(gw_out.buf, sizeof(gw_out.buf));
return ret;
}

188
arch/s390/crypto/paes_s390.c
View File

@ -19,7 +19,7 @@
#include <linux/init.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/semaphore.h>
#include <linux/spinlock.h>
#include <crypto/aes.h>
#include <crypto/algapi.h>
@ -45,7 +45,7 @@ module_param_named(clrkey, pkey_clrkey_allowed, bool, 0444);
MODULE_PARM_DESC(clrkey, "Allow clear key material (default N)");
static u8 *ctrblk;
static DEFINE_MUTEX(ctrblk_lock);
static DEFINE_SEMAPHORE(ctrblk_sem, 1);
static cpacf_mask_t km_functions, kmc_functions, kmctr_functions;
@ -220,6 +220,10 @@ static inline int convert_key(const u8 *key, unsigned int keylen,
xflags);
}
/* But finally map -EBUSY to -EIO to indicate an IO failure */
if (rc == -EBUSY)
rc = -EIO;
out:
pr_debug("rc=%d\n", rc);
return rc;
@ -432,8 +436,11 @@ static int ecb_paes_do_crypt(struct s390_paes_ctx *ctx,
n = nbytes & ~(AES_BLOCK_SIZE - 1);
k = cpacf_km(ctx->fc | req_ctx->modifier, param,
walk->dst.virt.addr, walk->src.virt.addr, n);
if (k)
if (k) {
rc = skcipher_walk_done(walk, nbytes - k);
if (rc)
goto out;
}
if (k < n) {
if (!maysleep) {
rc = -EKEYEXPIRED;
@ -460,6 +467,7 @@ static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier)
struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk *walk = &req_ctx->walk;
bool tested = crypto_skcipher_tested(tfm);
bool cleanup = true;
int rc;
/*
@ -491,15 +499,17 @@ static int ecb_paes_crypt(struct skcipher_request *req, unsigned long modifier)
if (rc == 0 || rc == -EKEYEXPIRED) {
atomic_inc(&ctx->via_engine_ctr);
rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req);
if (rc != -EINPROGRESS)
if (rc == -EINPROGRESS || rc == -EBUSY)
cleanup = false;
else
atomic_dec(&ctx->via_engine_ctr);
}
if (rc != -EINPROGRESS)
if (cleanup && walk->nbytes)
skcipher_walk_done(walk, rc);
out:
if (rc != -EINPROGRESS)
if (cleanup)
memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
pr_debug("rc=%d\n", rc);
return rc;
@ -549,7 +559,7 @@ static int ecb_paes_do_one_request(struct crypto_engine *engine, void *areq)
rc = ecb_paes_do_crypt(ctx, req_ctx, tested, true);
if (rc == -EKEYEXPIRED) {
return pkey_handle_expired();
} else if (rc) {
} else if (rc && walk->nbytes) {
skcipher_walk_done(walk, rc);
}
@ -559,7 +569,7 @@ static int ecb_paes_do_one_request(struct crypto_engine *engine, void *areq)
atomic_dec(&ctx->via_engine_ctr);
crypto_finalize_skcipher_request(engine, req, rc);
local_bh_enable();
return rc;
return 0;
}
static struct skcipher_engine_alg ecb_paes_alg = {
@ -567,6 +577,7 @@ static struct skcipher_engine_alg ecb_paes_alg = {
.base.cra_name = "ecb(paes)",
.base.cra_driver_name = "ecb-paes-s390",
.base.cra_priority = 401, /* combo: aes + ecb + 1 */
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct s390_paes_ctx),
.base.cra_module = THIS_MODULE,
@ -690,6 +701,8 @@ static int cbc_paes_do_crypt(struct s390_paes_ctx *ctx,
if (k) {
memcpy(walk->iv, param->iv, AES_BLOCK_SIZE);
rc = skcipher_walk_done(walk, nbytes - k);
if (rc)
goto out;
}
if (k < n) {
if (!maysleep) {
@ -717,6 +730,7 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier)
struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk *walk = &req_ctx->walk;
bool tested = crypto_skcipher_tested(tfm);
bool cleanup = true;
int rc;
/*
@ -748,15 +762,17 @@ static int cbc_paes_crypt(struct skcipher_request *req, unsigned long modifier)
if (rc == 0 || rc == -EKEYEXPIRED) {
atomic_inc(&ctx->via_engine_ctr);
rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req);
if (rc != -EINPROGRESS)
if (rc == -EINPROGRESS || rc == -EBUSY)
cleanup = false;
else
atomic_dec(&ctx->via_engine_ctr);
}
if (rc != -EINPROGRESS)
if (cleanup && walk->nbytes)
skcipher_walk_done(walk, rc);
out:
if (rc != -EINPROGRESS)
if (cleanup)
memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
pr_debug("rc=%d\n", rc);
return rc;
@ -806,7 +822,7 @@ static int cbc_paes_do_one_request(struct crypto_engine *engine, void *areq)
rc = cbc_paes_do_crypt(ctx, req_ctx, tested, true);
if (rc == -EKEYEXPIRED) {
return pkey_handle_expired();
} else if (rc) {
} else if (rc && walk->nbytes) {
skcipher_walk_done(walk, rc);
}
@ -816,7 +832,7 @@ static int cbc_paes_do_one_request(struct crypto_engine *engine, void *areq)
atomic_dec(&ctx->via_engine_ctr);
crypto_finalize_skcipher_request(engine, req, rc);
local_bh_enable();
return rc;
return 0;
}
static struct skcipher_engine_alg cbc_paes_alg = {
@ -824,6 +840,7 @@ static struct skcipher_engine_alg cbc_paes_alg = {
.base.cra_name = "cbc(paes)",
.base.cra_driver_name = "cbc-paes-s390",
.base.cra_priority = 402, /* cbc-paes-s390 + 1 */
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct s390_paes_ctx),
.base.cra_module = THIS_MODULE,
@ -914,15 +931,62 @@ static inline unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes
return n;
}
static int __ctr_paes_do_crypt(struct s390_paes_ctx *ctx,
struct ctr_param *param,
struct skcipher_walk *walk,
bool tested, bool maysleep, bool locked)
{
unsigned int nbytes, n, k;
u8 *ctrptr;
int rc = 0;
/*
* Note that in case of partial processing or failure the walk
* is NOT unmapped here. So a follow up task may reuse the walk
* or in case of unrecoverable failure needs to unmap it.
*/
while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) {
n = AES_BLOCK_SIZE;
if (nbytes >= 2 * AES_BLOCK_SIZE && locked)
n = __ctrblk_init(ctrblk, walk->iv, nbytes);
ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv;
k = cpacf_kmctr(ctx->fc, param, walk->dst.virt.addr,
walk->src.virt.addr, n, ctrptr);
if (k) {
if (ctrptr == ctrblk)
memcpy(walk->iv, ctrptr + k - AES_BLOCK_SIZE,
AES_BLOCK_SIZE);
crypto_inc(walk->iv, AES_BLOCK_SIZE);
rc = skcipher_walk_done(walk, nbytes - k);
if (rc)
goto out;
}
if (k < n) {
if (!maysleep) {
rc = -EKEYEXPIRED;
goto out;
}
rc = paes_convert_key(ctx, tested);
if (rc)
goto out;
spin_lock_bh(&ctx->pk_lock);
memcpy(param->key, ctx->pk.protkey, sizeof(param->key));
spin_unlock_bh(&ctx->pk_lock);
}
}
out:
return rc;
}
static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx,
struct s390_pctr_req_ctx *req_ctx,
bool tested, bool maysleep)
{
struct ctr_param *param = &req_ctx->param;
struct skcipher_walk *walk = &req_ctx->walk;
u8 buf[AES_BLOCK_SIZE], *ctrptr;
unsigned int nbytes, n, k;
int pk_state, locked, rc = 0;
u8 buf[AES_BLOCK_SIZE];
int pk_state, rc = 0;
if (!req_ctx->param_init_done) {
/* fetch and check protected key state */
@ -948,52 +1012,17 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx,
if (rc)
goto out;
locked = mutex_trylock(&ctrblk_lock);
/*
* Note that in case of partial processing or failure the walk
* is NOT unmapped here. So a follow up task may reuse the walk
* or in case of unrecoverable failure needs to unmap it.
*/
while ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE) {
n = AES_BLOCK_SIZE;
if (nbytes >= 2 * AES_BLOCK_SIZE && locked)
n = __ctrblk_init(ctrblk, walk->iv, nbytes);
ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv;
k = cpacf_kmctr(ctx->fc, param, walk->dst.virt.addr,
walk->src.virt.addr, n, ctrptr);
if (k) {
if (ctrptr == ctrblk)
memcpy(walk->iv, ctrptr + k - AES_BLOCK_SIZE,
AES_BLOCK_SIZE);
crypto_inc(walk->iv, AES_BLOCK_SIZE);
rc = skcipher_walk_done(walk, nbytes - k);
}
if (k < n) {
if (!maysleep) {
if (locked)
mutex_unlock(&ctrblk_lock);
rc = -EKEYEXPIRED;
goto out;
}
rc = paes_convert_key(ctx, tested);
if (rc) {
if (locked)
mutex_unlock(&ctrblk_lock);
goto out;
}
spin_lock_bh(&ctx->pk_lock);
memcpy(param->key, ctx->pk.protkey, sizeof(param->key));
spin_unlock_bh(&ctx->pk_lock);
}
if (down_trylock(&ctrblk_sem) == 0) {
rc = __ctr_paes_do_crypt(ctx, param, walk, tested, maysleep, true);
up(&ctrblk_sem);
} else {
rc = __ctr_paes_do_crypt(ctx, param, walk, tested, maysleep, false);
}
if (locked)
mutex_unlock(&ctrblk_lock);
/* final block may be < AES_BLOCK_SIZE, copy only nbytes */
if (nbytes) {
if (!rc && walk->nbytes > 0) {
memset(buf, 0, AES_BLOCK_SIZE);
memcpy(buf, walk->src.virt.addr, nbytes);
memcpy(buf, walk->src.virt.addr, walk->nbytes);
while (1) {
if (cpacf_kmctr(ctx->fc, param, buf,
buf, AES_BLOCK_SIZE,
@ -1010,12 +1039,13 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx,
memcpy(param->key, ctx->pk.protkey, sizeof(param->key));
spin_unlock_bh(&ctx->pk_lock);
}
memcpy(walk->dst.virt.addr, buf, nbytes);
memcpy(walk->dst.virt.addr, buf, walk->nbytes);
crypto_inc(walk->iv, AES_BLOCK_SIZE);
rc = skcipher_walk_done(walk, 0);
}
out:
memzero_explicit(buf, sizeof(buf));
pr_debug("rc=%d\n", rc);
return rc;
}
@ -1027,6 +1057,7 @@ static int ctr_paes_crypt(struct skcipher_request *req)
struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk *walk = &req_ctx->walk;
bool tested = crypto_skcipher_tested(tfm);
bool cleanup = true;
int rc;
/*
@ -1057,15 +1088,17 @@ static int ctr_paes_crypt(struct skcipher_request *req)
if (rc == 0 || rc == -EKEYEXPIRED) {
atomic_inc(&ctx->via_engine_ctr);
rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req);
if (rc != -EINPROGRESS)
if (rc == -EINPROGRESS || rc == -EBUSY)
cleanup = false;
else
atomic_dec(&ctx->via_engine_ctr);
}
if (rc != -EINPROGRESS)
if (cleanup && walk->nbytes)
skcipher_walk_done(walk, rc);
out:
if (rc != -EINPROGRESS)
if (cleanup)
memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
pr_debug("rc=%d\n", rc);
return rc;
@ -1105,7 +1138,7 @@ static int ctr_paes_do_one_request(struct crypto_engine *engine, void *areq)
rc = ctr_paes_do_crypt(ctx, req_ctx, tested, true);
if (rc == -EKEYEXPIRED) {
return pkey_handle_expired();
} else if (rc) {
} else if (rc && walk->nbytes) {
skcipher_walk_done(walk, rc);
}
@ -1115,7 +1148,7 @@ static int ctr_paes_do_one_request(struct crypto_engine *engine, void *areq)
atomic_dec(&ctx->via_engine_ctr);
crypto_finalize_skcipher_request(engine, req, rc);
local_bh_enable();
return rc;
return 0;
}
static struct skcipher_engine_alg ctr_paes_alg = {
@ -1123,6 +1156,7 @@ static struct skcipher_engine_alg ctr_paes_alg = {
.base.cra_name = "ctr(paes)",
.base.cra_driver_name = "ctr-paes-s390",
.base.cra_priority = 402, /* ecb-paes-s390 + 1 */
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct s390_paes_ctx),
.base.cra_module = THIS_MODULE,
@ -1283,8 +1317,11 @@ static int xts_paes_do_crypt_fullkey(struct s390_pxts_ctx *ctx,
n = nbytes & ~(AES_BLOCK_SIZE - 1);
k = cpacf_km(ctx->fc | req_ctx->modifier, param->key + offset,
walk->dst.virt.addr, walk->src.virt.addr, n);
if (k)
if (k) {
rc = skcipher_walk_done(walk, nbytes - k);
if (rc)
goto out;
}
if (k < n) {
if (!maysleep) {
rc = -EKEYEXPIRED;
@ -1337,7 +1374,7 @@ static inline int __xts_2keys_prep_param(struct s390_pxts_ctx *ctx,
memcpy(param->init, pcc_param.xts, 16);
}
memzero_explicit(pcc_param.key, sizeof(pcc_param.key));
memzero_explicit(&pcc_param, sizeof(pcc_param));
return rc;
}
@ -1377,8 +1414,11 @@ static int xts_paes_do_crypt_2keys(struct s390_pxts_ctx *ctx,
n = nbytes & ~(AES_BLOCK_SIZE - 1);
k = cpacf_km(ctx->fc | req_ctx->modifier, param->key + offset,
walk->dst.virt.addr, walk->src.virt.addr, n);
if (k)
if (k) {
rc = skcipher_walk_done(walk, nbytes - k);
if (rc)
goto out;
}
if (k < n) {
if (!maysleep) {
rc = -EKEYEXPIRED;
@ -1450,6 +1490,7 @@ static inline int xts_paes_crypt(struct skcipher_request *req, unsigned long mod
struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk *walk = &req_ctx->walk;
bool tested = crypto_skcipher_tested(tfm);
bool cleanup = true;
int rc;
/*
@ -1481,15 +1522,17 @@ static inline int xts_paes_crypt(struct skcipher_request *req, unsigned long mod
if (rc == 0 || rc == -EKEYEXPIRED) {
atomic_inc(&ctx->via_engine_ctr);
rc = crypto_transfer_skcipher_request_to_engine(paes_crypto_engine, req);
if (rc != -EINPROGRESS)
if (rc == -EINPROGRESS || rc == -EBUSY)
cleanup = false;
else
atomic_dec(&ctx->via_engine_ctr);
}
if (rc != -EINPROGRESS)
if (cleanup && walk->nbytes)
skcipher_walk_done(walk, rc);
out:
if (rc != -EINPROGRESS)
if (cleanup)
memzero_explicit(&req_ctx->param, sizeof(req_ctx->param));
pr_debug("rc=%d\n", rc);
return rc;
@ -1539,7 +1582,7 @@ static int xts_paes_do_one_request(struct crypto_engine *engine, void *areq)
rc = xts_paes_do_crypt(ctx, req_ctx, tested, true);
if (rc == -EKEYEXPIRED) {
return pkey_handle_expired();
} else if (rc) {
} else if (rc && walk->nbytes) {
skcipher_walk_done(walk, rc);
}
@ -1549,7 +1592,7 @@ static int xts_paes_do_one_request(struct crypto_engine *engine, void *areq)
atomic_dec(&ctx->via_engine_ctr);
crypto_finalize_skcipher_request(engine, req, rc);
local_bh_enable();
return rc;
return 0;
}
static struct skcipher_engine_alg xts_paes_alg = {
@ -1557,6 +1600,7 @@ static struct skcipher_engine_alg xts_paes_alg = {
.base.cra_name = "xts(paes)",
.base.cra_driver_name = "xts-paes-s390",
.base.cra_priority = 402, /* ecb-paes-s390 + 1 */
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct s390_pxts_ctx),
.base.cra_module = THIS_MODULE,

36
arch/s390/crypto/phmac_s390.c
View File

@ -62,8 +62,10 @@ static inline int hwh_prepare(struct ahash_request *req,
*/
static inline int hwh_advance(struct hash_walk_helper *hwh, int n)
{
if (n < 0)
if (n < 0) {
hwh->walkbytes = n;
return crypto_hash_walk_done(&hwh->walk, n);
}
hwh->walkbytes -= n;
hwh->walkaddr += n;
@ -339,6 +341,10 @@ static inline int convert_key(const u8 *key, unsigned int keylen,
xflags);
}
/* But finally map -EBUSY to -EIO to indicate an IO failure */
if (rc == -EBUSY)
rc = -EIO;
out:
pr_debug("rc=%d\n", rc);
return rc;
@ -606,6 +612,7 @@ static int phmac_update(struct ahash_request *req)
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
struct hash_walk_helper *hwh = &req_ctx->hwh;
bool cleanup = true;
int rc;
/* prep the walk in the request context */
@ -629,12 +636,15 @@ static int phmac_update(struct ahash_request *req)
req_ctx->async_op = OP_UPDATE;
atomic_inc(&tfm_ctx->via_engine_ctr);
rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
if (rc != -EINPROGRESS)
if (rc == -EINPROGRESS || rc == -EBUSY)
cleanup = false;
else
atomic_dec(&tfm_ctx->via_engine_ctr);
}
if (rc != -EINPROGRESS) {
hwh_advance(hwh, rc);
if (cleanup) {
if (hwh->walkbytes > 0)
hwh_advance(hwh, rc);
memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
}
@ -649,6 +659,7 @@ static int phmac_final(struct ahash_request *req)
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
bool cleanup = true;
int rc = 0;
/* Try synchronous operation if no active engine usage */
@ -667,12 +678,14 @@ static int phmac_final(struct ahash_request *req)
req_ctx->async_op = OP_FINAL;
atomic_inc(&tfm_ctx->via_engine_ctr);
rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
if (rc != -EINPROGRESS)
if (rc == -EINPROGRESS || rc == -EBUSY)
cleanup = false;
else
atomic_dec(&tfm_ctx->via_engine_ctr);
}
out:
if (rc != -EINPROGRESS)
if (cleanup)
memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
pr_debug("rc=%d\n", rc);
return rc;
@ -685,6 +698,7 @@ static int phmac_finup(struct ahash_request *req)
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
struct hash_walk_helper *hwh = &req_ctx->hwh;
bool cleanup = true;
int rc;
/* prep the walk in the request context */
@ -716,15 +730,17 @@ static int phmac_finup(struct ahash_request *req)
/* req->async_op has been set to either OP_FINUP or OP_FINAL */
atomic_inc(&tfm_ctx->via_engine_ctr);
rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
if (rc != -EINPROGRESS)
if (rc == -EINPROGRESS || rc == -EBUSY)
cleanup = false;
else
atomic_dec(&tfm_ctx->via_engine_ctr);
}
if (rc != -EINPROGRESS)
if (cleanup && hwh->walkbytes > 0)
hwh_advance(hwh, rc);
out:
if (rc != -EINPROGRESS)
if (cleanup)
memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
pr_debug("rc=%d\n", rc);
return rc;
@ -914,7 +930,7 @@ static int phmac_do_one_request(struct crypto_engine *engine, void *areq)
atomic_dec(&tfm_ctx->via_engine_ctr);
crypto_finalize_hash_request(engine, req, rc);
local_bh_enable();
return rc;
return 0;
}
#define S390_ASYNC_PHMAC_ALG(x) \

1
arch/s390/include/asm/pai.h
View File

@ -76,7 +76,6 @@ static __always_inline void pai_kernel_exit(struct pt_regs *regs)
}
#define PAI_SAVE_AREA(x) ((x)->hw.event_base)
#define PAI_CPU_MASK(x) ((x)->hw.addr_filters)
#define PAI_PMU_IDX(x) ((x)->hw.last_tag)
#define PAI_SWLIST(x) (&(x)->hw.tp_list)

54
arch/s390/kernel/debug.c
View File

@ -182,7 +182,7 @@ static struct debug_param_t {
static int debug_param_num;
/* functions */
static void debug_get_param(const char *name, int *level, int *pages)
static void debug_get_param(const char *name, int *level, int *pages, bool quiet)
{
struct debug_param_t *p;
int i;
@ -192,11 +192,13 @@ static void debug_get_param(const char *name, int *level, int *pages)
if (!glob_match(p->name, name))
continue;
if (level && p->level != PARAM_UNSET) {
pr_info("%s: override level to %d\n", name, p->level);
if (!quiet)
pr_info("%s: override level to %d\n", name, p->level);
*level = p->level;
}
if (pages && p->pages != PARAM_UNSET) {
pr_info("%s: override pages to %d\n", name, p->pages);
if (!quiet)
pr_info("%s: override pages to %d\n", name, p->pages);
*pages = p->pages;
}
}
@ -251,7 +253,7 @@ static int __init s390dbf_parse(char *arg)
* regular memory allocations are possible.
*/
for (i = 0, id = __s390dbf_info; &id[i] < __s390dbf_info_end; i++)
debug_get_param(id[i]->name, &id[i]->level, NULL);
debug_get_param(id[i]->name, &id[i]->level, NULL, false);
return rc;
}
@ -395,7 +397,7 @@ static debug_info_t *debug_info_create(const char *name, int pages_per_area,
int level = DEBUG_DEFAULT_LEVEL;
debug_info_t *rc;
debug_get_param(name, &level, &pages_per_area);
debug_get_param(name, &level, &pages_per_area, false);
rc = debug_info_alloc(name, pages_per_area, nr_areas, buf_size, level, ALL_AREAS);
if (!rc)
goto out;
@ -960,7 +962,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas)
return;
}
debug_get_param(id->name, &id->level, &pages_per_area);
debug_get_param(id->name, &id->level, &pages_per_area, false);
copy = debug_info_alloc("", pages_per_area, nr_areas, id->buf_size,
id->level, ALL_AREAS);
if (!copy) {
@ -1074,9 +1076,6 @@ static void _debug_set_level(debug_info_t *id, int new_level)
{
unsigned long flags;
if (!id)
return;
if (new_level == DEBUG_OFF_LEVEL) {
pr_info("%s: switched off\n", id->name);
} else if ((new_level > DEBUG_MAX_LEVEL) || (new_level < 0)) {
@ -1101,8 +1100,14 @@ static void _debug_set_level(debug_info_t *id, int new_level)
*/
void debug_set_level(debug_info_t *id, int new_level)
{
/* Level specified via kernel parameter takes precedence */
debug_get_param(id->name, &new_level, NULL);
if (!id)
return;
/*
* Level specified via kernel parameter takes precedence. The override
* was already announced during registration, so stay quiet here.
*/
debug_get_param(id->name, &new_level, NULL, true);
_debug_set_level(id, new_level);
}
@ -1278,7 +1283,7 @@ void debug_set_critical(void)
debug_entry_t *debug_event_common(debug_info_t *id, int level, const void *buf,
int len)
{
debug_entry_t *active;
debug_entry_t *active = NULL;
unsigned long flags;
if (!debug_active || !id->areas)
@ -1289,6 +1294,8 @@ debug_entry_t *debug_event_common(debug_info_t *id, int level, const void *buf,
} else {
raw_spin_lock_irqsave(&id->lock, flags);
}
if (!id->areas)
goto out;
do {
active = get_active_entry(id);
memcpy(DEBUG_DATA(active), buf, min(len, id->buf_size));
@ -1298,7 +1305,7 @@ debug_entry_t *debug_event_common(debug_info_t *id, int level, const void *buf,
len -= id->buf_size;
buf += id->buf_size;
} while (len > 0);
out:
raw_spin_unlock_irqrestore(&id->lock, flags);
return active;
}
@ -1311,7 +1318,7 @@ EXPORT_SYMBOL(debug_event_common);
debug_entry_t *debug_exception_common(debug_info_t *id, int level,
const void *buf, int len)
{
debug_entry_t *active;
debug_entry_t *active = NULL;
unsigned long flags;
if (!debug_active || !id->areas)
@ -1322,6 +1329,8 @@ debug_entry_t *debug_exception_common(debug_info_t *id, int level,
} else {
raw_spin_lock_irqsave(&id->lock, flags);
}
if (!id->areas)
goto out;
do {
active = get_active_entry(id);
memcpy(DEBUG_DATA(active), buf, min(len, id->buf_size));
@ -1331,7 +1340,7 @@ debug_entry_t *debug_exception_common(debug_info_t *id, int level,
len -= id->buf_size;
buf += id->buf_size;
} while (len > 0);
out:
raw_spin_unlock_irqrestore(&id->lock, flags);
return active;
}
@ -1357,7 +1366,7 @@ static inline int debug_count_numargs(char *string)
debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string, ...)
{
debug_sprintf_entry_t *curr_event;
debug_entry_t *active;
debug_entry_t *active = NULL;
unsigned long flags;
int numargs, idx;
va_list ap;
@ -1372,6 +1381,8 @@ debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string,
} else {
raw_spin_lock_irqsave(&id->lock, flags);
}
if (!id->areas)
goto out;
active = get_active_entry(id);
curr_event = (debug_sprintf_entry_t *) DEBUG_DATA(active);
va_start(ap, string);
@ -1380,6 +1391,7 @@ debug_entry_t *__debug_sprintf_event(debug_info_t *id, int level, char *string,
curr_event->args[idx] = va_arg(ap, long);
va_end(ap);
debug_finish_entry(id, active, level, 0);
out:
raw_spin_unlock_irqrestore(&id->lock, flags);
return active;
@ -1392,7 +1404,7 @@ EXPORT_SYMBOL(__debug_sprintf_event);
debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *string, ...)
{
debug_sprintf_entry_t *curr_event;
debug_entry_t *active;
debug_entry_t *active = NULL;
unsigned long flags;
int numargs, idx;
va_list ap;
@ -1408,6 +1420,8 @@ debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *stri
} else {
raw_spin_lock_irqsave(&id->lock, flags);
}
if (!id->areas)
goto out;
active = get_active_entry(id);
curr_event = (debug_sprintf_entry_t *)DEBUG_DATA(active);
va_start(ap, string);
@ -1416,6 +1430,7 @@ debug_entry_t *__debug_sprintf_exception(debug_info_t *id, int level, char *stri
curr_event->args[idx] = va_arg(ap, long);
va_end(ap);
debug_finish_entry(id, active, level, 1);
out:
raw_spin_unlock_irqrestore(&id->lock, flags);
return active;
@ -1658,9 +1673,11 @@ static void debug_flush(debug_info_t *id, int area)
unsigned long flags;
int i, j;
if (!id || !id->areas)
if (!id)
return;
raw_spin_lock_irqsave(&id->lock, flags);
if (!id->areas)
goto out;
if (area == DEBUG_FLUSH_ALL) {
id->active_area = 0;
memset(id->active_entries, 0, id->nr_areas * sizeof(int));
@ -1675,6 +1692,7 @@ static void debug_flush(debug_info_t *id, int area)
for (i = 0; i < id->pages_per_area; i++)
memset(id->areas[area][i], 0, PAGE_SIZE);
}
out:
raw_spin_unlock_irqrestore(&id->lock, flags);
}

196
arch/s390/kernel/perf_pai.c
View File

@ -67,6 +67,7 @@ struct pai_mapptr {
static struct pai_root { /* Anchor to per CPU data */
refcount_t refcnt; /* Overall active events */
atomic_t tskctx; /* Overall per-task events */
struct pai_mapptr __percpu *mapptr;
} pai_root[PAI_PMU_MAX];
@ -93,14 +94,15 @@ struct pai_pmu { /* Define PAI PMU characteristics */
static struct pai_pmu pai_pmu[]; /* Forward declaration */
/* Free per CPU data when the last event is removed. */
static void pai_root_free(int idx)
static void pai_root_free(int idx, int tasks)
{
if (refcount_dec_and_test(&pai_root[idx].refcnt)) {
if (refcount_sub_and_test(tasks, &pai_root[idx].refcnt)) {
free_percpu(pai_root[idx].mapptr);
pai_root[idx].mapptr = NULL;
}
debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d\n", __func__,
idx, refcount_read(&pai_root[idx].refcnt));
debug_sprintf_event(paidbg, 5, "%s root[%d].refcount %d tskctx %d\n",
__func__, idx, refcount_read(&pai_root[idx].refcnt),
atomic_read(&pai_root[idx].tskctx));
}
/*
@ -137,40 +139,54 @@ static void pai_free(struct pai_mapptr *mp)
mp->mapptr = NULL;
}
/* Adjust usage counters and remove allocated memory when all users are
* gone.
*/
static void pai_event_destroy_cpu(struct perf_event *event, int cpu)
/* Called under mutex_lock */
static void pai_event_destroy_cpu(int idx, int cpu, bool hotplug)
{
int idx = PAI_PMU_IDX(event);
struct pai_mapptr *mp = per_cpu_ptr(pai_root[idx].mapptr, cpu);
struct pai_map *cpump = mp->mapptr;
struct pai_mapptr *mp;
struct pai_map *cpump;
int tasks = 1;
mutex_lock(&pai_reserve_mutex);
debug_sprintf_event(paidbg, 5, "%s event %#llx idx %d cpu %d users %d "
"refcnt %u\n", __func__, event->attr.config, idx,
event->cpu, cpump->active_events,
refcount_read(&cpump->refcnt));
if (refcount_dec_and_test(&cpump->refcnt))
/* Check reference count and return when all gone.
* 1. An event is installed on online CPU X.
* 2. CPU x is offlined and the per-CPU data is removed.
* 3. Event is destroyed via close system call.
*/
if (!refcount_read(&pai_root[idx].refcnt))
return; /* No events at all */
mp = per_cpu_ptr(pai_root[idx].mapptr, cpu);
if (!mp || !mp->mapptr) /* No events on that CPU */
return;
/* When hotplug is true, invocation is from CPU hotplug callback.
* Delete per-CPU resource and adjust refcnt when per-task events
* are currently active. This can be more than one.
* In this case adjust counters.
*/
if (hotplug)
tasks = atomic_read(&pai_root[idx].tskctx);
cpump = mp->mapptr;
if (refcount_sub_and_test(tasks, &cpump->refcnt))
pai_free(mp);
pai_root_free(idx);
mutex_unlock(&pai_reserve_mutex);
pai_root_free(idx, tasks);
}
static void pai_event_destroy(struct perf_event *event)
{
int cpu;
int cpu = 0, idx = PAI_PMU_IDX(event);
free_page(PAI_SAVE_AREA(event));
cpus_read_lock();
mutex_lock(&pai_reserve_mutex);
if (event->cpu == -1) {
struct cpumask *mask = PAI_CPU_MASK(event);
for_each_cpu(cpu, mask)
pai_event_destroy_cpu(event, cpu);
kfree(mask);
atomic_dec(&pai_root[idx].tskctx);
for_each_online_cpu(cpu)
pai_event_destroy_cpu(idx, cpu, false);
} else {
pai_event_destroy_cpu(event, event->cpu);
pai_event_destroy_cpu(idx, event->cpu, false);
}
mutex_unlock(&pai_reserve_mutex);
cpus_read_unlock();
}
static void paicrypt_event_destroy(struct perf_event *event)
@ -234,25 +250,30 @@ static u64 paicrypt_getall(struct perf_event *event)
return sum;
}
/* Check concurrent access of counting and sampling for crypto events.
* This function is called in process context and it is save to block.
* When the event initialization functions fails, no other call back will
* be invoked.
*
* Allocate the memory for the event.
*/
static int pai_alloc_cpu(struct perf_event *event, int cpu)
/* Called under mutex_lock */
static int pai_alloc_cpu(int idx, int cpu, bool hotplug)
{
int rc, idx = PAI_PMU_IDX(event);
struct pai_map *cpump = NULL;
bool need_paiext_cb = false;
struct pai_mapptr *mp;
int tasks = 1, rc = 0;
/* When hotplug is true, invocation is from CPU hotplug callback.
* Allocate per-CPU resource when per-task events are currently active.
* This can be more than one. In this case adjust all reference
* counters. Otherwise return, this ensures memory is only allocated
* when needed.
*/
if (hotplug) {
tasks = atomic_read(&pai_root[idx].tskctx);
if (!tasks)
goto out;
}
mutex_lock(&pai_reserve_mutex);
/* Allocate root node */
rc = pai_root_alloc(idx);
if (rc)
goto unlock;
goto out;
/* Allocate node for this event */
mp = per_cpu_ptr(pai_root[idx].mapptr, cpu);
@ -296,28 +317,45 @@ static int pai_alloc_cpu(struct perf_event *event, int cpu)
goto undo;
}
INIT_LIST_HEAD(&cpump->syswide_list);
refcount_set(&cpump->refcnt, 1);
refcount_set(&cpump->refcnt, tasks);
rc = 0;
} else {
refcount_inc(&cpump->refcnt);
refcount_add(tasks, &cpump->refcnt);
}
/* If tasks is greater than 1, we are called from CPU hotplug path
* and need to adjust the pai_root[idx].refcnt by the number of
* per-process events. Function pai_root_alloc(idx) already
* incremented by one. Adjust for the rest.
*/
if (tasks > 1)
refcount_add(tasks - 1, &pai_root[idx].refcnt);
undo:
if (rc) {
/* Error in allocation of event, decrement anchor. Since
* the event in not created, its destroy() function is never
* invoked. Adjust the reference counter for the anchor.
* The failure happened in the case of variable
* cpump == NULL branch above. The pai_root[XXX].refcnt has
* been incremented by one. Then the per-CPU allocation
* failed, so decrement it by one, regardless of tasks.
*/
pai_root_free(idx);
pai_root_free(idx, 1);
}
unlock:
mutex_unlock(&pai_reserve_mutex);
out:
/* If rc is non-zero, no increment of counter/sampler was done. */
return rc;
}
/* Check concurrent access of counting and sampling for PAI events.
* This function is called in process context and it is safe to block.
* When the event initialization functions fails, no other call back will
* be invoked.
* Called under mutex_lock.
*/
static int pai_alloc(struct perf_event *event)
{
int idx = PAI_PMU_IDX(event);
struct cpumask *maskptr;
int cpu, rc = -ENOMEM;
@ -326,24 +364,20 @@ static int pai_alloc(struct perf_event *event)
goto out;
for_each_online_cpu(cpu) {
rc = pai_alloc_cpu(event, cpu);
rc = pai_alloc_cpu(idx, cpu, false);
if (rc) {
for_each_cpu(cpu, maskptr)
pai_event_destroy_cpu(event, cpu);
kfree(maskptr);
goto out;
pai_event_destroy_cpu(idx, cpu, false);
goto undo;
}
cpumask_set_cpu(cpu, maskptr);
}
/*
* On error all cpumask are freed and all events have been destroyed.
* Save of which CPUs data structures have been allocated for.
* Release them in pai_event_destroy call back function
* for this event.
*/
PAI_CPU_MASK(event) = maskptr;
rc = 0;
/* Trace per-task events for CPU hotplug. */
atomic_inc(&pai_root[idx].tskctx);
undo:
kfree(maskptr);
out:
return rc;
}
@ -391,10 +425,14 @@ static int pai_event_init(struct perf_event *event, int idx)
}
}
cpus_read_lock();
mutex_lock(&pai_reserve_mutex);
if (event->cpu >= 0)
rc = pai_alloc_cpu(event, event->cpu);
rc = pai_alloc_cpu(idx, event->cpu, false);
else
rc = pai_alloc(event);
mutex_unlock(&pai_reserve_mutex);
cpus_read_unlock();
if (rc) {
free_page(PAI_SAVE_AREA(event));
goto out;
@ -1239,8 +1277,35 @@ static int __init paipmu_setup(void)
return install_ok;
}
static int pai_online_cpu(unsigned int cpu)
{
int rc;
mutex_lock(&pai_reserve_mutex);
rc = pai_alloc_cpu(PAI_PMU_CRYPTO, cpu, true);
if (rc)
goto out;
rc = pai_alloc_cpu(PAI_PMU_EXT, cpu, true);
if (rc)
pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true);
out:
mutex_unlock(&pai_reserve_mutex);
return rc;
}
static int pai_offline_cpu(unsigned int cpu)
{
mutex_lock(&pai_reserve_mutex);
pai_event_destroy_cpu(PAI_PMU_CRYPTO, cpu, true);
pai_event_destroy_cpu(PAI_PMU_EXT, cpu, true);
mutex_unlock(&pai_reserve_mutex);
return 0;
}
static int __init pai_init(void)
{
int state, rc;
/* Setup s390dbf facility */
paidbg = debug_register("pai", 1, 1, 128);
if (!paidbg) {
@ -1249,13 +1314,24 @@ static int __init pai_init(void)
}
debug_register_view(paidbg, &debug_sprintf_view);
if (!paipmu_setup()) {
/* No PMU registration, no need for debug buffer */
debug_unregister_view(paidbg, &debug_sprintf_view);
debug_unregister(paidbg);
return -ENODEV;
}
/* CPUHP_BP_PREPARE_DYN --> before CPU is brought online */
state = cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "perf/pai:prepare",
pai_online_cpu, pai_offline_cpu);
rc = state < 0 ? state : 0;
if (rc < 0)
goto out_debug;
rc = -ENODEV;
if (!paipmu_setup())
goto out_cpuhp;
return 0;
out_cpuhp:
cpuhp_remove_state(state);
out_debug:
debug_unregister_view(paidbg, &debug_sprintf_view);
debug_unregister(paidbg);
return rc;
}
device_initcall(pai_init);