diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 10561aa687c7..0be6fa779d2c 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -26,14 +26,14 @@ #include #include #include -#include #include +#include #include #include #include static u8 *ctrblk; -static DEFINE_MUTEX(ctrblk_lock); +static DEFINE_SEMAPHORE(ctrblk_sem, 1); static cpacf_mask_t km_functions, kmc_functions, kmctr_functions, kma_functions; @@ -562,46 +562,61 @@ static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) return n; } +static int __ctr_aes_crypt(struct s390_aes_ctx *sctx, + struct skcipher_walk *walk, bool locked) +{ + unsigned int n, nbytes; + int ret = 0; + u8 *ctrptr; + + while (!ret && ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE)) { + n = AES_BLOCK_SIZE; + if (nbytes >= 2 * AES_BLOCK_SIZE && locked) + n = __ctrblk_init(ctrblk, walk->iv, nbytes); + ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; + cpacf_kmctr(sctx->fc, sctx->key, walk->dst.virt.addr, + walk->src.virt.addr, n, ctrptr); + if (ctrptr == ctrblk) + memcpy(walk->iv, ctrptr + n - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + crypto_inc(walk->iv, AES_BLOCK_SIZE); + ret = skcipher_walk_done(walk, nbytes - n); + } + + return ret; +} + static int ctr_aes_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm); - u8 buf[AES_BLOCK_SIZE], *ctrptr; struct skcipher_walk walk; - unsigned int n, nbytes; - int ret, locked; + u8 buf[AES_BLOCK_SIZE]; + int ret; if (unlikely(!sctx->fc)) return fallback_skcipher_crypt(sctx, req, 0); - locked = mutex_trylock(&ctrblk_lock); - ret = skcipher_walk_virt(&walk, req, false); - while (!ret && ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE)) { - n = AES_BLOCK_SIZE; + if (ret) + return ret; - if (nbytes >= 2*AES_BLOCK_SIZE && locked) - n = __ctrblk_init(ctrblk, walk.iv, nbytes); - ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv; - cpacf_kmctr(sctx->fc, sctx->key, walk.dst.virt.addr, - walk.src.virt.addr, n, ctrptr); - if (ctrptr == ctrblk) - memcpy(walk.iv, ctrptr + n - AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - crypto_inc(walk.iv, AES_BLOCK_SIZE); - ret = skcipher_walk_done(&walk, nbytes - n); + if (down_trylock(&ctrblk_sem) == 0) { + ret = __ctr_aes_crypt(sctx, &walk, true); + up(&ctrblk_sem); + } else { + ret = __ctr_aes_crypt(sctx, &walk, false); } - if (locked) - mutex_unlock(&ctrblk_lock); + /* * final block may be < AES_BLOCK_SIZE, copy only nbytes */ - if (!ret && nbytes) { + if (!ret && walk.nbytes > 0) { memset(buf, 0, AES_BLOCK_SIZE); - memcpy(buf, walk.src.virt.addr, nbytes); + memcpy(buf, walk.src.virt.addr, walk.nbytes); cpacf_kmctr(sctx->fc, sctx->key, buf, buf, AES_BLOCK_SIZE, walk.iv); - memcpy(walk.dst.virt.addr, buf, nbytes); + memcpy(walk.dst.virt.addr, buf, walk.nbytes); crypto_inc(walk.iv, AES_BLOCK_SIZE); ret = skcipher_walk_done(&walk, 0); memzero_explicit(buf, sizeof(buf));