diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 47dccdd49637..cbcf76953d83 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -100,16 +100,11 @@ config CRYPTO_CHACHA20_P10 default CRYPTO_LIB_CHACHA_INTERNAL config CRYPTO_POLY1305_P10 - tristate "Hash functions: Poly1305 (P10 or later)" + tristate depends on PPC64 && CPU_LITTLE_ENDIAN && VSX - select CRYPTO_HASH + select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_LIB_POLY1305_GENERIC - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: PowerPC64 - - Power10 or later - - Little-endian + default CRYPTO_LIB_POLY1305_INTERNAL config CRYPTO_DEV_VMX bool "Support for VMX cryptographic acceleration instructions" diff --git a/arch/powerpc/crypto/poly1305-p10-glue.c b/arch/powerpc/crypto/poly1305-p10-glue.c index 369686e9370b..87bd5eba4497 100644 --- a/arch/powerpc/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/crypto/poly1305-p10-glue.c @@ -4,15 +4,11 @@ * * Copyright 2023- IBM Corp. All rights reserved. */ - -#include -#include #include #include #include -#include -#include #include +#include #include #include #include @@ -22,6 +18,8 @@ asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen); asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit); asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); + static void vsx_begin(void) { preempt_disable(); @@ -34,51 +32,29 @@ static void vsx_end(void) preempt_enable(); } -static int crypto_poly1305_p10_init(struct shash_desc *desc) +void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE]) { - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + if (!static_key_enabled(&have_p10)) + return poly1305_init_generic(dctx, key); - poly1305_core_init(&dctx->h); + dctx->h = (struct poly1305_state){}; + dctx->core_r.key.r64[0] = get_unaligned_le64(key + 0); + dctx->core_r.key.r64[1] = get_unaligned_le64(key + 8); + dctx->s[0] = get_unaligned_le32(key + 16); + dctx->s[1] = get_unaligned_le32(key + 20); + dctx->s[2] = get_unaligned_le32(key + 24); + dctx->s[3] = get_unaligned_le32(key + 28); dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; } +EXPORT_SYMBOL(poly1305_init_arch); -static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, - const u8 *inp, unsigned int len) +void poly1305_update_arch(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen) { - unsigned int acc = 0; + unsigned int bytes; - if (unlikely(!dctx->sset)) { - if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) { - struct poly1305_core_key *key = &dctx->core_r; - - key->key.r64[0] = get_unaligned_le64(&inp[0]); - key->key.r64[1] = get_unaligned_le64(&inp[8]); - inp += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - acc += POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(&inp[0]); - dctx->s[1] = get_unaligned_le32(&inp[4]); - dctx->s[2] = get_unaligned_le32(&inp[8]); - dctx->s[3] = get_unaligned_le32(&inp[12]); - acc += POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return acc; -} - -static int crypto_poly1305_p10_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int bytes, used; + if (!static_key_enabled(&have_p10)) + return poly1305_update_generic(dctx, src, srclen); if (unlikely(dctx->buflen)) { bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); @@ -86,26 +62,16 @@ static int crypto_poly1305_p10_update(struct shash_desc *desc, src += bytes; srclen -= bytes; dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, - POLY1305_BLOCK_SIZE))) { - vsx_begin(); - poly1305_64s(&dctx->h, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - vsx_end(); - } - dctx->buflen = 0; - } + if (dctx->buflen < POLY1305_BLOCK_SIZE) + return; + vsx_begin(); + poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); + vsx_end(); + dctx->buflen = 0; } if (likely(srclen >= POLY1305_BLOCK_SIZE)) { bytes = round_down(srclen, POLY1305_BLOCK_SIZE); - used = crypto_poly1305_setdctxkey(dctx, src, bytes); - if (likely(used)) { - srclen -= used; - src += used; - } if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) { vsx_begin(); poly1305_p10le_4blocks(&dctx->h, src, srclen); @@ -126,61 +92,35 @@ static int crypto_poly1305_p10_update(struct shash_desc *desc, dctx->buflen = srclen; memcpy(dctx->buf, src, srclen); } - - return 0; } +EXPORT_SYMBOL(poly1305_update_arch); -static int crypto_poly1305_p10_final(struct shash_desc *desc, u8 *dst) +void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) { - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + if (!static_key_enabled(&have_p10)) + return poly1305_final_generic(dctx, dst); - if (unlikely(!dctx->sset)) - return -ENOKEY; - - if ((dctx->buflen)) { + if (dctx->buflen) { dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, POLY1305_BLOCK_SIZE - dctx->buflen); vsx_begin(); poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); vsx_end(); - dctx->buflen = 0; } poly1305_emit_64(&dctx->h, &dctx->s, dst); - return 0; } - -static struct shash_alg poly1305_alg = { - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_poly1305_p10_init, - .update = crypto_poly1305_p10_update, - .final = crypto_poly1305_p10_final, - .descsize = sizeof(struct poly1305_desc_ctx), - .base = { - .cra_name = "poly1305", - .cra_driver_name = "poly1305-p10", - .cra_priority = 300, - .cra_blocksize = POLY1305_BLOCK_SIZE, - .cra_module = THIS_MODULE, - }, -}; +EXPORT_SYMBOL(poly1305_final_arch); static int __init poly1305_p10_init(void) { - return crypto_register_shash(&poly1305_alg); + if (cpu_has_feature(CPU_FTR_ARCH_31)) + static_branch_enable(&have_p10); + return 0; } - -static void __exit poly1305_p10_exit(void) -{ - crypto_unregister_shash(&poly1305_alg); -} - -module_cpu_feature_match(PPC_MODULE_FEATURE_P10, poly1305_p10_init); -module_exit(poly1305_p10_exit); +arch_initcall(poly1305_p10_init); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Danny Tsen "); MODULE_DESCRIPTION("Optimized Poly1305 for P10"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-p10");