diff --git a/MAINTAINERS b/MAINTAINERS index 77fdfcb55f06..f088f4085653 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12267,10 +12267,10 @@ F: arch/powerpc/crypto/aes_cbc.c F: arch/powerpc/crypto/aes_ctr.c F: arch/powerpc/crypto/aes_xts.c F: arch/powerpc/crypto/aesp8-ppc.* -F: arch/powerpc/crypto/ghash.c -F: arch/powerpc/crypto/ghashp8-ppc.pl F: arch/powerpc/crypto/ppc-xlate.pl F: arch/powerpc/crypto/vmx.c +F: lib/crypto/powerpc/gf128hash.h +F: lib/crypto/powerpc/ghashp8-ppc.pl IBM ServeRAID RAID DRIVER S: Orphan diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 2d056f1fc90f..b247f7ed973e 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -54,12 +54,11 @@ config CRYPTO_DEV_VMX_ENCRYPT select CRYPTO_AES select CRYPTO_CBC select CRYPTO_CTR - select CRYPTO_GHASH select CRYPTO_XTS default m help Support for VMX cryptographic acceleration instructions on Power8 CPU. - This module supports acceleration for AES and GHASH in hardware. If you - choose 'M' here, this module will be called vmx-crypto. + This module supports acceleration for AES in hardware. If you choose + 'M' here, this module will be called vmx-crypto. endmenu diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile index 3ac0886282a2..a1fe102a90ae 100644 --- a/arch/powerpc/crypto/Makefile +++ b/arch/powerpc/crypto/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o aes-ppc-spe-y := aes-spe-glue.o aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o -vmx-crypto-objs := vmx.o ghashp8-ppc.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o +vmx-crypto-objs := vmx.o aes_cbc.o aes_ctr.o aes_xts.o ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y) override flavour := linux-ppc64le @@ -26,14 +26,10 @@ endif quiet_cmd_perl = PERL $@ cmd_perl = $(PERL) $< $(flavour) > $@ -targets += aesp10-ppc.S ghashp10-ppc.S ghashp8-ppc.S +targets += aesp10-ppc.S ghashp10-ppc.S $(obj)/aesp10-ppc.S $(obj)/ghashp10-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE $(call if_changed,perl) -$(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE - $(call if_changed,perl) - OBJECT_FILES_NON_STANDARD_aesp10-ppc.o := y OBJECT_FILES_NON_STANDARD_ghashp10-ppc.o := y -OBJECT_FILES_NON_STANDARD_ghashp8-ppc.o := y diff --git a/arch/powerpc/crypto/aesp8-ppc.h b/arch/powerpc/crypto/aesp8-ppc.h index 6862c605cc33..c68f5b6965fa 100644 --- a/arch/powerpc/crypto/aesp8-ppc.h +++ b/arch/powerpc/crypto/aesp8-ppc.h @@ -2,7 +2,6 @@ #include #include -extern struct shash_alg p8_ghash_alg; extern struct skcipher_alg p8_aes_cbc_alg; extern struct skcipher_alg p8_aes_ctr_alg; extern struct skcipher_alg p8_aes_xts_alg; diff --git a/arch/powerpc/crypto/ghash.c b/arch/powerpc/crypto/ghash.c deleted file mode 100644 index 7308735bdb33..000000000000 --- a/arch/powerpc/crypto/ghash.c +++ /dev/null @@ -1,160 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * GHASH routines supporting VMX instructions on the Power 8 - * - * Copyright (C) 2015, 2019 International Business Machines Inc. - * - * Author: Marcelo Henrique Cerri - * - * Extended by Daniel Axtens to replace the fallback - * mechanism. The new approach is based on arm64 code, which is: - * Copyright (C) 2014 - 2018 Linaro Ltd. - */ - -#include "aesp8-ppc.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -void gcm_init_p8(u128 htable[16], const u64 Xi[2]); -void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); -void gcm_ghash_p8(u64 Xi[2], const u128 htable[16], - const u8 *in, size_t len); - -struct p8_ghash_ctx { - /* key used by vector asm */ - u128 htable[16]; - /* key used by software fallback */ - be128 key; -}; - -struct p8_ghash_desc_ctx { - u64 shash[2]; -}; - -static int p8_ghash_init(struct shash_desc *desc) -{ - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memset(dctx->shash, 0, GHASH_DIGEST_SIZE); - return 0; -} - -static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) -{ - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm)); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - gcm_init_p8(ctx->htable, (const u64 *) key); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - - memcpy(&ctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static inline void __ghash_block(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx, - const u8 *src) -{ - if (crypto_simd_usable()) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - gcm_ghash_p8(dctx->shash, ctx->htable, src, GHASH_BLOCK_SIZE); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - } else { - crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); - gf128mul_lle((be128 *)dctx->shash, &ctx->key); - } -} - -static inline int __ghash_blocks(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - int remain = srclen - round_down(srclen, GHASH_BLOCK_SIZE); - - srclen -= remain; - if (crypto_simd_usable()) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - gcm_ghash_p8(dctx->shash, ctx->htable, - src, srclen); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - } else { - do { - crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); - gf128mul_lle((be128 *)dctx->shash, &ctx->key); - srclen -= GHASH_BLOCK_SIZE; - src += GHASH_BLOCK_SIZE; - } while (srclen); - } - - return remain; -} - -static int p8_ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - return __ghash_blocks(ctx, dctx, src, srclen); -} - -static int p8_ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - __ghash_block(ctx, dctx, buf); - memzero_explicit(buf, sizeof(buf)); - } - memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); - return 0; -} - -struct shash_alg p8_ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = p8_ghash_init, - .update = p8_ghash_update, - .finup = p8_ghash_finup, - .setkey = p8_ghash_setkey, - .descsize = sizeof(struct p8_ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "p8_ghash", - .cra_priority = 1000, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct p8_ghash_ctx), - .cra_module = THIS_MODULE, - }, -}; diff --git a/arch/powerpc/crypto/vmx.c b/arch/powerpc/crypto/vmx.c index 7d2beb774f99..08da5311dfdf 100644 --- a/arch/powerpc/crypto/vmx.c +++ b/arch/powerpc/crypto/vmx.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include "aesp8-ppc.h" @@ -23,13 +22,9 @@ static int __init p8_init(void) { int ret; - ret = crypto_register_shash(&p8_ghash_alg); - if (ret) - goto err; - ret = crypto_register_skcipher(&p8_aes_cbc_alg); if (ret) - goto err_unregister_ghash; + goto err; ret = crypto_register_skcipher(&p8_aes_ctr_alg); if (ret) @@ -45,8 +40,6 @@ static int __init p8_init(void) crypto_unregister_skcipher(&p8_aes_ctr_alg); err_unregister_aes_cbc: crypto_unregister_skcipher(&p8_aes_cbc_alg); -err_unregister_ghash: - crypto_unregister_shash(&p8_ghash_alg); err: return ret; } @@ -56,7 +49,6 @@ static void __exit p8_exit(void) crypto_unregister_skcipher(&p8_aes_xts_alg); crypto_unregister_skcipher(&p8_aes_ctr_alg); crypto_unregister_skcipher(&p8_aes_cbc_alg); - crypto_unregister_shash(&p8_ghash_alg); } module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, p8_init); diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h index 5090fbaa87f8..650652dd6003 100644 --- a/include/crypto/gf128hash.h +++ b/include/crypto/gf128hash.h @@ -41,6 +41,10 @@ struct polyval_elem { * Use ghash_preparekey() to initialize this. */ struct ghash_key { +#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64) + /** @htable: GHASH key format used by the POWER8 assembly code */ + u64 htable[4][2]; +#endif /** @h: The hash key H, in POLYVAL format */ struct polyval_elem h; }; diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 4f1a79883a56..f54add7d9070 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -121,6 +121,7 @@ config CRYPTO_LIB_GF128HASH_ARCH depends on CRYPTO_LIB_GF128HASH && !UML default y if ARM && KERNEL_MODE_NEON default y if ARM64 + default y if PPC64 && VSX default y if X86_64 config CRYPTO_LIB_MD5 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 4ce0bac8fd93..8a9084188778 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -8,6 +8,10 @@ quiet_cmd_perlasm = PERLASM $@ quiet_cmd_perlasm_with_args = PERLASM $@ cmd_perlasm_with_args = $(PERL) $(<) void $(@) +ppc64-perlasm-flavour-y := linux-ppc64 +ppc64-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2 +ppc64-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le + obj-$(CONFIG_KUNIT) += tests/ obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o @@ -36,11 +40,8 @@ libaes-y += powerpc/aes-spe-core.o \ powerpc/aes-tab-4k.o else libaes-y += powerpc/aesp8-ppc.o -aes-perlasm-flavour-y := linux-ppc64 -aes-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2 -aes-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le quiet_cmd_perlasm_aes = PERLASM $@ - cmd_perlasm_aes = $(PERL) $< $(aes-perlasm-flavour-y) $@ + cmd_perlasm_aes = $(PERL) $< $(ppc64-perlasm-flavour-y) $@ # Use if_changed instead of cmd, in case the flavour changed. $(obj)/powerpc/aesp8-ppc.S: $(src)/powerpc/aesp8-ppc.pl FORCE $(call if_changed,perlasm_aes) @@ -161,9 +162,23 @@ CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH) libgf128hash-$(CONFIG_ARM) += arm/ghash-neon-core.o libgf128hash-$(CONFIG_ARM64) += arm64/ghash-neon-core.o \ arm64/polyval-ce-core.o -libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o + +ifeq ($(CONFIG_PPC),y) +libgf128hash-y += powerpc/ghashp8-ppc.o +quiet_cmd_perlasm_ghash = PERLASM $@ + cmd_perlasm_ghash = $(PERL) $< $(ppc64-perlasm-flavour-y) $@ +$(obj)/powerpc/ghashp8-ppc.S: $(src)/powerpc/ghashp8-ppc.pl FORCE + $(call if_changed,perlasm_ghash) +targets += powerpc/ghashp8-ppc.S +OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y endif +libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o +endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH + +# clean-files must be defined unconditionally +clean-files += powerpc/ghashp8-ppc.S + ################################################################################ obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o diff --git a/lib/crypto/powerpc/.gitignore b/lib/crypto/powerpc/.gitignore index 598ca7aff6b1..7aa71d83f739 100644 --- a/lib/crypto/powerpc/.gitignore +++ b/lib/crypto/powerpc/.gitignore @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only aesp8-ppc.S +ghashp8-ppc.S diff --git a/lib/crypto/powerpc/gf128hash.h b/lib/crypto/powerpc/gf128hash.h new file mode 100644 index 000000000000..629cd325d0c7 --- /dev/null +++ b/lib/crypto/powerpc/gf128hash.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * GHASH routines supporting VMX instructions on the Power 8 + * + * Copyright (C) 2015, 2019 International Business Machines Inc. + * Copyright (C) 2014 - 2018 Linaro Ltd. + * Copyright 2026 Google LLC + */ + +#include +#include +#include +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto); + +void gcm_init_p8(u64 htable[4][2], const u8 h[16]); +void gcm_gmult_p8(u8 Xi[16], const u64 htable[4][2]); +void gcm_ghash_p8(u8 Xi[16], const u64 htable[4][2], const u8 *in, size_t len); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + ghash_key_to_polyval(raw_key, &key->h); + + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_init_p8(key->htable, raw_key); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + } else { + /* This reproduces gcm_init_p8() on both LE and BE systems. */ + key->htable[0][0] = 0; + key->htable[0][1] = 0xc200000000000000; + + key->htable[1][0] = 0; + key->htable[1][1] = le64_to_cpu(key->h.lo); + + key->htable[2][0] = le64_to_cpu(key->h.lo); + key->htable[2][1] = le64_to_cpu(key->h.hi); + + key->htable[3][0] = le64_to_cpu(key->h.hi); + key->htable[3][1] = 0; + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_gmult_p8(ghash_acc, key->htable); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + polyval_mul_generic(acc, &key->h); + } +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_ghash_p8(ghash_acc, key->htable, data, + nblocks * GHASH_BLOCK_SIZE); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_has_feature(CPU_FTR_ARCH_207S) && + (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) + static_branch_enable(&have_vec_crypto); +} diff --git a/arch/powerpc/crypto/ghashp8-ppc.pl b/lib/crypto/powerpc/ghashp8-ppc.pl similarity index 98% rename from arch/powerpc/crypto/ghashp8-ppc.pl rename to lib/crypto/powerpc/ghashp8-ppc.pl index 041e633c214f..7c38eedc02cc 100644 --- a/arch/powerpc/crypto/ghashp8-ppc.pl +++ b/lib/crypto/powerpc/ghashp8-ppc.pl @@ -47,6 +47,7 @@ if ($flavour =~ /64/) { $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +( $xlate="${dir}../../../arch/powerpc/crypto/ppc-xlate.pl" and -f $xlate) or die "can't locate ppc-xlate.pl"; open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!";