github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-12 16:18:45 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

lib/crypto: gf128hash: Support GF128HASH_ARCH without all POLYVAL functions

Browse Source 
Currently, some architectures (arm64 and x86) have optimized code for
both GHASH and POLYVAL.  Others (arm, powerpc, riscv, and s390) have
optimized code only for GHASH.  While POLYVAL support could be
implemented on these other architectures, until then we need to support
the case where arch-optimized functions are present only for GHASH.

Therefore, update the support for arch-optimized POLYVAL functions to
allow architectures to opt into supporting these functions individually.

The new meaning of CONFIG_CRYPTO_LIB_GF128HASH_ARCH is that some level
of GHASH and/or POLYVAL acceleration is provided.

Also provide an implementation of polyval_mul() based on
polyval_blocks_arch(), for when polyval_mul_arch() isn't implemented.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-3-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Eric Biggers 2026-03-18 23:17:03 -07:00
parent 61f66c5216
commit b3b6e8f9b3
4 changed files with 21 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
include/crypto/gf128hash.h
View File

@ -44,20 +44,14 @@ struct polyval_elem {
* exponentiation repeats the POLYVAL dot operation, with its "extra" x^-128.
*/
struct polyval_key {
#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
#ifdef CONFIG_ARM64
/** @h_powers: Powers of the hash key H^8 through H^1 */
struct polyval_elem h_powers[8];
#elif defined(CONFIG_X86)
#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \
(defined(CONFIG_ARM64) || defined(CONFIG_X86))
/** @h_powers: Powers of the hash key H^8 through H^1 */
struct polyval_elem h_powers[8];
#else
#error "Unhandled arch"
#endif
#else /* CONFIG_CRYPTO_LIB_GF128HASH_ARCH */
/** @h: The hash key H */
struct polyval_elem h;
#endif /* !CONFIG_CRYPTO_LIB_GF128HASH_ARCH */
#endif
};
/**
@ -84,19 +78,9 @@ struct polyval_ctx {
*
* Context: Any context.
*/
#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
void polyval_preparekey(struct polyval_key *key,
const u8 raw_key[POLYVAL_BLOCK_SIZE]);
#else
static inline void polyval_preparekey(struct polyval_key *key,
const u8 raw_key[POLYVAL_BLOCK_SIZE])
{
/* Just a simple copy, so inline it. */
memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE);
}
#endif
/**
* polyval_init() - Initialize a POLYVAL context for a new message
* @ctx: The context to initialize

3
lib/crypto/arm64/gf128hash.h
View File

@ -17,6 +17,7 @@ asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
const struct polyval_key *key,
const u8 *data, size_t nblocks);
#define polyval_preparekey_arch polyval_preparekey_arch
static void polyval_preparekey_arch(struct polyval_key *key,
const u8 raw_key[POLYVAL_BLOCK_SIZE])
{
@ -40,6 +41,7 @@ static void polyval_preparekey_arch(struct polyval_key *key,
}
}
#define polyval_mul_arch polyval_mul_arch
static void polyval_mul_arch(struct polyval_elem *acc,
const struct polyval_key *key)
{
@ -51,6 +53,7 @@ static void polyval_mul_arch(struct polyval_elem *acc,
}
}
#define polyval_blocks_arch polyval_blocks_arch
static void polyval_blocks_arch(struct polyval_elem *acc,
const struct polyval_key *key,
const u8 *data, size_t nblocks)

16
lib/crypto/gf128hash.c
View File

@ -217,16 +217,20 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key,
} while (--nblocks);
}
/* Include the arch-optimized implementation of POLYVAL, if one is available. */
#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
#include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */
#endif
void polyval_preparekey(struct polyval_key *key,
const u8 raw_key[POLYVAL_BLOCK_SIZE])
{
#ifdef polyval_preparekey_arch
polyval_preparekey_arch(key, raw_key);
#else
memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE);
#endif
}
EXPORT_SYMBOL_GPL(polyval_preparekey);
#endif /* Else, polyval_preparekey() is an inline function. */
/*
* polyval_mul_generic() and polyval_blocks_generic() take the key as a
@ -238,8 +242,12 @@ EXPORT_SYMBOL_GPL(polyval_preparekey);
static void polyval_mul(struct polyval_ctx *ctx)
{
#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
#ifdef polyval_mul_arch
polyval_mul_arch(&ctx->acc, ctx->key);
#elif defined(polyval_blocks_arch)
static const u8 zeroes[POLYVAL_BLOCK_SIZE];
polyval_blocks_arch(&ctx->acc, ctx->key, zeroes, 1);
#else
polyval_mul_generic(&ctx->acc, &ctx->key->h);
#endif
@ -248,7 +256,7 @@ static void polyval_mul(struct polyval_ctx *ctx)
static void polyval_blocks(struct polyval_ctx *ctx,
const u8 *data, size_t nblocks)
{
#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
#ifdef polyval_blocks_arch
polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks);
#else
polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks);

3
lib/crypto/x86/gf128hash.h
View File

@ -17,6 +17,7 @@ asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc,
const struct polyval_key *key,
const u8 *data, size_t nblocks);
#define polyval_preparekey_arch polyval_preparekey_arch
static void polyval_preparekey_arch(struct polyval_key *key,
const u8 raw_key[POLYVAL_BLOCK_SIZE])
{
@ -40,6 +41,7 @@ static void polyval_preparekey_arch(struct polyval_key *key,
}
}
#define polyval_mul_arch polyval_mul_arch
static void polyval_mul_arch(struct polyval_elem *acc,
const struct polyval_key *key)
{
@ -52,6 +54,7 @@ static void polyval_mul_arch(struct polyval_elem *acc,
}
}
#define polyval_blocks_arch polyval_blocks_arch
static void polyval_blocks_arch(struct polyval_elem *acc,
const struct polyval_key *key,
const u8 *data, size_t nblocks)