github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

lib/crypto: x86/aes-ecb: Add AES-NI optimization

Browse Source 
Optimize the crypto library's AES-ECB support with AES-NI, bringing its
performance on par with the "ecb-aes-aesni" skcipher algorithm it will
supersede.

The new assembly functions are written from scratch to fit well into the
crypto library.  However, they are functionally very similar to the
functions in arch/x86/crypto/aesni-intel_asm.S that they will supersede
and are intended to provide parity with those -- including supporting
32-bit mode, having the inner loops do 4 AES blocks per iteration, etc.

Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://patch.msgid.link/20260927224418.109759-5-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Eric Biggers 2026-09-27 15:42:55 -07:00
parent 7a0c7b6afe
commit d825a5f349
3 changed files with 90 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crypto/aes.c
View File

@ -610,7 +610,7 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "ecb(aes)",
.base.cra_driver_name = "ecb-aes-lib",
.base.cra_priority = 110,
.base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
.base.cra_module = THIS_MODULE,

54
lib/crypto/x86/aes-aesni.S
View File

@ -348,3 +348,57 @@ SYM_FUNC_END(aes_encrypt_aesni)
SYM_FUNC_START(aes_decrypt_aesni)
_aes_crypt_aesni 0
SYM_FUNC_END(aes_decrypt_aesni)
.macro _ecb_crypt enc
// Arguments
.set DST, ARG0
.set SRC, ARG1
.set NBLOCKS, ARG2
.set NBLOCKS32, ARG2_32 // Used for improved code density
.set KEY, ARG3
// Other local variables
.set RNDKEY_PTR, ARG4 // Temporary register for _do_aes
.set NROUNDS, TMP_32 // Temporary register for _do_aes
.set AESDATA0, %xmm0
.set AESDATA1, %xmm1
.set AESDATA2, %xmm2
.set AESDATA3, %xmm3
.set RNDKEY, %xmm4 // Temporary register for _do_aes
_prologue uses_arg3=2, uses_arg4=1
sub $4, NBLOCKS
jl .Lecb_loop4_done\@
.p2align 5
.Lecb_loop4\@:
_do_aes_ecb \enc, 0,1,2,3
add $64, SRC
add $64, DST
sub $4, NBLOCKS
jge .Lecb_loop4\@
.Lecb_loop4_done\@:
add $4, NBLOCKS32
jz .Lecb_done\@
.Lecb_loop1\@:
_do_aes_ecb \enc, 0
add $16, SRC
add $16, DST
dec NBLOCKS32
jnz .Lecb_loop1\@
.Lecb_done\@:
_epilogue
.endm
// void aes_ecb_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
// const struct aes_enckey *key);
SYM_FUNC_START(aes_ecb_encrypt_aesni)
_ecb_crypt 1
SYM_FUNC_END(aes_ecb_encrypt_aesni)
// void aes_ecb_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
// const struct aes_key *key);
SYM_FUNC_START(aes_ecb_decrypt_aesni)
_ecb_crypt 0
SYM_FUNC_END(aes_ecb_decrypt_aesni)

35
lib/crypto/x86/aes.h
View File

@ -12,6 +12,8 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aesni);
/* The assembly code assumes the following offsets. */
static_assert(offsetof(struct aes_enckey, nrounds) == 4);
static_assert(offsetof(struct aes_enckey, k.rndkeys) == 16);
static_assert(offsetof(struct aes_key, nrounds) == 4);
static_assert(offsetof(struct aes_key, k.rndkeys) == 16);
static_assert(offsetof(struct aes_key, inv_k.inv_rndkeys) == 256);
void aes128_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys,
@ -81,6 +83,39 @@ static void aes_decrypt_arch(const struct aes_key *key,
}
}
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB)
void aes_ecb_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
const struct aes_enckey *key);
void aes_ecb_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
const struct aes_key *key);
/* len is always a positive multiple of AES_BLOCK_SIZE here. */
#define aes_ecb_encrypt_arch aes_ecb_encrypt_arch
static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len,
const struct aes_enckey *key)
{
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
kernel_fpu_begin();
aes_ecb_encrypt_aesni(dst, src, len / AES_BLOCK_SIZE, key);
kernel_fpu_end();
return true;
}
/* len is always a positive multiple of AES_BLOCK_SIZE here. */
#define aes_ecb_decrypt_arch aes_ecb_decrypt_arch
static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
const struct aes_key *key)
{
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
kernel_fpu_begin();
aes_ecb_decrypt_aesni(dst, src, len / AES_BLOCK_SIZE, key);
kernel_fpu_end();
return true;
}
#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{