mirror of
https://github.com/torvalds/linux.git
synced 2026-11-07 06:17:02 +01:00
lib/crypto: x86/aes-xts: Migrate AVX-optimized code into library
Migrate aes-xts-avx-x86_64.S into lib/crypto/, wiring it up to the XTS library functions instead of the crypto_skcipher API. It still remains available through crypto_skcipher via crypto/aes.c. Some adjustments to the assembly code were needed: - Take 'struct aes_key' instead of 'struct crypto_aes_ctx'. - Remove the ciphertext stealing support from the assembly code, as the library implements it in a generic way instead. (This does slightly reduce performance when the length isn't a multiple of 16 bytes; however, that case seems to never be reached in practice in the kernel. So it makes sense to not extensively optimize for it yet.) - Change 'int len' to 'long nblocks' for compatibility with the library's use of size_t lengths. - Remove the CFI stubs, as the functions are now called directly. - Adjust the argument order to match the caller. Not strictly required, but it's easiest to handle this now when changing the function prototypes anyway and adding the new glue code. This makes the remaining code in aesni-intel_asm.S (which just handled key expansion) unused, so remove that too. Don't remove the selection of CRYPTO_SKCIPHER from CRYPTO_AES_NI_INTEL yet, since it's still needed for the "skcipher_walk" functions. Reviewed-by: Ard Biesheuvel <ardb@kernel.org> Link: https://patch.msgid.link/20260927224418.109759-15-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Eric Biggers
parent
7e58d75b7c
commit
2bd6888849
|
|
@ -3,7 +3,7 @@
|
|||
menu "Accelerated Cryptographic Algorithms for CPU (x86)"
|
||||
|
||||
config CRYPTO_AES_NI_INTEL
|
||||
tristate "Ciphers: AES, modes: XTS, GCM (AES-NI/VAES)"
|
||||
tristate "Ciphers: AES, modes: GCM (AES-NI/VAES)"
|
||||
depends on 64BIT
|
||||
select CRYPTO_AEAD
|
||||
select CRYPTO_LIB_AES
|
||||
|
|
@ -11,12 +11,14 @@ config CRYPTO_AES_NI_INTEL
|
|||
select CRYPTO_SKCIPHER
|
||||
help
|
||||
AEAD cipher: AES with GCM
|
||||
Length-preserving ciphers: AES with XTS
|
||||
|
||||
Architecture: x86_64 using:
|
||||
- AES-NI (AES new instructions)
|
||||
- VAES (Vector AES)
|
||||
|
||||
Note: this option no longer provides the accelerated XTS, CBC, CTR,
|
||||
and ECB code. For those just use CRYPTO_XTS, CRYPTO_CBC, etc.
|
||||
|
||||
config CRYPTO_BLOWFISH_X86_64
|
||||
tristate "Ciphers: Blowfish, modes: ECB, CBC"
|
||||
depends on 64BIT
|
||||
|
|
|
|||
|
|
@ -40,12 +40,10 @@ obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o
|
|||
aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
|
||||
aesni-intel-y := aesni-intel_asm.o \
|
||||
aesni-intel_glue.o \
|
||||
aesni-intel-y := aesni-intel_glue.o \
|
||||
aes-gcm-aesni-x86_64.o \
|
||||
aes-gcm-vaes-avx2.o \
|
||||
aes-gcm-vaes-avx512.o \
|
||||
aes-xts-avx-x86_64.o
|
||||
aes-gcm-vaes-avx512.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o
|
||||
sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o
|
||||
|
|
|
|||
|
|
@ -1,220 +0,0 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
/*
|
||||
* Implement AES algorithm in Intel AES-NI instructions.
|
||||
*
|
||||
* The white paper of AES-NI instructions can be downloaded from:
|
||||
* http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
|
||||
*
|
||||
* Copyright (C) 2008, Intel Corp.
|
||||
* Author: Huang Ying <ying.huang@intel.com>
|
||||
* Vinodh Gopal <vinodh.gopal@intel.com>
|
||||
* Kahraman Akdemir
|
||||
*
|
||||
* Copyright (c) 2010, Intel Corporation.
|
||||
*
|
||||
* Ported x86_64 version to x86:
|
||||
* Author: Mathias Krause <minipli@googlemail.com>
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/objtool.h>
|
||||
#include <asm/frame.h>
|
||||
|
||||
#ifdef __x86_64__
|
||||
#define AREG %rax
|
||||
#define KEYP %rdi
|
||||
#define OUTP %rsi
|
||||
#define UKEYP OUTP
|
||||
#define T1 %r10
|
||||
#define TKEYP T1
|
||||
#define T2 %r11
|
||||
#else
|
||||
#define AREG %eax
|
||||
#define KEYP %edi
|
||||
#define OUTP AREG
|
||||
#define UKEYP OUTP
|
||||
#define T1 %ecx
|
||||
#define TKEYP T1
|
||||
#endif
|
||||
|
||||
SYM_FUNC_START_LOCAL(_key_expansion_256a)
|
||||
pshufd $0b11111111, %xmm1, %xmm1
|
||||
shufps $0b00010000, %xmm0, %xmm4
|
||||
pxor %xmm4, %xmm0
|
||||
shufps $0b10001100, %xmm0, %xmm4
|
||||
pxor %xmm4, %xmm0
|
||||
pxor %xmm1, %xmm0
|
||||
movaps %xmm0, (TKEYP)
|
||||
add $0x10, TKEYP
|
||||
RET
|
||||
SYM_FUNC_END(_key_expansion_256a)
|
||||
SYM_FUNC_ALIAS_LOCAL(_key_expansion_128, _key_expansion_256a)
|
||||
|
||||
SYM_FUNC_START_LOCAL(_key_expansion_192a)
|
||||
pshufd $0b01010101, %xmm1, %xmm1
|
||||
shufps $0b00010000, %xmm0, %xmm4
|
||||
pxor %xmm4, %xmm0
|
||||
shufps $0b10001100, %xmm0, %xmm4
|
||||
pxor %xmm4, %xmm0
|
||||
pxor %xmm1, %xmm0
|
||||
|
||||
movaps %xmm2, %xmm5
|
||||
movaps %xmm2, %xmm6
|
||||
pslldq $4, %xmm5
|
||||
pshufd $0b11111111, %xmm0, %xmm3
|
||||
pxor %xmm3, %xmm2
|
||||
pxor %xmm5, %xmm2
|
||||
|
||||
movaps %xmm0, %xmm1
|
||||
shufps $0b01000100, %xmm0, %xmm6
|
||||
movaps %xmm6, (TKEYP)
|
||||
shufps $0b01001110, %xmm2, %xmm1
|
||||
movaps %xmm1, 0x10(TKEYP)
|
||||
add $0x20, TKEYP
|
||||
RET
|
||||
SYM_FUNC_END(_key_expansion_192a)
|
||||
|
||||
SYM_FUNC_START_LOCAL(_key_expansion_192b)
|
||||
pshufd $0b01010101, %xmm1, %xmm1
|
||||
shufps $0b00010000, %xmm0, %xmm4
|
||||
pxor %xmm4, %xmm0
|
||||
shufps $0b10001100, %xmm0, %xmm4
|
||||
pxor %xmm4, %xmm0
|
||||
pxor %xmm1, %xmm0
|
||||
|
||||
movaps %xmm2, %xmm5
|
||||
pslldq $4, %xmm5
|
||||
pshufd $0b11111111, %xmm0, %xmm3
|
||||
pxor %xmm3, %xmm2
|
||||
pxor %xmm5, %xmm2
|
||||
|
||||
movaps %xmm0, (TKEYP)
|
||||
add $0x10, TKEYP
|
||||
RET
|
||||
SYM_FUNC_END(_key_expansion_192b)
|
||||
|
||||
SYM_FUNC_START_LOCAL(_key_expansion_256b)
|
||||
pshufd $0b10101010, %xmm1, %xmm1
|
||||
shufps $0b00010000, %xmm2, %xmm4
|
||||
pxor %xmm4, %xmm2
|
||||
shufps $0b10001100, %xmm2, %xmm4
|
||||
pxor %xmm4, %xmm2
|
||||
pxor %xmm1, %xmm2
|
||||
movaps %xmm2, (TKEYP)
|
||||
add $0x10, TKEYP
|
||||
RET
|
||||
SYM_FUNC_END(_key_expansion_256b)
|
||||
|
||||
/*
|
||||
* void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
|
||||
* unsigned int key_len)
|
||||
*/
|
||||
SYM_FUNC_START(aesni_set_key)
|
||||
FRAME_BEGIN
|
||||
#ifndef __x86_64__
|
||||
pushl KEYP
|
||||
movl (FRAME_OFFSET+8)(%esp), KEYP # ctx
|
||||
movl (FRAME_OFFSET+12)(%esp), UKEYP # in_key
|
||||
movl (FRAME_OFFSET+16)(%esp), %edx # key_len
|
||||
#endif
|
||||
movups (UKEYP), %xmm0 # user key (first 16 bytes)
|
||||
movaps %xmm0, (KEYP)
|
||||
lea 0x10(KEYP), TKEYP # key addr
|
||||
movl %edx, 480(KEYP)
|
||||
pxor %xmm4, %xmm4 # xmm4 is assumed 0 in _key_expansion_x
|
||||
cmp $24, %dl
|
||||
jb .Lenc_key128
|
||||
je .Lenc_key192
|
||||
movups 0x10(UKEYP), %xmm2 # other user key
|
||||
movaps %xmm2, (TKEYP)
|
||||
add $0x10, TKEYP
|
||||
aeskeygenassist $0x1, %xmm2, %xmm1 # round 1
|
||||
call _key_expansion_256a
|
||||
aeskeygenassist $0x1, %xmm0, %xmm1
|
||||
call _key_expansion_256b
|
||||
aeskeygenassist $0x2, %xmm2, %xmm1 # round 2
|
||||
call _key_expansion_256a
|
||||
aeskeygenassist $0x2, %xmm0, %xmm1
|
||||
call _key_expansion_256b
|
||||
aeskeygenassist $0x4, %xmm2, %xmm1 # round 3
|
||||
call _key_expansion_256a
|
||||
aeskeygenassist $0x4, %xmm0, %xmm1
|
||||
call _key_expansion_256b
|
||||
aeskeygenassist $0x8, %xmm2, %xmm1 # round 4
|
||||
call _key_expansion_256a
|
||||
aeskeygenassist $0x8, %xmm0, %xmm1
|
||||
call _key_expansion_256b
|
||||
aeskeygenassist $0x10, %xmm2, %xmm1 # round 5
|
||||
call _key_expansion_256a
|
||||
aeskeygenassist $0x10, %xmm0, %xmm1
|
||||
call _key_expansion_256b
|
||||
aeskeygenassist $0x20, %xmm2, %xmm1 # round 6
|
||||
call _key_expansion_256a
|
||||
aeskeygenassist $0x20, %xmm0, %xmm1
|
||||
call _key_expansion_256b
|
||||
aeskeygenassist $0x40, %xmm2, %xmm1 # round 7
|
||||
call _key_expansion_256a
|
||||
jmp .Ldec_key
|
||||
.Lenc_key192:
|
||||
movq 0x10(UKEYP), %xmm2 # other user key
|
||||
aeskeygenassist $0x1, %xmm2, %xmm1 # round 1
|
||||
call _key_expansion_192a
|
||||
aeskeygenassist $0x2, %xmm2, %xmm1 # round 2
|
||||
call _key_expansion_192b
|
||||
aeskeygenassist $0x4, %xmm2, %xmm1 # round 3
|
||||
call _key_expansion_192a
|
||||
aeskeygenassist $0x8, %xmm2, %xmm1 # round 4
|
||||
call _key_expansion_192b
|
||||
aeskeygenassist $0x10, %xmm2, %xmm1 # round 5
|
||||
call _key_expansion_192a
|
||||
aeskeygenassist $0x20, %xmm2, %xmm1 # round 6
|
||||
call _key_expansion_192b
|
||||
aeskeygenassist $0x40, %xmm2, %xmm1 # round 7
|
||||
call _key_expansion_192a
|
||||
aeskeygenassist $0x80, %xmm2, %xmm1 # round 8
|
||||
call _key_expansion_192b
|
||||
jmp .Ldec_key
|
||||
.Lenc_key128:
|
||||
aeskeygenassist $0x1, %xmm0, %xmm1 # round 1
|
||||
call _key_expansion_128
|
||||
aeskeygenassist $0x2, %xmm0, %xmm1 # round 2
|
||||
call _key_expansion_128
|
||||
aeskeygenassist $0x4, %xmm0, %xmm1 # round 3
|
||||
call _key_expansion_128
|
||||
aeskeygenassist $0x8, %xmm0, %xmm1 # round 4
|
||||
call _key_expansion_128
|
||||
aeskeygenassist $0x10, %xmm0, %xmm1 # round 5
|
||||
call _key_expansion_128
|
||||
aeskeygenassist $0x20, %xmm0, %xmm1 # round 6
|
||||
call _key_expansion_128
|
||||
aeskeygenassist $0x40, %xmm0, %xmm1 # round 7
|
||||
call _key_expansion_128
|
||||
aeskeygenassist $0x80, %xmm0, %xmm1 # round 8
|
||||
call _key_expansion_128
|
||||
aeskeygenassist $0x1b, %xmm0, %xmm1 # round 9
|
||||
call _key_expansion_128
|
||||
aeskeygenassist $0x36, %xmm0, %xmm1 # round 10
|
||||
call _key_expansion_128
|
||||
.Ldec_key:
|
||||
sub $0x10, TKEYP
|
||||
movaps (KEYP), %xmm0
|
||||
movaps (TKEYP), %xmm1
|
||||
movaps %xmm0, 240(TKEYP)
|
||||
movaps %xmm1, 240(KEYP)
|
||||
add $0x10, KEYP
|
||||
lea 240-16(TKEYP), UKEYP
|
||||
.align 4
|
||||
.Ldec_key_loop:
|
||||
movaps (KEYP), %xmm0
|
||||
aesimc %xmm0, %xmm1
|
||||
movaps %xmm1, (UKEYP)
|
||||
add $0x10, KEYP
|
||||
sub $0x10, UKEYP
|
||||
cmp TKEYP, KEYP
|
||||
jb .Ldec_key_loop
|
||||
#ifndef __x86_64__
|
||||
popl KEYP
|
||||
#endif
|
||||
FRAME_END
|
||||
RET
|
||||
SYM_FUNC_END(aesni_set_key)
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
/*
|
||||
* Support for AES-NI and VAES instructions. This file contains glue code.
|
||||
* The real AES implementations are in aesni-intel_asm.S and other .S files.
|
||||
* The real AES implementations are in .S files.
|
||||
*
|
||||
* Copyright (C) 2008, Intel Corp.
|
||||
* Author: Huang Ying <ying.huang@intel.com>
|
||||
|
|
@ -26,7 +26,6 @@
|
|||
#include <crypto/b128ops.h>
|
||||
#include <crypto/gcm.h>
|
||||
#include <crypto/gf128mul.h>
|
||||
#include <crypto/xts.h>
|
||||
#include <asm/cpu_device_id.h>
|
||||
#include <asm/simd.h>
|
||||
#include <crypto/scatterwalk.h>
|
||||
|
|
@ -38,217 +37,6 @@
|
|||
#include <linux/spinlock.h>
|
||||
#include <linux/static_call.h>
|
||||
|
||||
|
||||
#define AESNI_ALIGN 16
|
||||
#define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
|
||||
#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
|
||||
#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
|
||||
|
||||
struct aesni_xts_ctx {
|
||||
struct crypto_aes_ctx tweak_ctx AESNI_ALIGN_ATTR;
|
||||
struct crypto_aes_ctx crypt_ctx AESNI_ALIGN_ATTR;
|
||||
};
|
||||
|
||||
static inline void *aes_align_addr(void *addr)
|
||||
{
|
||||
if (crypto_tfm_ctx_alignment() >= AESNI_ALIGN)
|
||||
return addr;
|
||||
return PTR_ALIGN(addr, AESNI_ALIGN);
|
||||
}
|
||||
|
||||
asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
|
||||
unsigned int key_len);
|
||||
|
||||
static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm)
|
||||
{
|
||||
return aes_align_addr(crypto_skcipher_ctx(tfm));
|
||||
}
|
||||
|
||||
static int aes_set_key_common(struct crypto_aes_ctx *ctx,
|
||||
const u8 *in_key, unsigned int key_len)
|
||||
{
|
||||
int err;
|
||||
|
||||
if (!crypto_simd_usable())
|
||||
return aes_expandkey(ctx, in_key, key_len);
|
||||
|
||||
err = aes_check_keylen(key_len);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
kernel_fpu_begin();
|
||||
aesni_set_key(ctx, in_key, key_len);
|
||||
kernel_fpu_end();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key,
|
||||
unsigned int keylen)
|
||||
{
|
||||
struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
|
||||
int err;
|
||||
|
||||
err = xts_verify_key(tfm, key, keylen);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
keylen /= 2;
|
||||
|
||||
/* first half of xts-key is for crypt */
|
||||
err = aes_set_key_common(&ctx->crypt_ctx, key, keylen);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
/* second half of xts-key is for tweak */
|
||||
return aes_set_key_common(&ctx->tweak_ctx, key + keylen, keylen);
|
||||
}
|
||||
|
||||
typedef void (*xts_encrypt_iv_func)(const struct crypto_aes_ctx *tweak_key,
|
||||
u8 iv[AES_BLOCK_SIZE]);
|
||||
typedef void (*xts_crypt_func)(const struct crypto_aes_ctx *key,
|
||||
const u8 *src, u8 *dst, int len,
|
||||
u8 tweak[AES_BLOCK_SIZE]);
|
||||
|
||||
/* This handles cases where the source and/or destination span pages. */
|
||||
static noinline int
|
||||
xts_crypt_slowpath(struct skcipher_request *req, xts_crypt_func crypt_func)
|
||||
{
|
||||
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
|
||||
const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
|
||||
int tail = req->cryptlen % AES_BLOCK_SIZE;
|
||||
struct scatterlist sg_src[2], sg_dst[2];
|
||||
struct skcipher_request subreq;
|
||||
struct skcipher_walk walk;
|
||||
struct scatterlist *src, *dst;
|
||||
int err;
|
||||
|
||||
/*
|
||||
* If the message length isn't divisible by the AES block size, then
|
||||
* separate off the last full block and the partial block. This ensures
|
||||
* that they are processed in the same call to the assembly function,
|
||||
* which is required for ciphertext stealing.
|
||||
*/
|
||||
if (tail) {
|
||||
skcipher_request_set_tfm(&subreq, tfm);
|
||||
skcipher_request_set_callback(&subreq,
|
||||
skcipher_request_flags(req),
|
||||
NULL, NULL);
|
||||
skcipher_request_set_crypt(&subreq, req->src, req->dst,
|
||||
req->cryptlen - tail - AES_BLOCK_SIZE,
|
||||
req->iv);
|
||||
req = &subreq;
|
||||
}
|
||||
|
||||
err = skcipher_walk_virt(&walk, req, false);
|
||||
|
||||
while (walk.nbytes) {
|
||||
kernel_fpu_begin();
|
||||
(*crypt_func)(&ctx->crypt_ctx,
|
||||
walk.src.virt.addr, walk.dst.virt.addr,
|
||||
walk.nbytes & ~(AES_BLOCK_SIZE - 1), req->iv);
|
||||
kernel_fpu_end();
|
||||
err = skcipher_walk_done(&walk,
|
||||
walk.nbytes & (AES_BLOCK_SIZE - 1));
|
||||
}
|
||||
|
||||
if (err || !tail)
|
||||
return err;
|
||||
|
||||
/* Do ciphertext stealing with the last full block and partial block. */
|
||||
|
||||
dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
|
||||
if (req->dst != req->src)
|
||||
dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
|
||||
|
||||
skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
|
||||
req->iv);
|
||||
|
||||
err = skcipher_walk_virt(&walk, req, false);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
kernel_fpu_begin();
|
||||
(*crypt_func)(&ctx->crypt_ctx, walk.src.virt.addr, walk.dst.virt.addr,
|
||||
walk.nbytes, req->iv);
|
||||
kernel_fpu_end();
|
||||
|
||||
return skcipher_walk_done(&walk, 0);
|
||||
}
|
||||
|
||||
/* __always_inline to avoid indirect call in fastpath */
|
||||
static __always_inline int
|
||||
xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv,
|
||||
xts_crypt_func crypt_func)
|
||||
{
|
||||
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
|
||||
const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
|
||||
|
||||
if (unlikely(req->cryptlen < AES_BLOCK_SIZE))
|
||||
return -EINVAL;
|
||||
|
||||
kernel_fpu_begin();
|
||||
(*encrypt_iv)(&ctx->tweak_ctx, req->iv);
|
||||
|
||||
/*
|
||||
* In practice, virtually all XTS plaintexts and ciphertexts are either
|
||||
* 512 or 4096 bytes and do not use multiple scatterlist elements. To
|
||||
* optimize the performance of these cases, the below fast-path handles
|
||||
* single-scatterlist-element messages as efficiently as possible. The
|
||||
* code is 64-bit specific, as it assumes no page mapping is needed.
|
||||
*/
|
||||
if (likely(req->src->length >= req->cryptlen &&
|
||||
req->dst->length >= req->cryptlen)) {
|
||||
(*crypt_func)(&ctx->crypt_ctx, sg_virt(req->src),
|
||||
sg_virt(req->dst), req->cryptlen, req->iv);
|
||||
kernel_fpu_end();
|
||||
return 0;
|
||||
}
|
||||
kernel_fpu_end();
|
||||
return xts_crypt_slowpath(req, crypt_func);
|
||||
}
|
||||
|
||||
asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
|
||||
u8 iv[AES_BLOCK_SIZE]);
|
||||
|
||||
#define DEFINE_AVX_SKCIPHER_ALGS(suffix, driver_name_suffix, priority) \
|
||||
\
|
||||
asmlinkage void \
|
||||
aes_xts_encrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \
|
||||
u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \
|
||||
asmlinkage void \
|
||||
aes_xts_decrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \
|
||||
u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \
|
||||
\
|
||||
static int xts_encrypt_##suffix(struct skcipher_request *req) \
|
||||
{ \
|
||||
return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_encrypt_##suffix); \
|
||||
} \
|
||||
\
|
||||
static int xts_decrypt_##suffix(struct skcipher_request *req) \
|
||||
{ \
|
||||
return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_decrypt_##suffix); \
|
||||
} \
|
||||
\
|
||||
static struct skcipher_alg skcipher_algs_##suffix[] = {{ \
|
||||
.base.cra_name = "xts(aes)", \
|
||||
.base.cra_driver_name = "xts-aes-" driver_name_suffix, \
|
||||
.base.cra_priority = priority, \
|
||||
.base.cra_blocksize = AES_BLOCK_SIZE, \
|
||||
.base.cra_ctxsize = XTS_AES_CTX_SIZE, \
|
||||
.base.cra_module = THIS_MODULE, \
|
||||
.min_keysize = 2 * AES_MIN_KEY_SIZE, \
|
||||
.max_keysize = 2 * AES_MAX_KEY_SIZE, \
|
||||
.ivsize = AES_BLOCK_SIZE, \
|
||||
.walksize = 2 * AES_BLOCK_SIZE, \
|
||||
.setkey = xts_setkey_aesni, \
|
||||
.encrypt = xts_encrypt_##suffix, \
|
||||
.decrypt = xts_decrypt_##suffix, \
|
||||
}}
|
||||
|
||||
DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500);
|
||||
DEFINE_AVX_SKCIPHER_ALGS(vaes_avx2, "vaes-avx2", 600);
|
||||
DEFINE_AVX_SKCIPHER_ALGS(vaes_avx512, "vaes-avx512", 800);
|
||||
|
||||
/* The common part of the x86_64 AES-GCM key struct */
|
||||
struct aes_gcm_key {
|
||||
/* Expanded AES key and the AES key length in bytes */
|
||||
|
|
@ -1002,10 +790,6 @@ static int __init register_avx_algs(void)
|
|||
|
||||
if (!boot_cpu_has(X86_FEATURE_AVX))
|
||||
return 0;
|
||||
err = crypto_register_skciphers(skcipher_algs_aesni_avx,
|
||||
ARRAY_SIZE(skcipher_algs_aesni_avx));
|
||||
if (err)
|
||||
return err;
|
||||
err = crypto_register_aeads(aes_gcm_algs_aesni_avx,
|
||||
ARRAY_SIZE(aes_gcm_algs_aesni_avx));
|
||||
if (err)
|
||||
|
|
@ -1022,10 +806,6 @@ static int __init register_avx_algs(void)
|
|||
!boot_cpu_has(X86_FEATURE_PCLMULQDQ) ||
|
||||
!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
|
||||
return 0;
|
||||
err = crypto_register_skciphers(skcipher_algs_vaes_avx2,
|
||||
ARRAY_SIZE(skcipher_algs_vaes_avx2));
|
||||
if (err)
|
||||
return err;
|
||||
err = crypto_register_aeads(aes_gcm_algs_vaes_avx2,
|
||||
ARRAY_SIZE(aes_gcm_algs_vaes_avx2));
|
||||
if (err)
|
||||
|
|
@ -1041,16 +821,10 @@ static int __init register_avx_algs(void)
|
|||
if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx512); i++)
|
||||
skcipher_algs_vaes_avx512[i].base.cra_priority = 1;
|
||||
for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx512); i++)
|
||||
aes_gcm_algs_vaes_avx512[i].base.cra_priority = 1;
|
||||
}
|
||||
|
||||
err = crypto_register_skciphers(skcipher_algs_vaes_avx512,
|
||||
ARRAY_SIZE(skcipher_algs_vaes_avx512));
|
||||
if (err)
|
||||
return err;
|
||||
err = crypto_register_aeads(aes_gcm_algs_vaes_avx512,
|
||||
ARRAY_SIZE(aes_gcm_algs_vaes_avx512));
|
||||
if (err)
|
||||
|
|
@ -1059,19 +833,13 @@ static int __init register_avx_algs(void)
|
|||
return 0;
|
||||
}
|
||||
|
||||
#define unregister_skciphers(A) \
|
||||
if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
|
||||
crypto_unregister_skciphers((A), ARRAY_SIZE(A))
|
||||
#define unregister_aeads(A) \
|
||||
if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
|
||||
crypto_unregister_aeads((A), ARRAY_SIZE(A))
|
||||
|
||||
static void unregister_avx_algs(void)
|
||||
{
|
||||
unregister_skciphers(skcipher_algs_aesni_avx);
|
||||
unregister_aeads(aes_gcm_algs_aesni_avx);
|
||||
unregister_skciphers(skcipher_algs_vaes_avx2);
|
||||
unregister_skciphers(skcipher_algs_vaes_avx512);
|
||||
unregister_aeads(aes_gcm_algs_vaes_avx2);
|
||||
unregister_aeads(aes_gcm_algs_vaes_avx512);
|
||||
}
|
||||
|
|
@ -1117,6 +885,7 @@ static void __exit aesni_exit(void)
|
|||
module_init(aesni_init);
|
||||
module_exit(aesni_exit);
|
||||
|
||||
MODULE_DESCRIPTION("AES cipher and modes, optimized with AES-NI or VAES instructions");
|
||||
MODULE_DESCRIPTION("AES-GCM, optimized with AES-NI or VAES instructions");
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_ALIAS_CRYPTO("aes");
|
||||
MODULE_ALIAS_CRYPTO("gcm(aes)");
|
||||
MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))");
|
||||
|
|
|
|||
|
|
@ -708,7 +708,7 @@ static struct skcipher_alg skcipher_algs[] = {
|
|||
{
|
||||
.base.cra_name = "xts(aes)",
|
||||
.base.cra_driver_name = "xts-aes-lib",
|
||||
.base.cra_priority = 110,
|
||||
.base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
|
||||
.base.cra_blocksize = AES_BLOCK_SIZE,
|
||||
.base.cra_ctxsize = sizeof(struct aes_xts_key),
|
||||
.base.cra_module = THIS_MODULE,
|
||||
|
|
|
|||
|
|
@ -57,6 +57,9 @@ libaes-$(CONFIG_X86) += x86/aes-aesni.o
|
|||
ifneq ($(CONFIG_CRYPTO_LIB_AES_CTR),)
|
||||
libaes-$(CONFIG_X86_64) += x86/aes-ctr-avx-x86_64.o
|
||||
endif
|
||||
ifneq ($(CONFIG_CRYPTO_LIB_AES_XTS),)
|
||||
libaes-$(CONFIG_X86_64) += x86/aes-xts-avx-x86_64.o
|
||||
endif
|
||||
endif # CONFIG_CRYPTO_LIB_AES_ARCH
|
||||
|
||||
# clean-files must be defined unconditionally
|
||||
|
|
|
|||
|
|
@ -80,14 +80,17 @@
|
|||
* any CPUs that support VAES but not VPCLMULQDQ. If that changes, we might
|
||||
* need to start also providing an implementation using VAES alone.
|
||||
*
|
||||
* The AES-XTS implementations in this file support everything required by the
|
||||
* crypto API, including support for arbitrary input lengths and multi-part
|
||||
* processing. However, they are most heavily optimized for the common case of
|
||||
* power-of-2 length inputs that are processed in a single part (disk sectors).
|
||||
* These assembly functions don't handle ciphertext stealing, i.e, lengths that
|
||||
* aren't a multiple of 16 bytes. That case is not actually reached in the
|
||||
* current use cases of AES-XTS in the kernel and is just handled by the C code.
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/cfi_types.h>
|
||||
|
||||
// Offsets in struct aes_key
|
||||
#define OFFSETOF_KEYLEN 0
|
||||
#define OFFSETOF_RNDKEYS 16
|
||||
#define OFFSETOF_INVRNDKEYS 256
|
||||
|
||||
.section .rodata
|
||||
.p2align 4
|
||||
|
|
@ -111,16 +114,6 @@
|
|||
.Llshift_amounts:
|
||||
.byte 0, 0, 1, 1, 2, 2, 3, 3
|
||||
|
||||
// This table contains constants for vpshufb and vpblendvb, used to
|
||||
// handle variable byte shifts and blending during ciphertext stealing
|
||||
// on CPUs that don't support AVX512-style masking.
|
||||
.Lcts_permute_table:
|
||||
.byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
|
||||
.byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
|
||||
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
|
||||
.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
|
||||
.byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
|
||||
.byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
|
||||
.text
|
||||
|
||||
.macro _define_Vi i
|
||||
|
|
@ -149,14 +142,13 @@
|
|||
.endif
|
||||
|
||||
// Function parameters
|
||||
.set KEY, %rdi // Initially points to crypto_aes_ctx, then is
|
||||
// advanced to point to 7th-from-last round key
|
||||
.set DST, %rdi // Pointer to next destination data
|
||||
.set SRC, %rsi // Pointer to next source data
|
||||
.set DST, %rdx // Pointer to next destination data
|
||||
.set LEN, %ecx // Remaining length in bytes
|
||||
.set LEN8, %cl
|
||||
.set LEN64, %rcx
|
||||
.set TWEAK, %r8 // Pointer to next tweak
|
||||
.set NBLOCKS, %rdx // Number of blocks remaining
|
||||
.set NBLOCKS32, %edx // Used for improved code density
|
||||
.set TWEAK, %rcx // Pointer to next tweak
|
||||
.set KEY, %r8 // Initially points to struct aes_key, then is
|
||||
// advanced to point to 7th-from-last round key
|
||||
|
||||
// %rax holds the AES key length in bytes.
|
||||
.set KEYLEN, %eax
|
||||
|
|
@ -468,9 +460,9 @@
|
|||
|
||||
// Select either the encryption round keys or the decryption round keys.
|
||||
.if \enc
|
||||
.set OFFS, 0
|
||||
.set OFFS, OFFSETOF_RNDKEYS
|
||||
.else
|
||||
.set OFFS, 240
|
||||
.set OFFS, OFFSETOF_INVRNDKEYS
|
||||
.endif
|
||||
|
||||
// Load the round key for "round 0".
|
||||
|
|
@ -615,19 +607,8 @@
|
|||
.macro _aes_xts_crypt enc
|
||||
_define_aliases
|
||||
|
||||
.if !\enc
|
||||
// When decrypting a message whose length isn't a multiple of the AES
|
||||
// block length, exclude the last full block from the main loop by
|
||||
// subtracting 16 from LEN. This is needed because ciphertext stealing
|
||||
// decryption uses the last two tweaks in reverse order. We'll handle
|
||||
// the last full block and the partial block specially at the end.
|
||||
lea -16(LEN), %eax
|
||||
test $15, LEN8
|
||||
cmovnz %eax, LEN
|
||||
.endif
|
||||
|
||||
// Load the AES key length: 16 (AES-128), 24 (AES-192), or 32 (AES-256).
|
||||
movl 480(KEY), KEYLEN
|
||||
movl OFFSETOF_KEYLEN(KEY), KEYLEN
|
||||
|
||||
// Setup the pointer to the round keys and cache as many as possible.
|
||||
_setup_round_keys \enc
|
||||
|
|
@ -635,7 +616,7 @@
|
|||
// Compute the first set of tweaks TWEAK[0-3].
|
||||
_compute_first_set_of_tweaks
|
||||
|
||||
add $-4*VL, LEN // shorter than 'sub 4*VL' when VL=32
|
||||
sub $4*VL/16, NBLOCKS
|
||||
jl .Lhandle_remainder\@
|
||||
|
||||
.Lmain_loop\@:
|
||||
|
|
@ -715,13 +696,13 @@
|
|||
|
||||
sub $-4*VL, SRC // shorter than 'add 4*VL' when VL=32
|
||||
sub $-4*VL, DST
|
||||
add $-4*VL, LEN
|
||||
sub $4*VL/16, NBLOCKS
|
||||
jge .Lmain_loop\@
|
||||
|
||||
// Check for the uncommon case where the data length isn't a multiple of
|
||||
// 4*VL. Handle it out-of-line in order to optimize for the common
|
||||
// case. In the common case, just fall through to the ret.
|
||||
test $4*VL-1, LEN8
|
||||
test $(4*VL/16)-1, NBLOCKS32
|
||||
jnz .Lhandle_remainder\@
|
||||
.Ldone\@:
|
||||
// Store the next tweak back to *TWEAK to support continuation calls.
|
||||
|
|
@ -733,9 +714,9 @@
|
|||
|
||||
.Lhandle_remainder\@:
|
||||
|
||||
// En/decrypt any remaining full blocks, one vector at a time.
|
||||
// En/decrypt any remaining blocks, one vector at a time.
|
||||
.if VL > 16
|
||||
add $3*VL, LEN // Undo extra sub of 4*VL, then sub VL.
|
||||
add $3*VL/16, NBLOCKS32
|
||||
jl .Lvec_at_a_time_done\@
|
||||
.Lvec_at_a_time\@:
|
||||
_vmovdqu (SRC), V0
|
||||
|
|
@ -744,16 +725,16 @@
|
|||
_next_tweakvec TWEAK0, V0, V1, TWEAK0
|
||||
add $VL, SRC
|
||||
add $VL, DST
|
||||
sub $VL, LEN
|
||||
sub $VL/16, NBLOCKS32
|
||||
jge .Lvec_at_a_time\@
|
||||
.Lvec_at_a_time_done\@:
|
||||
add $VL-16, LEN // Undo extra sub of VL, then sub 16.
|
||||
add $VL/16, NBLOCKS32
|
||||
.else
|
||||
add $4*VL-16, LEN // Undo extra sub of 4*VL, then sub 16.
|
||||
add $4*VL/16, NBLOCKS32
|
||||
.endif
|
||||
|
||||
// En/decrypt any remaining full blocks, one at a time.
|
||||
jl .Lblock_at_a_time_done\@
|
||||
// En/decrypt any remaining blocks, one at a time.
|
||||
jz .Ldone\@
|
||||
.Lblock_at_a_time\@:
|
||||
vmovdqu (SRC), %xmm0
|
||||
_aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0, tmp=%xmm1
|
||||
|
|
@ -761,92 +742,26 @@
|
|||
_next_tweak TWEAK0_XMM, %xmm0, TWEAK0_XMM
|
||||
add $16, SRC
|
||||
add $16, DST
|
||||
sub $16, LEN
|
||||
jge .Lblock_at_a_time\@
|
||||
.Lblock_at_a_time_done\@:
|
||||
add $16, LEN // Undo the extra sub of 16.
|
||||
// Now 0 <= LEN <= 15. If LEN is zero, we're done.
|
||||
jz .Ldone\@
|
||||
|
||||
// Otherwise 1 <= LEN <= 15, but the real remaining length is 16 + LEN.
|
||||
// Do ciphertext stealing to process the last 16 + LEN bytes.
|
||||
|
||||
.if \enc
|
||||
// If encrypting, the main loop already encrypted the last full block to
|
||||
// create the CTS intermediate ciphertext. Prepare for the rest of CTS
|
||||
// by rewinding the pointers and loading the intermediate ciphertext.
|
||||
sub $16, SRC
|
||||
sub $16, DST
|
||||
vmovdqu (DST), %xmm0
|
||||
.else
|
||||
// If decrypting, the main loop didn't decrypt the last full block
|
||||
// because CTS decryption uses the last two tweaks in reverse order.
|
||||
// Do it now by advancing the tweak and decrypting the last full block.
|
||||
_next_tweak TWEAK0_XMM, %xmm0, TWEAK1_XMM
|
||||
vmovdqu (SRC), %xmm0
|
||||
_aes_crypt \enc, _XMM, TWEAK1_XMM, %xmm0, tmp=%xmm1
|
||||
.endif
|
||||
|
||||
.if USE_AVX512
|
||||
// Create a mask that has the first LEN bits set.
|
||||
mov $-1, %r9d
|
||||
bzhi LEN, %r9d, %r9d
|
||||
kmovd %r9d, %k1
|
||||
|
||||
// Swap the first LEN bytes of the en/decryption of the last full block
|
||||
// with the partial block. Note that to support in-place en/decryption,
|
||||
// the load from the src partial block must happen before the store to
|
||||
// the dst partial block.
|
||||
vmovdqa %xmm0, %xmm1
|
||||
vmovdqu8 16(SRC), %xmm0{%k1}
|
||||
vmovdqu8 %xmm1, 16(DST){%k1}
|
||||
.else
|
||||
lea .Lcts_permute_table(%rip), %r9
|
||||
|
||||
// Load the src partial block, left-aligned. Note that to support
|
||||
// in-place en/decryption, this must happen before the store to the dst
|
||||
// partial block.
|
||||
vmovdqu (SRC, LEN64, 1), %xmm1
|
||||
|
||||
// Shift the first LEN bytes of the en/decryption of the last full block
|
||||
// to the end of a register, then store it to DST+LEN. This stores the
|
||||
// dst partial block. It also writes to the second part of the dst last
|
||||
// full block, but that part is overwritten later.
|
||||
vpshufb (%r9, LEN64, 1), %xmm0, %xmm2
|
||||
vmovdqu %xmm2, (DST, LEN64, 1)
|
||||
|
||||
// Make xmm3 contain [16-LEN,16-LEN+1,...,14,15,0x80,0x80,...].
|
||||
sub LEN64, %r9
|
||||
vmovdqu 32(%r9), %xmm3
|
||||
|
||||
// Shift the src partial block to the beginning of its register.
|
||||
vpshufb %xmm3, %xmm1, %xmm1
|
||||
|
||||
// Do a blend to generate the src partial block followed by the second
|
||||
// part of the en/decryption of the last full block.
|
||||
vpblendvb %xmm3, %xmm0, %xmm1, %xmm0
|
||||
.endif
|
||||
// En/decrypt again and store the last full block.
|
||||
_aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0, tmp=%xmm1
|
||||
vmovdqu %xmm0, (DST)
|
||||
dec NBLOCKS32
|
||||
jnz .Lblock_at_a_time\@
|
||||
jmp .Ldone\@
|
||||
.endm
|
||||
|
||||
// void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
|
||||
// void aes_xts_encrypt_iv(const struct aes_enckey *tweak_key,
|
||||
// u8 iv[AES_BLOCK_SIZE]);
|
||||
//
|
||||
// Encrypt |iv| using the AES key |tweak_key| to get the first tweak. Assumes
|
||||
// that the CPU supports AES-NI and AVX, but not necessarily VAES or AVX512.
|
||||
SYM_TYPED_FUNC_START(aes_xts_encrypt_iv)
|
||||
SYM_FUNC_START(aes_xts_encrypt_iv)
|
||||
.set TWEAK_KEY, %rdi
|
||||
.set IV, %rsi
|
||||
.set KEYLEN, %eax
|
||||
.set KEYLEN64, %rax
|
||||
|
||||
vmovdqu (IV), %xmm0
|
||||
vpxor (TWEAK_KEY), %xmm0, %xmm0
|
||||
movl 480(TWEAK_KEY), KEYLEN
|
||||
lea -16(TWEAK_KEY, KEYLEN64, 4), TWEAK_KEY
|
||||
vpxor OFFSETOF_RNDKEYS(TWEAK_KEY), %xmm0, %xmm0
|
||||
movl OFFSETOF_KEYLEN(TWEAK_KEY), KEYLEN
|
||||
lea OFFSETOF_RNDKEYS-16(TWEAK_KEY, KEYLEN64, 4), TWEAK_KEY
|
||||
cmp $24, KEYLEN
|
||||
jl .Lencrypt_iv_aes128
|
||||
je .Lencrypt_iv_aes192
|
||||
|
|
@ -867,39 +782,37 @@ SYM_FUNC_END(aes_xts_encrypt_iv)
|
|||
// Below are the actual AES-XTS encryption and decryption functions,
|
||||
// instantiated from the above macro. They all have the following prototype:
|
||||
//
|
||||
// void (*xts_crypt_func)(const struct crypto_aes_ctx *key,
|
||||
// const u8 *src, u8 *dst, int len,
|
||||
// u8 tweak[AES_BLOCK_SIZE]);
|
||||
// void aes_xts_{en,de}crypt_{cpufeature}(u8 *dst, const u8 *src, long nblocks,
|
||||
// u8 tweak[AES_BLOCK_SIZE],
|
||||
// const struct aes_key *key);
|
||||
//
|
||||
// |key| is the data key. |tweak| contains the next tweak; the encryption of
|
||||
// the original IV with the tweak key was already done. This function supports
|
||||
// incremental computation, but |len| must always be >= 16 (AES_BLOCK_SIZE), and
|
||||
// |len| must be a multiple of 16 except on the last call. If |len| is a
|
||||
// multiple of 16, then this function updates |tweak| to contain the next tweak.
|
||||
// `tweak` must have already been encrypted by the tweak key; `key` is just the
|
||||
// main key. `tweak` is updated to contain the next tweak, allowing for
|
||||
// incremental computation.
|
||||
|
||||
.set VL, 16
|
||||
.set USE_AVX512, 0
|
||||
SYM_TYPED_FUNC_START(aes_xts_encrypt_aesni_avx)
|
||||
SYM_FUNC_START(aes_xts_encrypt_aesni_avx)
|
||||
_aes_xts_crypt 1
|
||||
SYM_FUNC_END(aes_xts_encrypt_aesni_avx)
|
||||
SYM_TYPED_FUNC_START(aes_xts_decrypt_aesni_avx)
|
||||
SYM_FUNC_START(aes_xts_decrypt_aesni_avx)
|
||||
_aes_xts_crypt 0
|
||||
SYM_FUNC_END(aes_xts_decrypt_aesni_avx)
|
||||
|
||||
.set VL, 32
|
||||
.set USE_AVX512, 0
|
||||
SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx2)
|
||||
SYM_FUNC_START(aes_xts_encrypt_vaes_avx2)
|
||||
_aes_xts_crypt 1
|
||||
SYM_FUNC_END(aes_xts_encrypt_vaes_avx2)
|
||||
SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx2)
|
||||
SYM_FUNC_START(aes_xts_decrypt_vaes_avx2)
|
||||
_aes_xts_crypt 0
|
||||
SYM_FUNC_END(aes_xts_decrypt_vaes_avx2)
|
||||
|
||||
.set VL, 64
|
||||
.set USE_AVX512, 1
|
||||
SYM_TYPED_FUNC_START(aes_xts_encrypt_vaes_avx512)
|
||||
SYM_FUNC_START(aes_xts_encrypt_vaes_avx512)
|
||||
_aes_xts_crypt 1
|
||||
SYM_FUNC_END(aes_xts_encrypt_vaes_avx512)
|
||||
SYM_TYPED_FUNC_START(aes_xts_decrypt_vaes_avx512)
|
||||
SYM_FUNC_START(aes_xts_decrypt_vaes_avx512)
|
||||
_aes_xts_crypt 0
|
||||
SYM_FUNC_END(aes_xts_decrypt_vaes_avx512)
|
||||
|
|
@ -16,6 +16,7 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vaes_avx512);
|
|||
static_assert(offsetof(struct aes_enckey, len) == 0);
|
||||
static_assert(offsetof(struct aes_enckey, nrounds) == 4);
|
||||
static_assert(offsetof(struct aes_enckey, k.rndkeys) == 16);
|
||||
static_assert(offsetof(struct aes_key, len) == 0);
|
||||
static_assert(offsetof(struct aes_key, nrounds) == 4);
|
||||
static_assert(offsetof(struct aes_key, k.rndkeys) == 16);
|
||||
static_assert(offsetof(struct aes_key, inv_k.inv_rndkeys) == 256);
|
||||
|
|
@ -312,6 +313,26 @@ void aes_xts_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
|
|||
u8 tweak[AES_BLOCK_SIZE], const struct aes_key *key);
|
||||
void aes_xts_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
|
||||
u8 tweak[AES_BLOCK_SIZE], const struct aes_key *key);
|
||||
void aes_xts_encrypt_iv(const struct aes_enckey *tweak_key,
|
||||
u8 iv[AES_BLOCK_SIZE]);
|
||||
void aes_xts_encrypt_aesni_avx(u8 *dst, const u8 *src, long nblocks,
|
||||
u8 tweak[AES_BLOCK_SIZE],
|
||||
const struct aes_key *key);
|
||||
void aes_xts_decrypt_aesni_avx(u8 *dst, const u8 *src, long nblocks,
|
||||
u8 tweak[AES_BLOCK_SIZE],
|
||||
const struct aes_key *key);
|
||||
void aes_xts_encrypt_vaes_avx2(u8 *dst, const u8 *src, long nblocks,
|
||||
u8 tweak[AES_BLOCK_SIZE],
|
||||
const struct aes_key *key);
|
||||
void aes_xts_decrypt_vaes_avx2(u8 *dst, const u8 *src, long nblocks,
|
||||
u8 tweak[AES_BLOCK_SIZE],
|
||||
const struct aes_key *key);
|
||||
void aes_xts_encrypt_vaes_avx512(u8 *dst, const u8 *src, long nblocks,
|
||||
u8 tweak[AES_BLOCK_SIZE],
|
||||
const struct aes_key *key);
|
||||
void aes_xts_decrypt_vaes_avx512(u8 *dst, const u8 *src, long nblocks,
|
||||
u8 tweak[AES_BLOCK_SIZE],
|
||||
const struct aes_key *key);
|
||||
|
||||
/* len is always a positive multiple of AES_BLOCK_SIZE here. */
|
||||
static __always_inline bool
|
||||
|
|
@ -324,12 +345,46 @@ aes_xts_crypt_x86(u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE],
|
|||
return false;
|
||||
|
||||
kernel_fpu_begin();
|
||||
if (!cont)
|
||||
aes_encrypt_aesni(tweak, tweak, &key->tweak_key);
|
||||
if (enc)
|
||||
aes_xts_encrypt_aesni(dst, src, nblocks, tweak, &key->main_key);
|
||||
else
|
||||
aes_xts_decrypt_aesni(dst, src, nblocks, tweak, &key->main_key);
|
||||
if (IS_ENABLED(CONFIG_X86_64) &&
|
||||
static_branch_likely(&have_vaes_avx512)) {
|
||||
if (!cont)
|
||||
aes_xts_encrypt_iv(&key->tweak_key, tweak);
|
||||
if (enc)
|
||||
aes_xts_encrypt_vaes_avx512(dst, src, nblocks, tweak,
|
||||
&key->main_key);
|
||||
else
|
||||
aes_xts_decrypt_vaes_avx512(dst, src, nblocks, tweak,
|
||||
&key->main_key);
|
||||
} else if (IS_ENABLED(CONFIG_X86_64) &&
|
||||
static_branch_likely(&have_vaes_avx2)) {
|
||||
if (!cont)
|
||||
aes_xts_encrypt_iv(&key->tweak_key, tweak);
|
||||
if (enc)
|
||||
aes_xts_encrypt_vaes_avx2(dst, src, nblocks, tweak,
|
||||
&key->main_key);
|
||||
else
|
||||
aes_xts_decrypt_vaes_avx2(dst, src, nblocks, tweak,
|
||||
&key->main_key);
|
||||
} else if (IS_ENABLED(CONFIG_X86_64) &&
|
||||
static_branch_likely(&have_aesni_avx)) {
|
||||
if (!cont)
|
||||
aes_xts_encrypt_iv(&key->tweak_key, tweak);
|
||||
if (enc)
|
||||
aes_xts_encrypt_aesni_avx(dst, src, nblocks, tweak,
|
||||
&key->main_key);
|
||||
else
|
||||
aes_xts_decrypt_aesni_avx(dst, src, nblocks, tweak,
|
||||
&key->main_key);
|
||||
} else {
|
||||
if (!cont)
|
||||
aes_encrypt_aesni(tweak, tweak, &key->tweak_key);
|
||||
if (enc)
|
||||
aes_xts_encrypt_aesni(dst, src, nblocks, tweak,
|
||||
&key->main_key);
|
||||
else
|
||||
aes_xts_decrypt_aesni(dst, src, nblocks, tweak,
|
||||
&key->main_key);
|
||||
}
|
||||
kernel_fpu_end();
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user