diff --git a/lib/crypto/x86/aes-aesni.S b/lib/crypto/x86/aes-aesni.S index 51c8522416d7..297fe21ba830 100644 --- a/lib/crypto/x86/aes-aesni.S +++ b/lib/crypto/x86/aes-aesni.S @@ -4,7 +4,8 @@ // // Copyright 2026 Google LLC // -// The code in this file supports 32-bit and 64-bit CPUs, and it doesn't require +// The code in this file supports 32-bit and 64-bit CPUs (except for +// aes_ctr64_crypt_aesni() which supports 64-bit only), and it doesn't require // AVX. It does use up to SSE4.1, which all CPUs with AES-NI have. #include @@ -48,6 +49,12 @@ .section .rodata .p2align 4 +#ifdef __x86_64__ +.Lbswap_mask: + // A mask for pshufb that byte-reflects the value. + .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +#endif + .Lexpandkey_mask: // A mask for pshufb that extracts the last dword, rotates it right by 8 // bits, and copies the result to all four dwords. @@ -634,3 +641,124 @@ SYM_FUNC_END(aes_cbc_cts_encrypt_aesni) SYM_FUNC_START(aes_cbc_cts_decrypt_aesni) _aes_cbc_cts_crypt 0 SYM_FUNC_END(aes_cbc_cts_decrypt_aesni) + +#ifdef __x86_64__ +// void aes_ctr64_crypt_aesni(u8 *dst, const u8 *src, s64 len, +// const u64 le_ctr[2], +// const struct aes_enckey *key); +SYM_FUNC_START(aes_ctr64_crypt_aesni) + // Arguments + .set DST, ARG0 + .set SRC, ARG1 + .set LEN, ARG2 + .set LEN32, ARG2_32 // Used for improved code density + .set LE_CTR_PTR, ARG3 // Used as temp reg after LE_CTR is loaded + .set KEY, ARG4 + + // Other local variables + .set AESDATA0, %xmm0 + .set AESDATA1, %xmm1 + .set AESDATA2, %xmm2 + .set AESDATA3, %xmm3 + .set LE_CTR, %xmm4 // Current 128-bit little endian counter + .set LE_CTR_INC, %xmm5 // Initialized to (u64[])[1, 0] + .set BSWAP_MASK, %xmm6 + .set RNDKEY, %xmm7 + .set RNDKEY_PTR, LE_CTR_PTR // Temporary register for _do_aes + .set NROUNDS, TMP_32 // Temporary register for _do_aes + + // Initialize LE_CTR, BSWAP_MASK, and LE_CTR_INC. + movdqu (LE_CTR_PTR), LE_CTR + movdqa RODATA(.Lbswap_mask), BSWAP_MASK + mov $1, TMP_32 + movd TMP_32, LE_CTR_INC + + // Encrypt and XOR four blocks (64 bytes) at a time. + sub $64, LEN + jl .Lctr_loop4_done +.p2align 5 +.Lctr_loop4: +.irp i, 0,1,2,3 + movdqa LE_CTR, AESDATA\i + pshufb BSWAP_MASK, AESDATA\i // => big endian counter + paddq LE_CTR_INC, LE_CTR +.endr + _do_aes 1, 0,1,2,3 + // AESDATA[0-3] now contain four keystream blocks. +.irp i, 0,1,2,3 + movdqu \i*16(SRC), RNDKEY // Use RNDKEY as temp register. + pxor RNDKEY, AESDATA\i + movdqu AESDATA\i, \i*16(DST) +.endr + add $64, SRC + add $64, DST + sub $64, LEN + jge .Lctr_loop4 +.Lctr_loop4_done: + add $64, LEN32 + jz .Lctr_done + + // 1 <= LEN <= 63 bytes remain. Prepare four more keystream blocks. +.irp i, 0,1,2,3 + movdqa LE_CTR, AESDATA\i + pshufb BSWAP_MASK, AESDATA\i // => big endian counter + .if \i != 3 + paddq LE_CTR_INC, LE_CTR + .endif +.endr + _do_aes 1, 0,1,2,3 + // AESDATA[0-3] now contain four keystream blocks. + + // XOR one block (16 bytes) at a time. + sub $16, LEN32 + jl .Lctr_xor_partial +.Lctr_xor1: + movdqu (SRC), RNDKEY // Use RNDKEY as temp register. + pxor RNDKEY, AESDATA0 + movdqu AESDATA0, (DST) + movdqa AESDATA1, AESDATA0 + movdqa AESDATA2, AESDATA1 + movdqa AESDATA3, AESDATA2 + add $16, SRC + add $16, DST + sub $16, LEN32 + jge .Lctr_xor1 + + // XOR the remaining LEN mod 16 bytes. +.Lctr_xor_partial: + test $8, LEN32 + jz 1f + movq AESDATA0, TMP + xor (SRC), TMP // XOR 8 bytes. + mov TMP, (DST) + add $8, SRC + add $8, DST + psrldq $8, AESDATA0 +1: + test $4, LEN32 + jz 2f + movd AESDATA0, TMP_32 + xor (SRC), TMP_32 // XOR 4 bytes. + mov TMP_32, (DST) + add $4, SRC + add $4, DST + psrldq $4, AESDATA0 +2: + test $2, LEN32 + jz 3f + movd AESDATA0, TMP_32 + xor (SRC), TMP_16 // XOR 2 bytes. + mov TMP_16, (DST) + add $2, SRC + add $2, DST + psrldq $2, AESDATA0 +3: + test $1, LEN32 + jz .Lctr_done + movd AESDATA0, TMP_32 + xor (SRC), TMP_8 // XOR 1 byte. + mov TMP_8, (DST) +.Lctr_done: + RET +SYM_FUNC_END(aes_ctr64_crypt_aesni) +#endif // __x86_64__ diff --git a/lib/crypto/x86/aes.h b/lib/crypto/x86/aes.h index a0346cb3ecb5..2a2b26d10e87 100644 --- a/lib/crypto/x86/aes.h +++ b/lib/crypto/x86/aes.h @@ -205,6 +205,60 @@ static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, } #endif /* CONFIG_CRYPTO_LIB_AES_CBC */ +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) && IS_ENABLED(CONFIG_X86_64) +void aes_ctr64_crypt_aesni(u8 *dst, const u8 *src, s64 len, const u64 le_ctr[2], + const struct aes_enckey *key); + +static void aes_ctr64_x86(u8 *dst, const u8 *src, size_t len, + const u64 le_ctr[2], const struct aes_enckey *key) +{ + aes_ctr64_crypt_aesni(dst, src, len, le_ctr, key); +} + +#define aes_ctr_arch aes_ctr_arch +static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, + u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) +{ + u64 le_ctr[2]; + u64 ctr64; + size_t nblocks; + size_t part1_len; + + if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable())) + return false; + + ctr64 = le_ctr[0] = get_unaligned_be64(&ctr[8]); + le_ctr[1] = get_unaligned_be64(&ctr[0]); + + kernel_fpu_begin(); + + nblocks = DIV_ROUND_UP(len, AES_BLOCK_SIZE); + ctr64 += nblocks; + + if (likely(ctr64 >= nblocks)) { + /* The low 64 bits of the counter won't overflow. */ + aes_ctr64_x86(dst, src, len, le_ctr, key); + } else { + /* + * The low 64 bits of the counter will overflow. The + * assembly doesn't handle this case, so split the + * operation into two at the point where the overflow + * will occur. After the first part, add the carry bit. + */ + part1_len = min(len, (nblocks - ctr64) * AES_BLOCK_SIZE); + aes_ctr64_x86(dst, src, part1_len, le_ctr, key); + le_ctr[0] = 0; + le_ctr[1]++; + aes_ctr64_x86(dst + part1_len, src + part1_len, len - part1_len, + le_ctr, key); + } + kernel_fpu_end(); + put_unaligned_be64(ctr64, &ctr[8]); + put_unaligned_be64(le_ctr[1], &ctr[0]); + return true; +} +#endif /* CONFIG_CRYPTO_LIB_AES_CTR && CONFIG_X86_64 */ + #define aes_mod_init_arch aes_mod_init_arch static void aes_mod_init_arch(void) {