github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-12 16:18:45 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

lib/crypto: arm64/sm3: Migrate optimized code into library

Browse Source 
Instead of exposing the arm64-optimized SM3 code via arm64-specific
crypto_shash algorithms, instead just implement the sm3_blocks() library
function.  This is much simpler, it makes the SM3 library functions be
arm64-optimized, and it fixes the longstanding issue where the
arm64-optimized SM3 code was disabled by default.  SM3 still remains
available through crypto_shash, but individual architectures no longer
need to handle it.

Tweak the SM3 assembly function prototypes to match what the library
expects, including changing the block count from 'int' to 'size_t'.
sm3_ce_transform() had to be updated to access 'x2' instead of 'w2',
while sm3_neon_transform() already used 'x2'.

Remove the CFI stubs which are no longer needed because the SM3 assembly
functions are no longer ever indirectly called.

Remove the dependency on KERNEL_MODE_NEON.  It was unnecessary, because
KERNEL_MODE_NEON is always enabled on arm64.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260321040935.410034-8-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Eric Biggers 2026-03-20 21:09:30 -07:00
parent ed065bd06e
commit 9f69f52b46
10 changed files with 62 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/arm64/configs/defconfig
View File

@ -1916,9 +1916,9 @@ CONFIG_CRYPTO_BENCHMARK=m
CONFIG_CRYPTO_ECHAINIV=y
CONFIG_CRYPTO_MICHAEL_MIC=m
CONFIG_CRYPTO_SHA3=m
CONFIG_CRYPTO_SM3=m
CONFIG_CRYPTO_USER_API_RNG=m
CONFIG_CRYPTO_GHASH_ARM64_CE=y
CONFIG_CRYPTO_SM3_ARM64_CE=m
CONFIG_CRYPTO_AES_ARM64_CE_BLK=y
CONFIG_CRYPTO_AES_ARM64_BS=m
CONFIG_CRYPTO_AES_ARM64_CE_CCM=y

22
arch/arm64/crypto/Kconfig
View File

@ -14,28 +14,6 @@ config CRYPTO_GHASH_ARM64_CE
Architecture: arm64 using:
- ARMv8 Crypto Extensions
config CRYPTO_SM3_NEON
tristate "Hash functions: SM3 (NEON)"
depends on KERNEL_MODE_NEON
select CRYPTO_HASH
select CRYPTO_LIB_SM3
help
SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012)
Architecture: arm64 using:
- NEON (Advanced SIMD) extensions
config CRYPTO_SM3_ARM64_CE
tristate "Hash functions: SM3 (ARMv8.2 Crypto Extensions)"
depends on KERNEL_MODE_NEON
select CRYPTO_HASH
select CRYPTO_LIB_SM3
help
SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012)
Architecture: arm64 using:
- ARMv8.2 Crypto Extensions
config CRYPTO_AES_ARM64_CE_BLK
tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (ARMv8 Crypto Extensions)"
depends on KERNEL_MODE_NEON

6
arch/arm64/crypto/Makefile
View File

@ -5,12 +5,6 @@
# Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
#
obj-$(CONFIG_CRYPTO_SM3_NEON) += sm3-neon.o
sm3-neon-y := sm3-neon-glue.o sm3-neon-core.o
obj-$(CONFIG_CRYPTO_SM3_ARM64_CE) += sm3-ce.o
sm3-ce-y := sm3-ce-glue.o sm3-ce-core.o
obj-$(CONFIG_CRYPTO_SM4_ARM64_CE) += sm4-ce-cipher.o
sm4-ce-cipher-y := sm4-ce-cipher-glue.o sm4-ce-cipher-core.o

70
arch/arm64/crypto/sm3-ce-glue.c
View File

@ -1,70 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* sm3-ce-glue.c - SM3 secure hash using ARMv8.2 Crypto Extensions
*
* Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
#include <crypto/internal/hash.h>
#include <crypto/sm3.h>
#include <crypto/sm3_base.h>
#include <linux/cpufeature.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <asm/simd.h>
MODULE_DESCRIPTION("SM3 secure hash using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src,
int blocks);
static int sm3_ce_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
int remain;
scoped_ksimd() {
remain = sm3_base_do_update_blocks(desc, data, len, sm3_ce_transform);
}
return remain;
}
static int sm3_ce_finup(struct shash_desc *desc, const u8 *data,
unsigned int len, u8 *out)
{
scoped_ksimd() {
sm3_base_do_finup(desc, data, len, sm3_ce_transform);
}
return sm3_base_finish(desc, out);
}
static struct shash_alg sm3_alg = {
.digestsize = SM3_DIGEST_SIZE,
.init = sm3_base_init,
.update = sm3_ce_update,
.finup = sm3_ce_finup,
.descsize = SM3_STATE_SIZE,
.base.cra_name = "sm3",
.base.cra_driver_name = "sm3-ce",
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
CRYPTO_AHASH_ALG_FINUP_MAX,
.base.cra_blocksize = SM3_BLOCK_SIZE,
.base.cra_module = THIS_MODULE,
.base.cra_priority = 400,
};
static int __init sm3_ce_mod_init(void)
{
return crypto_register_shash(&sm3_alg);
}
static void __exit sm3_ce_mod_fini(void)
{
crypto_unregister_shash(&sm3_alg);
}
module_cpu_feature_match(SM3, sm3_ce_mod_init);
module_exit(sm3_ce_mod_fini);

67
arch/arm64/crypto/sm3-neon-glue.c
View File

@ -1,67 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* sm3-neon-glue.c - SM3 secure hash using NEON instructions
*
* Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
#include <asm/simd.h>
#include <crypto/internal/hash.h>
#include <crypto/sm3.h>
#include <crypto/sm3_base.h>
#include <linux/cpufeature.h>
#include <linux/kernel.h>
#include <linux/module.h>
asmlinkage void sm3_neon_transform(struct sm3_state *sst, u8 const *src,
int blocks);
static int sm3_neon_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
scoped_ksimd()
return sm3_base_do_update_blocks(desc, data, len,
sm3_neon_transform);
}
static int sm3_neon_finup(struct shash_desc *desc, const u8 *data,
unsigned int len, u8 *out)
{
scoped_ksimd()
sm3_base_do_finup(desc, data, len, sm3_neon_transform);
return sm3_base_finish(desc, out);
}
static struct shash_alg sm3_alg = {
.digestsize = SM3_DIGEST_SIZE,
.init = sm3_base_init,
.update = sm3_neon_update,
.finup = sm3_neon_finup,
.descsize = SM3_STATE_SIZE,
.base.cra_name = "sm3",
.base.cra_driver_name = "sm3-neon",
.base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
CRYPTO_AHASH_ALG_FINUP_MAX,
.base.cra_blocksize = SM3_BLOCK_SIZE,
.base.cra_module = THIS_MODULE,
.base.cra_priority = 200,
};
static int __init sm3_neon_init(void)
{
return crypto_register_shash(&sm3_alg);
}
static void __exit sm3_neon_fini(void)
{
crypto_unregister_shash(&sm3_alg);
}
module_init(sm3_neon_init);
module_exit(sm3_neon_fini);
MODULE_DESCRIPTION("SM3 secure hash using NEON instructions");
MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>");
MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
MODULE_LICENSE("GPL v2");

1
lib/crypto/Kconfig
View File

@ -279,6 +279,7 @@ config CRYPTO_LIB_SM3
config CRYPTO_LIB_SM3_ARCH
bool
depends on CRYPTO_LIB_SM3 && !UML
default y if ARM64
source "lib/crypto/tests/Kconfig"

13
lib/crypto/Makefile
View File

@ -368,13 +368,20 @@ endif # CONFIG_CRYPTO_LIB_SHA3_ARCH
################################################################################
obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o
libsm3-y := sm3.o
ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y)
CFLAGS_sm3.o += -I$(src)/$(SRCARCH)
libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \
arm64/sm3-neon-core.o
endif # CONFIG_CRYPTO_LIB_SM3_ARCH
################################################################################
obj-$(CONFIG_MPILIB) += mpi/
obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o
obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o
libsm3-y := sm3.o
# clean-files must be defined unconditionally
clean-files += arm/sha256-core.S arm/sha512-core.S
clean-files += arm64/sha256-core.S arm64/sha512-core.S

11
arch/arm64/crypto/sm3-ce-core.S → lib/crypto/arm64/sm3-ce-core.S
View File

@ -6,7 +6,6 @@
*/
#include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
@ -70,11 +69,11 @@
.endm
/*
* void sm3_ce_transform(struct sm3_state *sst, u8 const *src,
* int blocks)
* void sm3_ce_transform(struct sm3_block_state *state,
* const u8 *data, size_t nblocks)
*/
.text
SYM_TYPED_FUNC_START(sm3_ce_transform)
SYM_FUNC_START(sm3_ce_transform)
/* load state */
ld1 {v8.4s-v9.4s}, [x0]
rev64 v8.4s, v8.4s
@ -87,7 +86,7 @@ SYM_TYPED_FUNC_START(sm3_ce_transform)
/* load input */
0: ld1 {v0.16b-v3.16b}, [x1], #64
sub w2, w2, #1
sub x2, x2, #1
mov v15.16b, v8.16b
mov v16.16b, v9.16b
@ -123,7 +122,7 @@ CPU_LE( rev32 v3.16b, v3.16b )
eor v9.16b, v9.16b, v16.16b
/* handled all input blocks? */
cbnz w2, 0b
cbnz x2, 0b
/* save state */
rev64 v8.4s, v8.4s

9
arch/arm64/crypto/sm3-neon-core.S → lib/crypto/arm64/sm3-neon-core.S
View File

@ -9,7 +9,6 @@
*/
#include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
/* Context structure */
@ -345,14 +344,14 @@
/*
* Transform blocks*64 bytes (blocks*16 32-bit words) at 'src'.
* Transform nblocks*64 bytes (nblocks*16 32-bit words) at 'data'.
*
* void sm3_neon_transform(struct sm3_state *sst, u8 const *src,
* int blocks)
* void sm3_neon_transform(struct sm3_block_state *state,
* const u8 *data, size_t nblocks)
*/
.text
.align 3
SYM_TYPED_FUNC_START(sm3_neon_transform)
SYM_FUNC_START(sm3_neon_transform)
ldp ra, rb, [RSTATE, #0]
ldp rc, rd, [RSTATE, #8]
ldp re, rf, [RSTATE, #16]

41
lib/crypto/arm64/sm3.h Normal file
View File

@ -0,0 +1,41 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM3 optimized for ARM64
*
* Copyright 2026 Google LLC
*/
#include <asm/simd.h>
#include <linux/cpufeature.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
asmlinkage void sm3_neon_transform(struct sm3_block_state *state,
const u8 *data, size_t nblocks);
asmlinkage void sm3_ce_transform(struct sm3_block_state *state,
const u8 *data, size_t nblocks);
static void sm3_blocks(struct sm3_block_state *state,
const u8 *data, size_t nblocks)
{
if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
scoped_ksimd() {
if (static_branch_likely(&have_ce))
sm3_ce_transform(state, data, nblocks);
else
sm3_neon_transform(state, data, nblocks);
}
} else {
sm3_blocks_generic(state, data, nblocks);
}
}
#define sm3_mod_init_arch sm3_mod_init_arch
static void sm3_mod_init_arch(void)
{
if (cpu_have_named_feature(ASIMD)) {
static_branch_enable(&have_neon);
if (cpu_have_named_feature(SM3))
static_branch_enable(&have_ce);
}
}