github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 09:36:22 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

lib/crc: Turn NEON intrinsics crc64 implementation into common code

Browse Source 
Move and rename the CRC64 NEON intrinsics implementation source file and
rename the function name to reflect that it is NEON code that can be
shared. This will be wired up for 32-bit ARM in a subsequent patch.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://patch.msgid.link/20260422171655.3437334-14-ardb+git@google.com
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Ard Biesheuvel 2026-04-22 19:17:00 +02:00 committed by Eric Biggers
parent 4d3c5cbfe2
commit 061cef5fce
4 changed files with 30 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/crc/Makefile
View File

@ -39,9 +39,9 @@ crc64-y := crc64-main.o
ifeq ($(CONFIG_CRC64_ARCH),y)
CFLAGS_crc64-main.o += -I$(src)/$(SRCARCH)
CFLAGS_REMOVE_arm64/crc64-neon-inner.o += $(CC_FLAGS_NO_FPU)
CFLAGS_arm64/crc64-neon-inner.o += $(CC_FLAGS_FPU) -march=armv8-a+crypto
crc64-$(CONFIG_ARM64) += arm64/crc64-neon-inner.o
CFLAGS_REMOVE_crc64-neon.o += $(CC_FLAGS_NO_FPU)
CFLAGS_crc64-neon.o += $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH) -march=armv8-a+crypto
crc64-$(CONFIG_ARM64) += crc64-neon.o
crc64-$(CONFIG_RISCV) += riscv/crc64_lsb.o riscv/crc64_msb.o
crc64-$(CONFIG_X86) += x86/crc64-pclmul.o

21
lib/crc/arm64/crc64-neon.h Normal file
View File

@ -0,0 +1,21 @@
// SPDX-License-Identifier: GPL-2.0-only
static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b)
{
return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 0),
vgetq_lane_u64(b, 0)));
}
static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b)
{
poly64x2_t l = vreinterpretq_p64_u64(a);
poly64x2_t m = vreinterpretq_p64_u64(b);
return vreinterpretq_u64_p128(vmull_high_p64(l, m));
}
static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b)
{
return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 1),
vgetq_lane_u64(b, 0)));
}

4
lib/crc/arm64/crc64.h
View File

@ -8,7 +8,7 @@
#include <linux/minmax.h>
#include <linux/sizes.h>
u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len);
u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
#define crc64_be_arch crc64_be_generic
@ -19,7 +19,7 @@ static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
size_t chunk = len & ~15;
scoped_ksimd()
crc = crc64_nvme_arm64_c(crc, p, chunk);
crc = crc64_nvme_neon(crc, p, chunk);
p += chunk;
len &= 15;

26
lib/crc/arm64/crc64-neon-inner.c → lib/crc/crc64-neon.c
View File

@ -6,7 +6,9 @@
#include <linux/types.h>
#include <asm/neon-intrinsics.h>
u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len);
#include "crc64-neon.h"
u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
/* x^191 mod G, x^127 mod G */
static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL,
@ -15,27 +17,7 @@ static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL,
static const u64 bconsts_val[2] = { 0x27ecfa329aef9f77ULL,
0x34d926535897936aULL };
static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b)
{
return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 0),
vgetq_lane_u64(b, 0)));
}
static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b)
{
poly64x2_t l = vreinterpretq_p64_u64(a);
poly64x2_t m = vreinterpretq_p64_u64(b);
return vreinterpretq_u64_p128(vmull_high_p64(l, m));
}
static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b)
{
return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 1),
vgetq_lane_u64(b, 0)));
}
u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len)
u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
{
uint64x2_t fold_consts = vld1q_u64(fold_consts_val);
uint64x2_t v0 = { crc, 0 };