diff --git a/lib/crc/Makefile b/lib/crc/Makefile index ff213590e4e3..193257ae466f 100644 --- a/lib/crc/Makefile +++ b/lib/crc/Makefile @@ -39,9 +39,9 @@ crc64-y := crc64-main.o ifeq ($(CONFIG_CRC64_ARCH),y) CFLAGS_crc64-main.o += -I$(src)/$(SRCARCH) -CFLAGS_REMOVE_arm64/crc64-neon-inner.o += $(CC_FLAGS_NO_FPU) -CFLAGS_arm64/crc64-neon-inner.o += $(CC_FLAGS_FPU) -march=armv8-a+crypto -crc64-$(CONFIG_ARM64) += arm64/crc64-neon-inner.o +CFLAGS_REMOVE_crc64-neon.o += $(CC_FLAGS_NO_FPU) +CFLAGS_crc64-neon.o += $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH) -march=armv8-a+crypto +crc64-$(CONFIG_ARM64) += crc64-neon.o crc64-$(CONFIG_RISCV) += riscv/crc64_lsb.o riscv/crc64_msb.o crc64-$(CONFIG_X86) += x86/crc64-pclmul.o diff --git a/lib/crc/arm64/crc64-neon.h b/lib/crc/arm64/crc64-neon.h new file mode 100644 index 000000000000..fcd5b1e6f812 --- /dev/null +++ b/lib/crc/arm64/crc64-neon.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only + +static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) +{ + return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 0), + vgetq_lane_u64(b, 0))); +} + +static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) +{ + poly64x2_t l = vreinterpretq_p64_u64(a); + poly64x2_t m = vreinterpretq_p64_u64(b); + + return vreinterpretq_u64_p128(vmull_high_p64(l, m)); +} + +static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) +{ + return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 1), + vgetq_lane_u64(b, 0))); +} diff --git a/lib/crc/arm64/crc64.h b/lib/crc/arm64/crc64.h index 60151ec3035a..c7a69e1f3d8f 100644 --- a/lib/crc/arm64/crc64.h +++ b/lib/crc/arm64/crc64.h @@ -8,7 +8,7 @@ #include #include -u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len); +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); #define crc64_be_arch crc64_be_generic @@ -19,7 +19,7 @@ static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) size_t chunk = len & ~15; scoped_ksimd() - crc = crc64_nvme_arm64_c(crc, p, chunk); + crc = crc64_nvme_neon(crc, p, chunk); p += chunk; len &= 15; diff --git a/lib/crc/arm64/crc64-neon-inner.c b/lib/crc/crc64-neon.c similarity index 62% rename from lib/crc/arm64/crc64-neon-inner.c rename to lib/crc/crc64-neon.c index 28527e544ff6..4753fb94a4be 100644 --- a/lib/crc/arm64/crc64-neon-inner.c +++ b/lib/crc/crc64-neon.c @@ -6,7 +6,9 @@ #include #include -u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len); +#include "crc64-neon.h" + +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); /* x^191 mod G, x^127 mod G */ static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL, @@ -15,27 +17,7 @@ static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL, static const u64 bconsts_val[2] = { 0x27ecfa329aef9f77ULL, 0x34d926535897936aULL }; -static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) -{ - return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 0), - vgetq_lane_u64(b, 0))); -} - -static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) -{ - poly64x2_t l = vreinterpretq_p64_u64(a); - poly64x2_t m = vreinterpretq_p64_u64(b); - - return vreinterpretq_u64_p128(vmull_high_p64(l, m)); -} - -static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) -{ - return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 1), - vgetq_lane_u64(b, 0))); -} - -u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len) +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len) { uint64x2_t fold_consts = vld1q_u64(fold_consts_val); uint64x2_t v0 = { crc, 0 };