diff --git a/lib/crc/Kconfig b/lib/crc/Kconfig index f47bb4c706fb..927fc6a6b2b9 100644 --- a/lib/crc/Kconfig +++ b/lib/crc/Kconfig @@ -82,6 +82,7 @@ config CRC64 config CRC64_ARCH bool depends on CRC64 && CRC_OPTIMIZATIONS + default y if ARM && KERNEL_MODE_NEON && !CPU_BIG_ENDIAN default y if ARM64 default y if RISCV && RISCV_ISA_ZBC && 64BIT default y if X86_64 diff --git a/lib/crc/Makefile b/lib/crc/Makefile index 193257ae466f..386e9c175263 100644 --- a/lib/crc/Makefile +++ b/lib/crc/Makefile @@ -39,8 +39,11 @@ crc64-y := crc64-main.o ifeq ($(CONFIG_CRC64_ARCH),y) CFLAGS_crc64-main.o += -I$(src)/$(SRCARCH) +crc64-cflags-$(CONFIG_ARM) += -march=armv8-a -mfpu=crypto-neon-fp-armv8 +crc64-cflags-$(CONFIG_ARM64) += -march=armv8-a+crypto CFLAGS_REMOVE_crc64-neon.o += $(CC_FLAGS_NO_FPU) -CFLAGS_crc64-neon.o += $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH) -march=armv8-a+crypto +CFLAGS_crc64-neon.o += $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH) $(crc64-cflags-y) +crc64-$(CONFIG_ARM) += crc64-neon.o crc64-$(CONFIG_ARM64) += crc64-neon.o crc64-$(CONFIG_RISCV) += riscv/crc64_lsb.o riscv/crc64_msb.o diff --git a/lib/crc/arm/crc64-neon.h b/lib/crc/arm/crc64-neon.h new file mode 100644 index 000000000000..645f553220ff --- /dev/null +++ b/lib/crc/arm/crc64-neon.h @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only + +static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 0); + uint64_t m = vgetq_lane_u64(b, 0); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} + +static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 1); + uint64_t m = vgetq_lane_u64(b, 1); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} + +static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 1); + uint64_t m = vgetq_lane_u64(b, 0); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} diff --git a/lib/crc/arm/crc64.h b/lib/crc/arm/crc64.h new file mode 100644 index 000000000000..de274288af61 --- /dev/null +++ b/lib/crc/arm/crc64.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * CRC64 using ARM PMULL instructions + */ + +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); + +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); + +#define crc64_be_arch crc64_be_generic + +static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) +{ + if (len >= 128 && static_branch_likely(&have_pmull) && + likely(may_use_simd())) { + do { + size_t chunk = min_t(size_t, len & ~15, SZ_4K); + + scoped_ksimd() + crc = crc64_nvme_neon(crc, p, chunk); + + p += chunk; + len -= chunk; + } while (len >= 128); + } + return crc64_nvme_generic(crc, p, len); +} + +#define crc64_mod_init_arch crc64_mod_init_arch +static void crc64_mod_init_arch(void) +{ + if (elf_hwcap2 & HWCAP2_PMULL) + static_branch_enable(&have_pmull); +}