mirror of
https://github.com/torvalds/linux.git
synced 2026-05-12 16:18:45 +02:00
arm: move the XOR code to lib/raid/
Move the optimized XOR into lib/raid and include it it in the main xor.ko instead of building a separate module for it. Link: https://lkml.kernel.org/r/20260327061704.3707577-13-hch@lst.de Signed-off-by: Christoph Hellwig <hch@lst.de> Reviewed-by: Eric Biggers <ebiggers@kernel.org> Tested-by: Eric Biggers <ebiggers@kernel.org> Cc: Albert Ou <aou@eecs.berkeley.edu> Cc: Alexander Gordeev <agordeev@linux.ibm.com> Cc: Alexandre Ghiti <alex@ghiti.fr> Cc: Andreas Larsson <andreas@gaisler.com> Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com> Cc: Ard Biesheuvel <ardb@kernel.org> Cc: Arnd Bergmann <arnd@arndb.de> Cc: "Borislav Petkov (AMD)" <bp@alien8.de> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Chris Mason <clm@fb.com> Cc: Christian Borntraeger <borntraeger@linux.ibm.com> Cc: Dan Williams <dan.j.williams@intel.com> Cc: David S. Miller <davem@davemloft.net> Cc: David Sterba <dsterba@suse.com> Cc: Heiko Carstens <hca@linux.ibm.com> Cc: Herbert Xu <herbert@gondor.apana.org.au> Cc: "H. Peter Anvin" <hpa@zytor.com> Cc: Huacai Chen <chenhuacai@kernel.org> Cc: Ingo Molnar <mingo@redhat.com> Cc: Jason A. Donenfeld <jason@zx2c4.com> Cc: Johannes Berg <johannes@sipsolutions.net> Cc: Li Nan <linan122@huawei.com> Cc: Madhavan Srinivasan <maddy@linux.ibm.com> Cc: Magnus Lindholm <linmag7@gmail.com> Cc: Matt Turner <mattst88@gmail.com> Cc: Michael Ellerman <mpe@ellerman.id.au> Cc: Nicholas Piggin <npiggin@gmail.com> Cc: Palmer Dabbelt <palmer@dabbelt.com> Cc: Richard Henderson <richard.henderson@linaro.org> Cc: Richard Weinberger <richard@nod.at> Cc: Russell King <linux@armlinux.org.uk> Cc: Song Liu <song@kernel.org> Cc: Sven Schnelle <svens@linux.ibm.com> Cc: Ted Ts'o <tytso@mit.edu> Cc: Vasily Gorbik <gor@linux.ibm.com> Cc: WANG Xuerui <kernel@xen0n.name> Cc: Will Deacon <will@kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Christoph Hellwig
Andrew Morton
parent
503793b134
commit
0d64a24ec0
|
|
@ -1,198 +1,12 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* arch/arm/include/asm/xor.h
|
||||
*
|
||||
* Copyright (C) 2001 Russell King
|
||||
*/
|
||||
#include <asm-generic/xor.h>
|
||||
#include <asm/hwcap.h>
|
||||
#include <asm/neon.h>
|
||||
|
||||
#define __XOR(a1, a2) a1 ^= a2
|
||||
|
||||
#define GET_BLOCK_2(dst) \
|
||||
__asm__("ldmia %0, {%1, %2}" \
|
||||
: "=r" (dst), "=r" (a1), "=r" (a2) \
|
||||
: "0" (dst))
|
||||
|
||||
#define GET_BLOCK_4(dst) \
|
||||
__asm__("ldmia %0, {%1, %2, %3, %4}" \
|
||||
: "=r" (dst), "=r" (a1), "=r" (a2), "=r" (a3), "=r" (a4) \
|
||||
: "0" (dst))
|
||||
|
||||
#define XOR_BLOCK_2(src) \
|
||||
__asm__("ldmia %0!, {%1, %2}" \
|
||||
: "=r" (src), "=r" (b1), "=r" (b2) \
|
||||
: "0" (src)); \
|
||||
__XOR(a1, b1); __XOR(a2, b2);
|
||||
|
||||
#define XOR_BLOCK_4(src) \
|
||||
__asm__("ldmia %0!, {%1, %2, %3, %4}" \
|
||||
: "=r" (src), "=r" (b1), "=r" (b2), "=r" (b3), "=r" (b4) \
|
||||
: "0" (src)); \
|
||||
__XOR(a1, b1); __XOR(a2, b2); __XOR(a3, b3); __XOR(a4, b4)
|
||||
|
||||
#define PUT_BLOCK_2(dst) \
|
||||
__asm__ __volatile__("stmia %0!, {%2, %3}" \
|
||||
: "=r" (dst) \
|
||||
: "0" (dst), "r" (a1), "r" (a2))
|
||||
|
||||
#define PUT_BLOCK_4(dst) \
|
||||
__asm__ __volatile__("stmia %0!, {%2, %3, %4, %5}" \
|
||||
: "=r" (dst) \
|
||||
: "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4))
|
||||
|
||||
static void
|
||||
xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2)
|
||||
{
|
||||
unsigned int lines = bytes / sizeof(unsigned long) / 4;
|
||||
register unsigned int a1 __asm__("r4");
|
||||
register unsigned int a2 __asm__("r5");
|
||||
register unsigned int a3 __asm__("r6");
|
||||
register unsigned int a4 __asm__("r10");
|
||||
register unsigned int b1 __asm__("r8");
|
||||
register unsigned int b2 __asm__("r9");
|
||||
register unsigned int b3 __asm__("ip");
|
||||
register unsigned int b4 __asm__("lr");
|
||||
|
||||
do {
|
||||
GET_BLOCK_4(p1);
|
||||
XOR_BLOCK_4(p2);
|
||||
PUT_BLOCK_4(p1);
|
||||
} while (--lines);
|
||||
}
|
||||
|
||||
static void
|
||||
xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3)
|
||||
{
|
||||
unsigned int lines = bytes / sizeof(unsigned long) / 4;
|
||||
register unsigned int a1 __asm__("r4");
|
||||
register unsigned int a2 __asm__("r5");
|
||||
register unsigned int a3 __asm__("r6");
|
||||
register unsigned int a4 __asm__("r10");
|
||||
register unsigned int b1 __asm__("r8");
|
||||
register unsigned int b2 __asm__("r9");
|
||||
register unsigned int b3 __asm__("ip");
|
||||
register unsigned int b4 __asm__("lr");
|
||||
|
||||
do {
|
||||
GET_BLOCK_4(p1);
|
||||
XOR_BLOCK_4(p2);
|
||||
XOR_BLOCK_4(p3);
|
||||
PUT_BLOCK_4(p1);
|
||||
} while (--lines);
|
||||
}
|
||||
|
||||
static void
|
||||
xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3,
|
||||
const unsigned long * __restrict p4)
|
||||
{
|
||||
unsigned int lines = bytes / sizeof(unsigned long) / 2;
|
||||
register unsigned int a1 __asm__("r8");
|
||||
register unsigned int a2 __asm__("r9");
|
||||
register unsigned int b1 __asm__("ip");
|
||||
register unsigned int b2 __asm__("lr");
|
||||
|
||||
do {
|
||||
GET_BLOCK_2(p1);
|
||||
XOR_BLOCK_2(p2);
|
||||
XOR_BLOCK_2(p3);
|
||||
XOR_BLOCK_2(p4);
|
||||
PUT_BLOCK_2(p1);
|
||||
} while (--lines);
|
||||
}
|
||||
|
||||
static void
|
||||
xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3,
|
||||
const unsigned long * __restrict p4,
|
||||
const unsigned long * __restrict p5)
|
||||
{
|
||||
unsigned int lines = bytes / sizeof(unsigned long) / 2;
|
||||
register unsigned int a1 __asm__("r8");
|
||||
register unsigned int a2 __asm__("r9");
|
||||
register unsigned int b1 __asm__("ip");
|
||||
register unsigned int b2 __asm__("lr");
|
||||
|
||||
do {
|
||||
GET_BLOCK_2(p1);
|
||||
XOR_BLOCK_2(p2);
|
||||
XOR_BLOCK_2(p3);
|
||||
XOR_BLOCK_2(p4);
|
||||
XOR_BLOCK_2(p5);
|
||||
PUT_BLOCK_2(p1);
|
||||
} while (--lines);
|
||||
}
|
||||
|
||||
static struct xor_block_template xor_block_arm4regs = {
|
||||
.name = "arm4regs",
|
||||
.do_2 = xor_arm4regs_2,
|
||||
.do_3 = xor_arm4regs_3,
|
||||
.do_4 = xor_arm4regs_4,
|
||||
.do_5 = xor_arm4regs_5,
|
||||
};
|
||||
|
||||
#ifdef CONFIG_KERNEL_MODE_NEON
|
||||
|
||||
extern struct xor_block_template const xor_block_neon_inner;
|
||||
|
||||
static void
|
||||
xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2)
|
||||
{
|
||||
kernel_neon_begin();
|
||||
xor_block_neon_inner.do_2(bytes, p1, p2);
|
||||
kernel_neon_end();
|
||||
}
|
||||
|
||||
static void
|
||||
xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3)
|
||||
{
|
||||
kernel_neon_begin();
|
||||
xor_block_neon_inner.do_3(bytes, p1, p2, p3);
|
||||
kernel_neon_end();
|
||||
}
|
||||
|
||||
static void
|
||||
xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3,
|
||||
const unsigned long * __restrict p4)
|
||||
{
|
||||
kernel_neon_begin();
|
||||
xor_block_neon_inner.do_4(bytes, p1, p2, p3, p4);
|
||||
kernel_neon_end();
|
||||
}
|
||||
|
||||
static void
|
||||
xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3,
|
||||
const unsigned long * __restrict p4,
|
||||
const unsigned long * __restrict p5)
|
||||
{
|
||||
kernel_neon_begin();
|
||||
xor_block_neon_inner.do_5(bytes, p1, p2, p3, p4, p5);
|
||||
kernel_neon_end();
|
||||
}
|
||||
|
||||
static struct xor_block_template xor_block_neon = {
|
||||
.name = "neon",
|
||||
.do_2 = xor_neon_2,
|
||||
.do_3 = xor_neon_3,
|
||||
.do_4 = xor_neon_4,
|
||||
.do_5 = xor_neon_5
|
||||
};
|
||||
|
||||
#endif /* CONFIG_KERNEL_MODE_NEON */
|
||||
extern struct xor_block_template xor_block_arm4regs;
|
||||
extern struct xor_block_template xor_block_neon;
|
||||
|
||||
#define arch_xor_init arch_xor_init
|
||||
static __always_inline void __init arch_xor_init(void)
|
||||
|
|
|
|||
|
|
@ -39,9 +39,4 @@ endif
|
|||
$(obj)/csumpartialcopy.o: $(obj)/csumpartialcopygeneric.S
|
||||
$(obj)/csumpartialcopyuser.o: $(obj)/csumpartialcopygeneric.S
|
||||
|
||||
ifeq ($(CONFIG_KERNEL_MODE_NEON),y)
|
||||
CFLAGS_xor-neon.o += $(CC_FLAGS_FPU)
|
||||
obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o
|
||||
endif
|
||||
|
||||
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
|
||||
|
|
|
|||
|
|
@ -9,3 +9,11 @@ xor-y += xor-8regs-prefetch.o
|
|||
xor-y += xor-32regs-prefetch.o
|
||||
|
||||
xor-$(CONFIG_ALPHA) += alpha/xor.o
|
||||
xor-$(CONFIG_ARM) += arm/xor.o
|
||||
ifeq ($(CONFIG_ARM),y)
|
||||
xor-$(CONFIG_KERNEL_MODE_NEON) += arm/xor-neon.o arm/xor-neon-glue.o
|
||||
endif
|
||||
|
||||
|
||||
CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU)
|
||||
CFLAGS_REMOVE_arm/xor-neon.o += $(CC_FLAGS_NO_FPU)
|
||||
|
|
|
|||
58
lib/raid/xor/arm/xor-neon-glue.c
Normal file
58
lib/raid/xor/arm/xor-neon-glue.c
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (C) 2001 Russell King
|
||||
*/
|
||||
#include <linux/raid/xor_impl.h>
|
||||
#include <asm/xor.h>
|
||||
|
||||
extern struct xor_block_template const xor_block_neon_inner;
|
||||
|
||||
static void
|
||||
xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2)
|
||||
{
|
||||
kernel_neon_begin();
|
||||
xor_block_neon_inner.do_2(bytes, p1, p2);
|
||||
kernel_neon_end();
|
||||
}
|
||||
|
||||
static void
|
||||
xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3)
|
||||
{
|
||||
kernel_neon_begin();
|
||||
xor_block_neon_inner.do_3(bytes, p1, p2, p3);
|
||||
kernel_neon_end();
|
||||
}
|
||||
|
||||
static void
|
||||
xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3,
|
||||
const unsigned long * __restrict p4)
|
||||
{
|
||||
kernel_neon_begin();
|
||||
xor_block_neon_inner.do_4(bytes, p1, p2, p3, p4);
|
||||
kernel_neon_end();
|
||||
}
|
||||
|
||||
static void
|
||||
xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3,
|
||||
const unsigned long * __restrict p4,
|
||||
const unsigned long * __restrict p5)
|
||||
{
|
||||
kernel_neon_begin();
|
||||
xor_block_neon_inner.do_5(bytes, p1, p2, p3, p4, p5);
|
||||
kernel_neon_end();
|
||||
}
|
||||
|
||||
struct xor_block_template xor_block_neon = {
|
||||
.name = "neon",
|
||||
.do_2 = xor_neon_2,
|
||||
.do_3 = xor_neon_3,
|
||||
.do_4 = xor_neon_4,
|
||||
.do_5 = xor_neon_5
|
||||
};
|
||||
|
|
@ -1,16 +1,9 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* linux/arch/arm/lib/xor-neon.c
|
||||
*
|
||||
* Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
|
||||
*/
|
||||
|
||||
#include <linux/raid/xor.h>
|
||||
#include <linux/raid/xor_impl.h>
|
||||
#include <linux/module.h>
|
||||
|
||||
MODULE_DESCRIPTION("NEON accelerated XOR implementation");
|
||||
MODULE_LICENSE("GPL");
|
||||
|
||||
#ifndef __ARM_NEON__
|
||||
#error You should compile this file with '-march=armv7-a -mfloat-abi=softfp -mfpu=neon'
|
||||
|
|
@ -27,7 +20,7 @@ MODULE_LICENSE("GPL");
|
|||
#endif
|
||||
|
||||
#define NO_TEMPLATE
|
||||
#include "../../../lib/raid/xor/xor-8regs.c"
|
||||
#include "../xor-8regs.c"
|
||||
|
||||
struct xor_block_template const xor_block_neon_inner = {
|
||||
.name = "__inner_neon__",
|
||||
|
|
@ -36,4 +29,3 @@ struct xor_block_template const xor_block_neon_inner = {
|
|||
.do_4 = xor_8regs_4,
|
||||
.do_5 = xor_8regs_5,
|
||||
};
|
||||
EXPORT_SYMBOL(xor_block_neon_inner);
|
||||
136
lib/raid/xor/arm/xor.c
Normal file
136
lib/raid/xor/arm/xor.c
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (C) 2001 Russell King
|
||||
*/
|
||||
#include <linux/raid/xor_impl.h>
|
||||
#include <asm/xor.h>
|
||||
|
||||
#define __XOR(a1, a2) a1 ^= a2
|
||||
|
||||
#define GET_BLOCK_2(dst) \
|
||||
__asm__("ldmia %0, {%1, %2}" \
|
||||
: "=r" (dst), "=r" (a1), "=r" (a2) \
|
||||
: "0" (dst))
|
||||
|
||||
#define GET_BLOCK_4(dst) \
|
||||
__asm__("ldmia %0, {%1, %2, %3, %4}" \
|
||||
: "=r" (dst), "=r" (a1), "=r" (a2), "=r" (a3), "=r" (a4) \
|
||||
: "0" (dst))
|
||||
|
||||
#define XOR_BLOCK_2(src) \
|
||||
__asm__("ldmia %0!, {%1, %2}" \
|
||||
: "=r" (src), "=r" (b1), "=r" (b2) \
|
||||
: "0" (src)); \
|
||||
__XOR(a1, b1); __XOR(a2, b2);
|
||||
|
||||
#define XOR_BLOCK_4(src) \
|
||||
__asm__("ldmia %0!, {%1, %2, %3, %4}" \
|
||||
: "=r" (src), "=r" (b1), "=r" (b2), "=r" (b3), "=r" (b4) \
|
||||
: "0" (src)); \
|
||||
__XOR(a1, b1); __XOR(a2, b2); __XOR(a3, b3); __XOR(a4, b4)
|
||||
|
||||
#define PUT_BLOCK_2(dst) \
|
||||
__asm__ __volatile__("stmia %0!, {%2, %3}" \
|
||||
: "=r" (dst) \
|
||||
: "0" (dst), "r" (a1), "r" (a2))
|
||||
|
||||
#define PUT_BLOCK_4(dst) \
|
||||
__asm__ __volatile__("stmia %0!, {%2, %3, %4, %5}" \
|
||||
: "=r" (dst) \
|
||||
: "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4))
|
||||
|
||||
static void
|
||||
xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2)
|
||||
{
|
||||
unsigned int lines = bytes / sizeof(unsigned long) / 4;
|
||||
register unsigned int a1 __asm__("r4");
|
||||
register unsigned int a2 __asm__("r5");
|
||||
register unsigned int a3 __asm__("r6");
|
||||
register unsigned int a4 __asm__("r10");
|
||||
register unsigned int b1 __asm__("r8");
|
||||
register unsigned int b2 __asm__("r9");
|
||||
register unsigned int b3 __asm__("ip");
|
||||
register unsigned int b4 __asm__("lr");
|
||||
|
||||
do {
|
||||
GET_BLOCK_4(p1);
|
||||
XOR_BLOCK_4(p2);
|
||||
PUT_BLOCK_4(p1);
|
||||
} while (--lines);
|
||||
}
|
||||
|
||||
static void
|
||||
xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3)
|
||||
{
|
||||
unsigned int lines = bytes / sizeof(unsigned long) / 4;
|
||||
register unsigned int a1 __asm__("r4");
|
||||
register unsigned int a2 __asm__("r5");
|
||||
register unsigned int a3 __asm__("r6");
|
||||
register unsigned int a4 __asm__("r10");
|
||||
register unsigned int b1 __asm__("r8");
|
||||
register unsigned int b2 __asm__("r9");
|
||||
register unsigned int b3 __asm__("ip");
|
||||
register unsigned int b4 __asm__("lr");
|
||||
|
||||
do {
|
||||
GET_BLOCK_4(p1);
|
||||
XOR_BLOCK_4(p2);
|
||||
XOR_BLOCK_4(p3);
|
||||
PUT_BLOCK_4(p1);
|
||||
} while (--lines);
|
||||
}
|
||||
|
||||
static void
|
||||
xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3,
|
||||
const unsigned long * __restrict p4)
|
||||
{
|
||||
unsigned int lines = bytes / sizeof(unsigned long) / 2;
|
||||
register unsigned int a1 __asm__("r8");
|
||||
register unsigned int a2 __asm__("r9");
|
||||
register unsigned int b1 __asm__("ip");
|
||||
register unsigned int b2 __asm__("lr");
|
||||
|
||||
do {
|
||||
GET_BLOCK_2(p1);
|
||||
XOR_BLOCK_2(p2);
|
||||
XOR_BLOCK_2(p3);
|
||||
XOR_BLOCK_2(p4);
|
||||
PUT_BLOCK_2(p1);
|
||||
} while (--lines);
|
||||
}
|
||||
|
||||
static void
|
||||
xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1,
|
||||
const unsigned long * __restrict p2,
|
||||
const unsigned long * __restrict p3,
|
||||
const unsigned long * __restrict p4,
|
||||
const unsigned long * __restrict p5)
|
||||
{
|
||||
unsigned int lines = bytes / sizeof(unsigned long) / 2;
|
||||
register unsigned int a1 __asm__("r8");
|
||||
register unsigned int a2 __asm__("r9");
|
||||
register unsigned int b1 __asm__("ip");
|
||||
register unsigned int b2 __asm__("lr");
|
||||
|
||||
do {
|
||||
GET_BLOCK_2(p1);
|
||||
XOR_BLOCK_2(p2);
|
||||
XOR_BLOCK_2(p3);
|
||||
XOR_BLOCK_2(p4);
|
||||
XOR_BLOCK_2(p5);
|
||||
PUT_BLOCK_2(p1);
|
||||
} while (--lines);
|
||||
}
|
||||
|
||||
struct xor_block_template xor_block_arm4regs = {
|
||||
.name = "arm4regs",
|
||||
.do_2 = xor_arm4regs_2,
|
||||
.do_3 = xor_arm4regs_3,
|
||||
.do_4 = xor_arm4regs_4,
|
||||
.do_5 = xor_arm4regs_5,
|
||||
};
|
||||
Loading…
Reference in New Issue
Block a user