diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index ae9d954aa052..bf5a5beab366 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -114,6 +114,7 @@ config LOONGARCH select GENERIC_TIME_VSYSCALL select GPIOLIB select HAS_IOPORT + select HAVE_ALIGNED_STRUCT_PAGE select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_BITREVERSE select HAVE_ARCH_JUMP_LABEL @@ -130,6 +131,7 @@ config LOONGARCH select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_ARCH_USERFAULTFD_MINOR if USERFAULTFD select HAVE_ASM_MODVERSIONS + select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL select HAVE_CONTEXT_TRACKING_USER select HAVE_C_RECORDMCOUNT diff --git a/arch/loongarch/include/asm/cmpxchg.h b/arch/loongarch/include/asm/cmpxchg.h index 0494c2ab553e..58cabab6d90d 100644 --- a/arch/loongarch/include/asm/cmpxchg.h +++ b/arch/loongarch/include/asm/cmpxchg.h @@ -8,6 +8,7 @@ #include #include #include +#include #define __xchg_amo_asm(amswap_db, m, val) \ ({ \ @@ -236,6 +237,59 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ arch_cmpxchg((ptr), (o), (n)); \ }) + +union __u128_halves { + u128 full; + struct { + u64 low; + u64 high; + }; +}; + +#define system_has_cmpxchg128() cpu_opt(LOONGARCH_CPU_SCQ) + +#define __arch_cmpxchg128(ptr, old, new, llsc_mb) \ +({ \ + union __u128_halves __old, __new, __ret; \ + volatile u64 *__ptr = (volatile u64 *)(ptr); \ + \ + __old.full = (old); \ + __new.full = (new); \ + \ + __asm__ __volatile__( \ + "1: ll.d %0, %3 # 128-bit cmpxchg low \n" \ + llsc_mb \ + " ld.d %1, %4 # 128-bit cmpxchg high \n" \ + " move $t0, %0 \n" \ + " move $t1, %1 \n" \ + " bne %0, %z5, 2f \n" \ + " bne %1, %z6, 2f \n" \ + " move $t0, %z7 \n" \ + " move $t1, %z8 \n" \ + "2: sc.q $t0, $t1, %2 \n" \ + " beqz $t0, 1b \n" \ + llsc_mb \ + : "=&r" (__ret.low), "=&r" (__ret.high) \ + : "r" (__ptr), \ + "ZC" (__ptr[0]), "m" (__ptr[1]), \ + "Jr" (__old.low), "Jr" (__old.high), \ + "Jr" (__new.low), "Jr" (__new.high) \ + : "t0", "t1", "memory"); \ + \ + __ret.full; \ +}) + +#define arch_cmpxchg128(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 16); \ + __arch_cmpxchg128(ptr, o, n, __WEAK_LLSC_MB); \ +}) + +#define arch_cmpxchg128_local(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 16); \ + __arch_cmpxchg128(ptr, o, n, ""); \ +}) #else #include #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n))