mirror of
https://github.com/torvalds/linux.git
synced 2026-11-07 06:17:02 +01:00
109f733202
cmpxchg_emu_u8() emulates one-byte cmpxchg() in terms of four-byte cmpxchg() for the architectures lacking native one-byte atomics. The same architectures also lack native two-byte cmpxchg(), where such an operation is not supported and either fails to compile via BUILD_BUG() or fails to link, because the bad pointer sentinels these architectures declare are never defined. Add cmpxchg_emu_u16(), the two-byte sibling. It reads the enclosing word with READ_ONCE(), splices the two target bytes through a union and loops on cmpxchg() of the full word until the compare succeeds. Like cmpxchg_emu_u8() it is fully ordered. Unlike cmpxchg_emu_u8() it casts the old and new values to u16 internally and returns unsigned long, taking the old and new values as unsigned long, per the suggestion from David Laight. The switch statements in the architecture macros instantiate every size case, so a cmpxchg() on a pointer type checks the two-byte case as well, and a u16 parameter or return would make the macro casts and return conversions warn there. With unsigned long parameters and return the call sites need no narrowing casts, pointer exchanges compile warning free, and the function still compares and returns exactly the 16 bits the caller asked for, which matches the hardware cmpxchg r16 behaviour where a 16-bit compare only looks at the low 16 bits of the register. cmpxchg_emu_u8() keeps returning the old value unmasked, so a caller that passes a wider old than 8 bits still gets it back as passed, matching the behaviour the one-byte emulator always had. The Kconfig symbol gating this file is renamed from ARCH_NEED_CMPXCHG_1_EMU to ARCH_NEED_CMPXCHG_1_2_EMU, as it now selects both the one-byte and the two-byte emulation. [ paulmck: Apply kernel test robot feedback. ] Suggested-by: Paul E. McKenney <paulmck@kernel.org> Suggested-by: David Laight <david.laight.linux@gmail.com> Signed-off-by: Bradley Morgan <brads@mainlining.org> Signed-off-by: Paul E. McKenney <paulmck@kernel.org> Reviewed-by: David Laight <david.laight.linux@gmail.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Arnd Bergmann <arnd@kernel.org>
76 lines
1.8 KiB
C
76 lines
1.8 KiB
C
// SPDX-License-Identifier: GPL-2.0+
|
|
/*
|
|
* Emulated 1-byte and 2-byte cmpxchg operations for architectures lacking
|
|
* direct support for these sizes. These are implemented in terms of
|
|
* 4-byte cmpxchg operations.
|
|
*
|
|
* Copyright (C) 2024 Paul E. McKenney <paulmck@kernel.org>
|
|
* Copyright (C) 2026 Bradley Morgan <brads@mainlining.org>
|
|
*/
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/export.h>
|
|
#include <linux/instrumented.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/panic.h>
|
|
#include <linux/bug.h>
|
|
#include <asm-generic/rwonce.h>
|
|
#include <linux/cmpxchg-emu.h>
|
|
|
|
union u8_32 {
|
|
u8 b[4];
|
|
u32 w;
|
|
};
|
|
|
|
/* Emulate one-byte cmpxchg() in terms of 4-byte cmpxchg. */
|
|
uintptr_t cmpxchg_emu_u8(volatile u8 *p, uintptr_t old, uintptr_t new)
|
|
{
|
|
u32 *p32 = (u32 *)(((uintptr_t)p) & ~0x3);
|
|
int i = ((uintptr_t)p) & 0x3;
|
|
union u8_32 old32;
|
|
union u8_32 new32;
|
|
u32 ret;
|
|
|
|
ret = READ_ONCE(*p32);
|
|
do {
|
|
old32.w = ret;
|
|
if (old32.b[i] != old)
|
|
return old32.b[i];
|
|
new32.w = old32.w;
|
|
new32.b[i] = new;
|
|
instrument_atomic_read_write(p, 1);
|
|
ret = data_race(cmpxchg(p32, old32.w, new32.w)); // Overridden above.
|
|
} while (ret != old32.w);
|
|
return old;
|
|
}
|
|
EXPORT_SYMBOL_GPL(cmpxchg_emu_u8);
|
|
|
|
union u16_32 {
|
|
u16 h[2];
|
|
u32 w;
|
|
};
|
|
|
|
/* Emulate two-byte cmpxchg() in terms of 4-byte cmpxchg. */
|
|
unsigned long cmpxchg_emu_u16(volatile u16 *p, unsigned long old, unsigned long new)
|
|
{
|
|
u32 *p32 = (u32 *)(((uintptr_t)p) & ~0x3);
|
|
int i = (((uintptr_t)p) & 0x2) / 2;
|
|
union u16_32 old32;
|
|
union u16_32 new32;
|
|
u32 ret;
|
|
|
|
WARN_ON_ONCE(((uintptr_t)p) & 0x1);
|
|
ret = READ_ONCE(*p32);
|
|
do {
|
|
old32.w = ret;
|
|
if (old32.h[i] != (u16)old)
|
|
return (unsigned long)old32.h[i];
|
|
new32.w = old32.w;
|
|
new32.h[i] = (u16)new;
|
|
instrument_atomic_read_write(p, 2);
|
|
ret = data_race(cmpxchg(p32, old32.w, new32.w)); // Overridden above.
|
|
} while (ret != old32.w);
|
|
return (u16)old;
|
|
}
|
|
EXPORT_SYMBOL_GPL(cmpxchg_emu_u16);
|