github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-25 23:52:08 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

s390/uaccess: Inline __clear_user()

Browse Source 
Rework __clear_user() similar to raw_copy_from_user() / raw_copy_to_user()
and inline the function saving the overhead of branches.

Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
This commit is contained in:
Heiko Carstens 2025-02-11 20:19:30 +01:00 committed by Vasily Gorbik
parent 88e87cb7b8
commit ee487b0120
2 changed files with 28 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
arch/s390/include/asm/uaccess.h
View File

@ -13,6 +13,7 @@
/*
* User space memory access functions
*/
#include <linux/pgtable.h>
#include <asm/asm-extable.h>
#include <asm/processor.h>
#include <asm/extable.h>
@ -362,12 +363,34 @@ long __must_check strncpy_from_user(char *dst, const char __user *src, long coun
long __must_check strnlen_user(const char __user *src, long count);
/*
* Zero Userspace
*/
unsigned long __must_check __clear_user(void __user *to, unsigned long size);
static uaccess_kmsan_or_inline __must_check unsigned long
__clear_user(void __user *to, unsigned long size)
{
unsigned long osize;
int cc;
static inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
while (1) {
osize = size;
asm_inline volatile(
" llilh %%r0,%[spec]\n"
"0: mvcos %[to],%[from],%[size]\n"
"1: nopr %%r7\n"
CC_IPM(cc)
EX_TABLE_UA_MVCOS_TO(0b, 0b)
EX_TABLE_UA_MVCOS_TO(1b, 0b)
: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
: [spec] "I" (0x81), [from] "Q" (*(const char *)empty_zero_page)
: CC_CLOBBER_LIST("memory", "0"));
if (__builtin_constant_p(osize) && osize <= 4096)
return osize - size;
if (CC_TRANSFORM(cc) == 0)
return osize - size;
size -= 4096;
to += 4096;
}
}
static __always_inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
{
might_fault();
return __clear_user(to, n);

36
arch/s390/lib/uaccess.c
View File

@ -144,39 +144,3 @@ unsigned long _copy_to_user_key(void __user *to, const void *from,
return raw_copy_to_user_key(to, from, n, key);
}
EXPORT_SYMBOL(_copy_to_user_key);
unsigned long __clear_user(void __user *to, unsigned long size)
{
unsigned long rem;
union oac spec = {
.oac1.as = PSW_BITS_AS_SECONDARY,
.oac1.a = 1,
};
asm volatile(
" lr 0,%[spec]\n"
"0: mvcos 0(%[to]),0(%[zeropg]),%[size]\n"
"1: jz 5f\n"
" algr %[size],%[val]\n"
" slgr %[to],%[val]\n"
" j 0b\n"
"2: la %[rem],4095(%[to])\n" /* rem = to + 4095 */
" nr %[rem],%[val]\n" /* rem = (to + 4095) & -4096 */
" slgr %[rem],%[to]\n"
" clgr %[size],%[rem]\n" /* copy crosses next page boundary? */
" jnh 6f\n"
"3: mvcos 0(%[to]),0(%[zeropg]),%[rem]\n"
"4: slgr %[size],%[rem]\n"
" j 6f\n"
"5: slgr %[size],%[size]\n"
"6:\n"
EX_TABLE(0b, 2b)
EX_TABLE(1b, 2b)
EX_TABLE(3b, 6b)
EX_TABLE(4b, 6b)
: [size] "+&a" (size), [to] "+&a" (to), [rem] "=&a" (rem)
: [val] "a" (-4096UL), [zeropg] "a" (empty_zero_page), [spec] "d" (spec.val)
: "cc", "memory", "0");
return size;
}
EXPORT_SYMBOL(__clear_user);