github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-28 18:21:24 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

s390: add support for DCACHE_WORD_ACCESS

Browse Source 
Implement load_unaligned_zeropad() and enable DCACHE_WORD_ACCESS to
speed up string operations in fs/dcache.c and fs/namei.c.

Reviewed-by: Sven Schnelle <svens@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
This commit is contained in:
Heiko Carstens 2023-10-06 15:42:42 +02:00 committed by Vasily Gorbik
parent 76292d7243
commit 802ba53eef
4 changed files with 45 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arch/s390/Kconfig
View File

@ -131,6 +131,7 @@ config S390
select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
select BUILDTIME_TABLE_SORT select BUILDTIME_TABLE_SORT
select CLONE_BACKWARDS2 select CLONE_BACKWARDS2
select DCACHE_WORD_ACCESS if !KMSAN
select DMA_OPS if PCI select DMA_OPS if PCI
select DYNAMIC_FTRACE if FUNCTION_TRACER select DYNAMIC_FTRACE if FUNCTION_TRACER
select FUNCTION_ALIGNMENT_8B if CC_IS_GCC select FUNCTION_ALIGNMENT_8B if CC_IS_GCC

4
arch/s390/include/asm/asm-extable.h
View File

@ -13,6 +13,7 @@
#define EX_TYPE_UA_LOAD_MEM 4 #define EX_TYPE_UA_LOAD_MEM 4
#define EX_TYPE_UA_LOAD_REG 5 #define EX_TYPE_UA_LOAD_REG 5
#define EX_TYPE_UA_LOAD_REGPAIR 6 #define EX_TYPE_UA_LOAD_REGPAIR 6
#define EX_TYPE_ZEROPAD 7
#define EX_DATA_REG_ERR_SHIFT 0 #define EX_DATA_REG_ERR_SHIFT 0
#define EX_DATA_REG_ERR GENMASK(3, 0) #define EX_DATA_REG_ERR GENMASK(3, 0)
@ -80,4 +81,7 @@
#define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \
__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0)
#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \
__EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0)
#endif /* __ASM_EXTABLE_H */ #endif /* __ASM_EXTABLE_H */

22
arch/s390/include/asm/word-at-a-time.h
View File

@ -3,6 +3,7 @@
#define _ASM_WORD_AT_A_TIME_H #define _ASM_WORD_AT_A_TIME_H
#include <linux/kernel.h> #include <linux/kernel.h>
#include <asm/asm-extable.h>
#include <asm/bitsperlong.h> #include <asm/bitsperlong.h>
struct word_at_a_time { struct word_at_a_time {
@ -39,4 +40,25 @@ static inline unsigned long zero_bytemask(unsigned long data)
return ~1UL << data; return ~1UL << data;
} }
/*
* Load an unaligned word from kernel space.
*
* In the (very unlikely) case of the word being a page-crosser
* and the next page not being mapped, take the exception and
* return zeroes in the non-existing part.
*/
static inline unsigned long load_unaligned_zeropad(const void *addr)
{
unsigned long data;
asm volatile(
"0: lg %[data],0(%[addr])\n"
"1: nopr %%r7\n"
EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr])
EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr])
: [data] "=d" (data)
: [addr] "a" (addr), "m" (*(unsigned long *)addr));
return data;
}
#endif /* _ASM_WORD_AT_A_TIME_H */ #endif /* _ASM_WORD_AT_A_TIME_H */

18
arch/s390/mm/extable.c
View File

@ -61,6 +61,22 @@ static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex,
return true; return true;
} }
static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs)
{
unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data);
unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data);
unsigned long data, addr, offset;
addr = regs->gprs[reg_addr];
offset = addr & (sizeof(unsigned long) - 1);
addr &= ~(sizeof(unsigned long) - 1);
data = *(unsigned long *)addr;
data <<= BITS_PER_BYTE * offset;
regs->gprs[reg_data] = data;
regs->psw.addr = extable_fixup(ex);
return true;
}
bool fixup_exception(struct pt_regs *regs) bool fixup_exception(struct pt_regs *regs)
{ {
const struct exception_table_entry *ex; const struct exception_table_entry *ex;
@ -81,6 +97,8 @@ bool fixup_exception(struct pt_regs *regs)
return ex_handler_ua_load_reg(ex, false, regs); return ex_handler_ua_load_reg(ex, false, regs);
case EX_TYPE_UA_LOAD_REGPAIR: case EX_TYPE_UA_LOAD_REGPAIR:
return ex_handler_ua_load_reg(ex, true, regs); return ex_handler_ua_load_reg(ex, true, regs);
case EX_TYPE_ZEROPAD:
return ex_handler_zeropad(ex, regs);
} }
panic("invalid exception table entry"); panic("invalid exception table entry");
} }