github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 09:36:22 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM: s390: Switch to new gmap

Browse Source 
Switch KVM/s390 to use the new gmap code.

Remove includes to <gmap.h> and include "gmap.h" instead; fix all the
existing users of the old gmap functions to use the new ones instead.

Fix guest storage key access functions to work with the new gmap.

Acked-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Claudio Imbrenda <imbrenda@linux.ibm.com>
This commit is contained in:
Claudio Imbrenda 2026-02-04 16:02:53 +01:00
parent d29a29a9e1
commit e38c884df9
21 changed files with 1129 additions and 1736 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/s390/Kconfig
View File

@ -33,7 +33,7 @@ config GENERIC_LOCKBREAK
def_bool y if PREEMPTION def_bool y if PREEMPTION
config PGSTE config PGSTE
def_bool y if KVM def_bool n
config AUDIT_ARCH config AUDIT_ARCH
def_bool y def_bool y

5
arch/s390/include/asm/kvm_host.h
View File

@ -442,7 +442,7 @@ struct kvm_vcpu_arch {
bool acrs_loaded; bool acrs_loaded;
struct kvm_s390_pv_vcpu pv; struct kvm_s390_pv_vcpu pv;
union diag318_info diag318_info; union diag318_info diag318_info;
void *mc; /* Placeholder */ struct kvm_s390_mmu_cache *mc;
}; };
struct kvm_vm_stat { struct kvm_vm_stat {
@ -636,6 +636,8 @@ struct kvm_s390_pv {
struct mutex import_lock; struct mutex import_lock;
}; };
struct kvm_s390_mmu_cache;
struct kvm_arch { struct kvm_arch {
struct esca_block *sca; struct esca_block *sca;
debug_info_t *dbf; debug_info_t *dbf;
@ -675,6 +677,7 @@ struct kvm_arch {
struct kvm_s390_pv pv; struct kvm_s390_pv pv;
struct list_head kzdev_list; struct list_head kzdev_list;
spinlock_t kzdev_list_lock; spinlock_t kzdev_list_lock;
struct kvm_s390_mmu_cache *mc;
}; };
#define KVM_HVA_ERR_BAD (-1UL) #define KVM_HVA_ERR_BAD (-1UL)

4
arch/s390/include/asm/mmu_context.h
View File

@ -30,11 +30,7 @@ static inline int init_new_context(struct task_struct *tsk,
mm->context.gmap_asce = 0; mm->context.gmap_asce = 0;
mm->context.flush_mm = 0; mm->context.flush_mm = 0;
#if IS_ENABLED(CONFIG_KVM) #if IS_ENABLED(CONFIG_KVM)
mm->context.has_pgste = 0;
mm->context.uses_skeys = 0;
mm->context.uses_cmm = 0;
mm->context.allow_cow_sharing = 1; mm->context.allow_cow_sharing = 1;
mm->context.allow_gmap_hpage_1m = 0;
#endif #endif
switch (mm->context.asce_limit) { switch (mm->context.asce_limit) {
default: default:

3
arch/s390/include/asm/tlb.h
View File

@ -36,7 +36,6 @@ static inline bool __tlb_remove_folio_pages(struct mmu_gather *tlb,
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include <asm-generic/tlb.h> #include <asm-generic/tlb.h>
#include <asm/gmap.h>
/* /*
* Release the page cache reference for a pte removed by * Release the page cache reference for a pte removed by
@ -85,8 +84,6 @@ static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
tlb->mm->context.flush_mm = 1; tlb->mm->context.flush_mm = 1;
tlb->freed_tables = 1; tlb->freed_tables = 1;
tlb->cleared_pmds = 1; tlb->cleared_pmds = 1;
if (mm_has_pgste(tlb->mm))
gmap_unlink(tlb->mm, (unsigned long *)pte, address);
tlb_remove_ptdesc(tlb, virt_to_ptdesc(pte)); tlb_remove_ptdesc(tlb, virt_to_ptdesc(pte));
} }

70
arch/s390/include/asm/uaccess.h
View File

@ -471,65 +471,15 @@ do { \
#define arch_get_kernel_nofault __mvc_kernel_nofault #define arch_get_kernel_nofault __mvc_kernel_nofault
#define arch_put_kernel_nofault __mvc_kernel_nofault #define arch_put_kernel_nofault __mvc_kernel_nofault
void __cmpxchg_user_key_called_with_bad_pointer(void); int __cmpxchg_key1(void *address, unsigned char *uval, unsigned char old,
unsigned char new, unsigned long key);
int __cmpxchg_user_key1(unsigned long address, unsigned char *uval, int __cmpxchg_key2(void *address, unsigned short *uval, unsigned short old,
unsigned char old, unsigned char new, unsigned long key); unsigned short new, unsigned long key);
int __cmpxchg_user_key2(unsigned long address, unsigned short *uval, int __cmpxchg_key4(void *address, unsigned int *uval, unsigned int old,
unsigned short old, unsigned short new, unsigned long key); unsigned int new, unsigned long key);
int __cmpxchg_user_key4(unsigned long address, unsigned int *uval, int __cmpxchg_key8(void *address, unsigned long *uval, unsigned long old,
unsigned int old, unsigned int new, unsigned long key); unsigned long new, unsigned long key);
int __cmpxchg_user_key8(unsigned long address, unsigned long *uval, int __cmpxchg_key16(void *address, __uint128_t *uval, __uint128_t old,
unsigned long old, unsigned long new, unsigned long key); __uint128_t new, unsigned long key);
int __cmpxchg_user_key16(unsigned long address, __uint128_t *uval,
__uint128_t old, __uint128_t new, unsigned long key);
static __always_inline int _cmpxchg_user_key(unsigned long address, void *uval,
__uint128_t old, __uint128_t new,
unsigned long key, int size)
{
switch (size) {
case 1: return __cmpxchg_user_key1(address, uval, old, new, key);
case 2: return __cmpxchg_user_key2(address, uval, old, new, key);
case 4: return __cmpxchg_user_key4(address, uval, old, new, key);
case 8: return __cmpxchg_user_key8(address, uval, old, new, key);
case 16: return __cmpxchg_user_key16(address, uval, old, new, key);
default: __cmpxchg_user_key_called_with_bad_pointer();
}
return 0;
}
/**
* cmpxchg_user_key() - cmpxchg with user space target, honoring storage keys
* @ptr: User space address of value to compare to @old and exchange with
* @new. Must be aligned to sizeof(*@ptr).
* @uval: Address where the old value of *@ptr is written to.
* @old: Old value. Compared to the content pointed to by @ptr in order to
* determine if the exchange occurs. The old value read from *@ptr is
* written to *@uval.
* @new: New value to place at *@ptr.
* @key: Access key to use for checking storage key protection.
*
* Perform a cmpxchg on a user space target, honoring storage key protection.
* @key alone determines how key checking is performed, neither
* storage-protection-override nor fetch-protection-override apply.
* The caller must compare *@uval and @old to determine if values have been
* exchanged. In case of an exception *@uval is set to zero.
*
* Return: 0: cmpxchg executed
* -EFAULT: an exception happened when trying to access *@ptr
* -EAGAIN: maxed out number of retries (byte and short only)
*/
#define cmpxchg_user_key(ptr, uval, old, new, key) \
({ \
__typeof__(ptr) __ptr = (ptr); \
__typeof__(uval) __uval = (uval); \
\
BUILD_BUG_ON(sizeof(*(__ptr)) != sizeof(*(__uval))); \
might_fault(); \
__chk_user_ptr(__ptr); \
_cmpxchg_user_key((unsigned long)(__ptr), (void *)(__uval), \
(old), (new), (key), sizeof(*(__ptr))); \
})
#endif /* __S390_UACCESS_H */ #endif /* __S390_UACCESS_H */

1
arch/s390/include/asm/uv.h
View File

@ -631,7 +631,6 @@ int uv_pin_shared(unsigned long paddr);
int uv_destroy_folio(struct folio *folio); int uv_destroy_folio(struct folio *folio);
int uv_destroy_pte(pte_t pte); int uv_destroy_pte(pte_t pte);
int uv_convert_from_secure_pte(pte_t pte); int uv_convert_from_secure_pte(pte_t pte);
int make_hva_secure(struct mm_struct *mm, unsigned long hva, struct uv_cb_header *uvcb);
int s390_wiggle_split_folio(struct mm_struct *mm, struct folio *folio); int s390_wiggle_split_folio(struct mm_struct *mm, struct folio *folio);
int __make_folio_secure(struct folio *folio, struct uv_cb_header *uvcb); int __make_folio_secure(struct folio *folio, struct uv_cb_header *uvcb);
int uv_convert_from_secure(unsigned long paddr); int uv_convert_from_secure(unsigned long paddr);

114
arch/s390/kernel/uv.c
View File

@ -209,39 +209,6 @@ int uv_convert_from_secure_pte(pte_t pte)
return uv_convert_from_secure_folio(pfn_folio(pte_pfn(pte))); return uv_convert_from_secure_folio(pfn_folio(pte_pfn(pte)));
} }
/**
* should_export_before_import - Determine whether an export is needed
* before an import-like operation
* @uvcb: the Ultravisor control block of the UVC to be performed
* @mm: the mm of the process
*
* Returns whether an export is needed before every import-like operation.
* This is needed for shared pages, which don't trigger a secure storage
* exception when accessed from a different guest.
*
* Although considered as one, the Unpin Page UVC is not an actual import,
* so it is not affected.
*
* No export is needed also when there is only one protected VM, because the
* page cannot belong to the wrong VM in that case (there is no "other VM"
* it can belong to).
*
* Return: true if an export is needed before every import, otherwise false.
*/
static bool should_export_before_import(struct uv_cb_header *uvcb, struct mm_struct *mm)
{
/*
* The misc feature indicates, among other things, that importing a
* shared page from a different protected VM will automatically also
* transfer its ownership.
*/
if (uv_has_feature(BIT_UV_FEAT_MISC))
return false;
if (uvcb->cmd == UVC_CMD_UNPIN_PAGE_SHARED)
return false;
return atomic_read(&mm->context.protected_count) > 1;
}
/* /*
* Calculate the expected ref_count for a folio that would otherwise have no * Calculate the expected ref_count for a folio that would otherwise have no
* further pins. This was cribbed from similar functions in other places in * further pins. This was cribbed from similar functions in other places in
@ -313,20 +280,6 @@ int __make_folio_secure(struct folio *folio, struct uv_cb_header *uvcb)
} }
EXPORT_SYMBOL(__make_folio_secure); EXPORT_SYMBOL(__make_folio_secure);
static int make_folio_secure(struct mm_struct *mm, struct folio *folio, struct uv_cb_header *uvcb)
{
int rc;
if (!folio_trylock(folio))
return -EAGAIN;
if (should_export_before_import(uvcb, mm))
uv_convert_from_secure(folio_to_phys(folio));
rc = __make_folio_secure(folio, uvcb);
folio_unlock(folio);
return rc;
}
/** /**
* s390_wiggle_split_folio() - try to drain extra references to a folio and * s390_wiggle_split_folio() - try to drain extra references to a folio and
* split the folio if it is large. * split the folio if it is large.
@ -414,56 +367,6 @@ int s390_wiggle_split_folio(struct mm_struct *mm, struct folio *folio)
} }
EXPORT_SYMBOL_GPL(s390_wiggle_split_folio); EXPORT_SYMBOL_GPL(s390_wiggle_split_folio);
int make_hva_secure(struct mm_struct *mm, unsigned long hva, struct uv_cb_header *uvcb)
{
struct vm_area_struct *vma;
struct folio_walk fw;
struct folio *folio;
int rc;
mmap_read_lock(mm);
vma = vma_lookup(mm, hva);
if (!vma) {
mmap_read_unlock(mm);
return -EFAULT;
}
folio = folio_walk_start(&fw, vma, hva, 0);
if (!folio) {
mmap_read_unlock(mm);
return -ENXIO;
}
folio_get(folio);
/*
* Secure pages cannot be huge and userspace should not combine both.
* In case userspace does it anyway this will result in an -EFAULT for
* the unpack. The guest is thus never reaching secure mode.
* If userspace plays dirty tricks and decides to map huge pages at a
* later point in time, it will receive a segmentation fault or
* KVM_RUN will return -EFAULT.
*/
if (folio_test_hugetlb(folio))
rc = -EFAULT;
else if (folio_test_large(folio))
rc = -E2BIG;
else if (!pte_write(fw.pte) || (pte_val(fw.pte) & _PAGE_INVALID))
rc = -ENXIO;
else
rc = make_folio_secure(mm, folio, uvcb);
folio_walk_end(&fw, vma);
mmap_read_unlock(mm);
if (rc == -E2BIG || rc == -EBUSY) {
rc = s390_wiggle_split_folio(mm, folio);
if (!rc)
rc = -EAGAIN;
}
folio_put(folio);
return rc;
}
EXPORT_SYMBOL_GPL(make_hva_secure);
/* /*
* To be called with the folio locked or with an extra reference! This will * To be called with the folio locked or with an extra reference! This will
* prevent kvm_s390_pv_make_secure() from touching the folio concurrently. * prevent kvm_s390_pv_make_secure() from touching the folio concurrently.
@ -474,21 +377,18 @@ int arch_make_folio_accessible(struct folio *folio)
{ {
int rc = 0; int rc = 0;
/* Large folios cannot be secure */
if (unlikely(folio_test_large(folio)))
return 0;
/* /*
* PG_arch_1 is used in 2 places: * PG_arch_1 is used as an indication that this small folio might be
* 1. for storage keys of hugetlb folios and KVM * secure. This can overindicate, e.g. we set the bit before calling
* 2. As an indication that this small folio might be secure. This can * convert_to_secure.
* overindicate, e.g. we set the bit before calling
* convert_to_secure.
* As secure pages are never large folios, both variants can co-exists.
*/ */
if (!test_bit(PG_arch_1, &folio->flags.f)) if (!test_bit(PG_arch_1, &folio->flags.f))
return 0; return 0;
/* Large folios cannot be secure. */
if (WARN_ON_ONCE(folio_test_large(folio)))
return -EFAULT;
rc = uv_pin_shared(folio_to_phys(folio)); rc = uv_pin_shared(folio_to_phys(folio));
if (!rc) { if (!rc) {
clear_bit(PG_arch_1, &folio->flags.f); clear_bit(PG_arch_1, &folio->flags.f);

2
arch/s390/kvm/Makefile
View File

@ -8,7 +8,7 @@ include $(srctree)/virt/kvm/Makefile.kvm
ccflags-y := -Ivirt/kvm -Iarch/s390/kvm ccflags-y := -Ivirt/kvm -Iarch/s390/kvm
kvm-y += kvm-s390.o intercept.o interrupt.o priv.o sigp.o kvm-y += kvm-s390.o intercept.o interrupt.o priv.o sigp.o
kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o gmap-vsie.o kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o
kvm-y += dat.o gmap.o faultin.o kvm-y += dat.o gmap.o faultin.o
kvm-$(CONFIG_VFIO_PCI_ZDEV_KVM) += pci.o kvm-$(CONFIG_VFIO_PCI_ZDEV_KVM) += pci.o

2
arch/s390/kvm/diag.c
View File

@ -10,13 +10,13 @@
#include <linux/kvm.h> #include <linux/kvm.h>
#include <linux/kvm_host.h> #include <linux/kvm_host.h>
#include <asm/gmap.h>
#include <asm/gmap_helpers.h> #include <asm/gmap_helpers.h>
#include <asm/virtio-ccw.h> #include <asm/virtio-ccw.h>
#include "kvm-s390.h" #include "kvm-s390.h"
#include "trace.h" #include "trace.h"
#include "trace-s390.h" #include "trace-s390.h"
#include "gaccess.h" #include "gaccess.h"
#include "gmap.h"
static void do_discard_gfn_range(struct kvm_vcpu *vcpu, gfn_t gfn_start, gfn_t gfn_end) static void do_discard_gfn_range(struct kvm_vcpu *vcpu, gfn_t gfn_start, gfn_t gfn_end)
{ {

899
arch/s390/kvm/gaccess.c
View File

File diff suppressed because it is too large Load Diff

18
arch/s390/kvm/gaccess.h
View File

@ -206,7 +206,7 @@ int access_guest_with_key(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar,
int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
void *data, unsigned long len, enum gacc_mode mode); void *data, unsigned long len, enum gacc_mode mode);
int cmpxchg_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, int len, union kvm_s390_quad *old_addr, int cmpxchg_guest_abs_with_key(struct kvm *kvm, gpa_t gpa, int len, union kvm_s390_quad *old,
union kvm_s390_quad new, u8 access_key, bool *success); union kvm_s390_quad new, u8 access_key, bool *success);
/** /**
@ -450,11 +450,17 @@ void ipte_unlock(struct kvm *kvm);
int ipte_lock_held(struct kvm *kvm); int ipte_lock_held(struct kvm *kvm);
int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra); int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra);
/* MVPG PEI indication bits */ union mvpg_pei {
#define PEI_DAT_PROT 2 unsigned long val;
#define PEI_NOT_PTE 4 struct {
unsigned long addr : 61;
unsigned long not_pte : 1;
unsigned long dat_prot: 1;
unsigned long real : 1;
};
};
int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *shadow, int gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr,
unsigned long saddr, unsigned long *datptr); union mvpg_pei *datptr, bool wr);
#endif /* __KVM_S390_GACCESS_H */ #endif /* __KVM_S390_GACCESS_H */

141
arch/s390/kvm/gmap-vsie.c
View File

@ -1,141 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Guest memory management for KVM/s390 nested VMs.
*
* Copyright IBM Corp. 2008, 2020, 2024
*
* Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com>
* Martin Schwidefsky <schwidefsky@de.ibm.com>
* David Hildenbrand <david@redhat.com>
* Janosch Frank <frankja@linux.vnet.ibm.com>
*/
#include <linux/compiler.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/pgtable.h>
#include <linux/pagemap.h>
#include <linux/mman.h>
#include <asm/lowcore.h>
#include <asm/gmap.h>
#include <asm/uv.h>
#include "kvm-s390.h"
/**
* gmap_find_shadow - find a specific asce in the list of shadow tables
* @parent: pointer to the parent gmap
* @asce: ASCE for which the shadow table is created
* @edat_level: edat level to be used for the shadow translation
*
* Returns the pointer to a gmap if a shadow table with the given asce is
* already available, ERR_PTR(-EAGAIN) if another one is just being created,
* otherwise NULL
*
* Context: Called with parent->shadow_lock held
*/
static struct gmap *gmap_find_shadow(struct gmap *parent, unsigned long asce, int edat_level)
{
struct gmap *sg;
lockdep_assert_held(&parent->shadow_lock);
list_for_each_entry(sg, &parent->children, list) {
if (!gmap_shadow_valid(sg, asce, edat_level))
continue;
if (!sg->initialized)
return ERR_PTR(-EAGAIN);
refcount_inc(&sg->ref_count);
return sg;
}
return NULL;
}
/**
* gmap_shadow - create/find a shadow guest address space
* @parent: pointer to the parent gmap
* @asce: ASCE for which the shadow table is created
* @edat_level: edat level to be used for the shadow translation
*
* The pages of the top level page table referred by the asce parameter
* will be set to read-only and marked in the PGSTEs of the kvm process.
* The shadow table will be removed automatically on any change to the
* PTE mapping for the source table.
*
* Returns a guest address space structure, ERR_PTR(-ENOMEM) if out of memory,
* ERR_PTR(-EAGAIN) if the caller has to retry and ERR_PTR(-EFAULT) if the
* parent gmap table could not be protected.
*/
struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, int edat_level)
{
struct gmap *sg, *new;
unsigned long limit;
int rc;
if (KVM_BUG_ON(parent->mm->context.allow_gmap_hpage_1m, (struct kvm *)parent->private) ||
KVM_BUG_ON(gmap_is_shadow(parent), (struct kvm *)parent->private))
return ERR_PTR(-EFAULT);
spin_lock(&parent->shadow_lock);
sg = gmap_find_shadow(parent, asce, edat_level);
spin_unlock(&parent->shadow_lock);
if (sg)
return sg;
/* Create a new shadow gmap */
limit = -1UL >> (33 - (((asce & _ASCE_TYPE_MASK) >> 2) * 11));
if (asce & _ASCE_REAL_SPACE)
limit = -1UL;
new = gmap_alloc(limit);
if (!new)
return ERR_PTR(-ENOMEM);
new->mm = parent->mm;
new->parent = gmap_get(parent);
new->private = parent->private;
new->orig_asce = asce;
new->edat_level = edat_level;
new->initialized = false;
spin_lock(&parent->shadow_lock);
/* Recheck if another CPU created the same shadow */
sg = gmap_find_shadow(parent, asce, edat_level);
if (sg) {
spin_unlock(&parent->shadow_lock);
gmap_free(new);
return sg;
}
if (asce & _ASCE_REAL_SPACE) {
/* only allow one real-space gmap shadow */
list_for_each_entry(sg, &parent->children, list) {
if (sg->orig_asce & _ASCE_REAL_SPACE) {
spin_lock(&sg->guest_table_lock);
gmap_unshadow(sg);
spin_unlock(&sg->guest_table_lock);
list_del(&sg->list);
gmap_put(sg);
break;
}
}
}
refcount_set(&new->ref_count, 2);
list_add(&new->list, &parent->children);
if (asce & _ASCE_REAL_SPACE) {
/* nothing to protect, return right away */
new->initialized = true;
spin_unlock(&parent->shadow_lock);
return new;
}
spin_unlock(&parent->shadow_lock);
/* protect after insertion, so it will get properly invalidated */
mmap_read_lock(parent->mm);
rc = __kvm_s390_mprotect_many(parent, asce & _ASCE_ORIGIN,
((asce & _ASCE_TABLE_LENGTH) + 1),
PROT_READ, GMAP_NOTIFY_SHADOW);
mmap_read_unlock(parent->mm);
spin_lock(&parent->shadow_lock);
new->initialized = true;
if (rc) {
list_del(&new->list);
gmap_free(new);
new = ERR_PTR(rc);
}
spin_unlock(&parent->shadow_lock);
return new;
}

15
arch/s390/kvm/intercept.c
View File

@ -21,6 +21,7 @@
#include "gaccess.h" #include "gaccess.h"
#include "trace.h" #include "trace.h"
#include "trace-s390.h" #include "trace-s390.h"
#include "faultin.h"
u8 kvm_s390_get_ilen(struct kvm_vcpu *vcpu) u8 kvm_s390_get_ilen(struct kvm_vcpu *vcpu)
{ {
@ -367,8 +368,11 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu)
reg2, &srcaddr, GACC_FETCH, 0); reg2, &srcaddr, GACC_FETCH, 0);
if (rc) if (rc)
return kvm_s390_inject_prog_cond(vcpu, rc); return kvm_s390_inject_prog_cond(vcpu, rc);
rc = kvm_s390_handle_dat_fault(vcpu, srcaddr, 0);
if (rc != 0) do {
rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gpa_to_gfn(srcaddr), false);
} while (rc == -EAGAIN);
if (rc)
return rc; return rc;
/* Ensure that the source is paged-in, no actual access -> no key checking */ /* Ensure that the source is paged-in, no actual access -> no key checking */
@ -376,8 +380,11 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu)
reg1, &dstaddr, GACC_STORE, 0); reg1, &dstaddr, GACC_STORE, 0);
if (rc) if (rc)
return kvm_s390_inject_prog_cond(vcpu, rc); return kvm_s390_inject_prog_cond(vcpu, rc);
rc = kvm_s390_handle_dat_fault(vcpu, dstaddr, FOLL_WRITE);
if (rc != 0) do {
rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gpa_to_gfn(dstaddr), true);
} while (rc == -EAGAIN);
if (rc)
return rc; return rc;
kvm_s390_retry_instr(vcpu); kvm_s390_retry_instr(vcpu);

6
arch/s390/kvm/interrupt.c
View File

@ -26,7 +26,6 @@
#include <linux/uaccess.h> #include <linux/uaccess.h>
#include <asm/sclp.h> #include <asm/sclp.h>
#include <asm/isc.h> #include <asm/isc.h>
#include <asm/gmap.h>
#include <asm/nmi.h> #include <asm/nmi.h>
#include <asm/airq.h> #include <asm/airq.h>
#include <asm/tpi.h> #include <asm/tpi.h>
@ -34,6 +33,7 @@
#include "gaccess.h" #include "gaccess.h"
#include "trace-s390.h" #include "trace-s390.h"
#include "pci.h" #include "pci.h"
#include "gmap.h"
#define PFAULT_INIT 0x0600 #define PFAULT_INIT 0x0600
#define PFAULT_DONE 0x0680 #define PFAULT_DONE 0x0680
@ -2632,12 +2632,12 @@ static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
case KVM_DEV_FLIC_APF_ENABLE: case KVM_DEV_FLIC_APF_ENABLE:
if (kvm_is_ucontrol(dev->kvm)) if (kvm_is_ucontrol(dev->kvm))
return -EINVAL; return -EINVAL;
dev->kvm->arch.gmap->pfault_enabled = 1; set_bit(GMAP_FLAG_PFAULT_ENABLED, &dev->kvm->arch.gmap->flags);
break; break;
case KVM_DEV_FLIC_APF_DISABLE_WAIT: case KVM_DEV_FLIC_APF_DISABLE_WAIT:
if (kvm_is_ucontrol(dev->kvm)) if (kvm_is_ucontrol(dev->kvm))
return -EINVAL; return -EINVAL;
dev->kvm->arch.gmap->pfault_enabled = 0; clear_bit(GMAP_FLAG_PFAULT_ENABLED, &dev->kvm->arch.gmap->flags);
/* /*
* Make sure no async faults are in transition when * Make sure no async faults are in transition when
* clearing the queues. So we don't need to worry * clearing the queues. So we don't need to worry

787
arch/s390/kvm/kvm-s390.c
View File

File diff suppressed because it is too large Load Diff

19
arch/s390/kvm/kvm-s390.h
View File

@ -19,6 +19,8 @@
#include <asm/facility.h> #include <asm/facility.h>
#include <asm/processor.h> #include <asm/processor.h>
#include <asm/sclp.h> #include <asm/sclp.h>
#include "dat.h"
#include "gmap.h"
#define KVM_S390_UCONTROL_MEMSLOT (KVM_USER_MEM_SLOTS + 0) #define KVM_S390_UCONTROL_MEMSLOT (KVM_USER_MEM_SLOTS + 0)
@ -114,9 +116,7 @@ static inline int is_vcpu_idle(struct kvm_vcpu *vcpu)
static inline int kvm_is_ucontrol(struct kvm *kvm) static inline int kvm_is_ucontrol(struct kvm *kvm)
{ {
#ifdef CONFIG_KVM_S390_UCONTROL #ifdef CONFIG_KVM_S390_UCONTROL
if (kvm->arch.gmap) return test_bit(GMAP_FLAG_IS_UCONTROL, &kvm->arch.gmap->flags);
return 0;
return 1;
#else #else
return 0; return 0;
#endif #endif
@ -440,14 +440,9 @@ int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu);
/* implemented in vsie.c */ /* implemented in vsie.c */
int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu); int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu);
void kvm_s390_vsie_kick(struct kvm_vcpu *vcpu); void kvm_s390_vsie_kick(struct kvm_vcpu *vcpu);
void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, unsigned long start, void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, gpa_t start, gpa_t end);
unsigned long end);
void kvm_s390_vsie_init(struct kvm *kvm); void kvm_s390_vsie_init(struct kvm *kvm);
void kvm_s390_vsie_destroy(struct kvm *kvm); void kvm_s390_vsie_destroy(struct kvm *kvm);
int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level);
/* implemented in gmap-vsie.c */
struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, int edat_level);
/* implemented in sigp.c */ /* implemented in sigp.c */
int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu); int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu);
@ -469,15 +464,9 @@ void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu);
void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm); void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm);
__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu); __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu);
int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc); int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc);
int __kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gfn_t gfn, gpa_t gaddr, unsigned int flags);
int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned int prot, int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned int prot,
unsigned long bits); unsigned long bits);
static inline int kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gpa_t gaddr, unsigned int flags)
{
return __kvm_s390_handle_dat_fault(vcpu, gpa_to_gfn(gaddr), gaddr, flags);
}
bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu);
/* implemented in diag.c */ /* implemented in diag.c */

213
arch/s390/kvm/priv.c
View File

@ -21,13 +21,14 @@
#include <asm/ebcdic.h> #include <asm/ebcdic.h>
#include <asm/sysinfo.h> #include <asm/sysinfo.h>
#include <asm/page-states.h> #include <asm/page-states.h>
#include <asm/gmap.h>
#include <asm/ptrace.h> #include <asm/ptrace.h>
#include <asm/sclp.h> #include <asm/sclp.h>
#include <asm/ap.h> #include <asm/ap.h>
#include <asm/gmap_helpers.h>
#include "gaccess.h" #include "gaccess.h"
#include "kvm-s390.h" #include "kvm-s390.h"
#include "trace.h" #include "trace.h"
#include "gmap.h"
static int handle_ri(struct kvm_vcpu *vcpu) static int handle_ri(struct kvm_vcpu *vcpu)
{ {
@ -222,7 +223,7 @@ int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu)
if (vcpu->arch.skey_enabled) if (vcpu->arch.skey_enabled)
return 0; return 0;
rc = s390_enable_skey(); rc = gmap_enable_skeys(vcpu->arch.gmap);
VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc); VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc);
if (rc) if (rc)
return rc; return rc;
@ -255,10 +256,9 @@ static int try_handle_skey(struct kvm_vcpu *vcpu)
static int handle_iske(struct kvm_vcpu *vcpu) static int handle_iske(struct kvm_vcpu *vcpu)
{ {
unsigned long gaddr, vmaddr; unsigned long gaddr;
unsigned char key;
int reg1, reg2; int reg1, reg2;
bool unlocked; union skey key;
int rc; int rc;
vcpu->stat.instruction_iske++; vcpu->stat.instruction_iske++;
@ -275,37 +275,21 @@ static int handle_iske(struct kvm_vcpu *vcpu)
gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); gaddr = kvm_s390_logical_to_effective(vcpu, gaddr);
gaddr = kvm_s390_real_to_abs(vcpu, gaddr); gaddr = kvm_s390_real_to_abs(vcpu, gaddr);
vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
if (kvm_is_error_hva(vmaddr)) rc = dat_get_storage_key(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr), &key);
return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); if (rc > 0)
retry: return kvm_s390_inject_program_int(vcpu, rc);
unlocked = false;
mmap_read_lock(current->mm);
rc = get_guest_storage_key(current->mm, vmaddr, &key);
if (rc) {
rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked);
if (!rc) {
mmap_read_unlock(current->mm);
goto retry;
}
}
mmap_read_unlock(current->mm);
if (rc == -EFAULT)
return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
if (rc < 0) if (rc < 0)
return rc; return rc;
vcpu->run->s.regs.gprs[reg1] &= ~0xff; vcpu->run->s.regs.gprs[reg1] &= ~0xff;
vcpu->run->s.regs.gprs[reg1] |= key; vcpu->run->s.regs.gprs[reg1] |= key.skey;
return 0; return 0;
} }
static int handle_rrbe(struct kvm_vcpu *vcpu) static int handle_rrbe(struct kvm_vcpu *vcpu)
{ {
unsigned long vmaddr, gaddr; unsigned long gaddr;
int reg1, reg2; int reg1, reg2;
bool unlocked;
int rc; int rc;
vcpu->stat.instruction_rrbe++; vcpu->stat.instruction_rrbe++;
@ -322,24 +306,10 @@ static int handle_rrbe(struct kvm_vcpu *vcpu)
gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); gaddr = kvm_s390_logical_to_effective(vcpu, gaddr);
gaddr = kvm_s390_real_to_abs(vcpu, gaddr); gaddr = kvm_s390_real_to_abs(vcpu, gaddr);
vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
if (kvm_is_error_hva(vmaddr)) rc = dat_reset_reference_bit(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr));
return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); if (rc > 0)
retry: return kvm_s390_inject_program_int(vcpu, rc);
unlocked = false;
mmap_read_lock(current->mm);
rc = reset_guest_reference_bit(current->mm, vmaddr);
if (rc < 0) {
rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked);
if (!rc) {
mmap_read_unlock(current->mm);
goto retry;
}
}
mmap_read_unlock(current->mm);
if (rc == -EFAULT)
return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
if (rc < 0) if (rc < 0)
return rc; return rc;
kvm_s390_set_psw_cc(vcpu, rc); kvm_s390_set_psw_cc(vcpu, rc);
@ -354,9 +324,8 @@ static int handle_sske(struct kvm_vcpu *vcpu)
{ {
unsigned char m3 = vcpu->arch.sie_block->ipb >> 28; unsigned char m3 = vcpu->arch.sie_block->ipb >> 28;
unsigned long start, end; unsigned long start, end;
unsigned char key, oldkey; union skey key, oldkey;
int reg1, reg2; int reg1, reg2;
bool unlocked;
int rc; int rc;
vcpu->stat.instruction_sske++; vcpu->stat.instruction_sske++;
@ -377,7 +346,7 @@ static int handle_sske(struct kvm_vcpu *vcpu)
kvm_s390_get_regs_rre(vcpu, &reg1, &reg2); kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
key = vcpu->run->s.regs.gprs[reg1] & 0xfe; key.skey = vcpu->run->s.regs.gprs[reg1] & 0xfe;
start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
start = kvm_s390_logical_to_effective(vcpu, start); start = kvm_s390_logical_to_effective(vcpu, start);
if (m3 & SSKE_MB) { if (m3 & SSKE_MB) {
@ -389,27 +358,17 @@ static int handle_sske(struct kvm_vcpu *vcpu)
} }
while (start != end) { while (start != end) {
unsigned long vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); scoped_guard(read_lock, &vcpu->kvm->mmu_lock) {
unlocked = false; rc = dat_cond_set_storage_key(vcpu->arch.mc, vcpu->arch.gmap->asce,
gpa_to_gfn(start), key, &oldkey,
if (kvm_is_error_hva(vmaddr)) m3 & SSKE_NQ, m3 & SSKE_MR, m3 & SSKE_MC);
return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
mmap_read_lock(current->mm);
rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey,
m3 & SSKE_NQ, m3 & SSKE_MR,
m3 & SSKE_MC);
if (rc < 0) {
rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked);
rc = !rc ? -EAGAIN : rc;
} }
mmap_read_unlock(current->mm); if (rc > 1)
if (rc == -EFAULT)
return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
if (rc == -EAGAIN) if (rc == -ENOMEM) {
kvm_s390_mmu_cache_topup(vcpu->arch.mc);
continue; continue;
}
if (rc < 0) if (rc < 0)
return rc; return rc;
start += PAGE_SIZE; start += PAGE_SIZE;
@ -422,7 +381,7 @@ static int handle_sske(struct kvm_vcpu *vcpu)
} else { } else {
kvm_s390_set_psw_cc(vcpu, rc); kvm_s390_set_psw_cc(vcpu, rc);
vcpu->run->s.regs.gprs[reg1] &= ~0xff00UL; vcpu->run->s.regs.gprs[reg1] &= ~0xff00UL;
vcpu->run->s.regs.gprs[reg1] |= (u64) oldkey << 8; vcpu->run->s.regs.gprs[reg1] |= (u64)oldkey.skey << 8;
} }
} }
if (m3 & SSKE_MB) { if (m3 & SSKE_MB) {
@ -1082,7 +1041,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu)
bool mr = false, mc = false, nq; bool mr = false, mc = false, nq;
int reg1, reg2; int reg1, reg2;
unsigned long start, end; unsigned long start, end;
unsigned char key; union skey key;
vcpu->stat.instruction_pfmf++; vcpu->stat.instruction_pfmf++;
@ -1110,7 +1069,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu)
} }
nq = vcpu->run->s.regs.gprs[reg1] & PFMF_NQ; nq = vcpu->run->s.regs.gprs[reg1] & PFMF_NQ;
key = vcpu->run->s.regs.gprs[reg1] & PFMF_KEY; key.skey = vcpu->run->s.regs.gprs[reg1] & PFMF_KEY;
start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
start = kvm_s390_logical_to_effective(vcpu, start); start = kvm_s390_logical_to_effective(vcpu, start);
@ -1141,14 +1100,6 @@ static int handle_pfmf(struct kvm_vcpu *vcpu)
} }
while (start != end) { while (start != end) {
unsigned long vmaddr;
bool unlocked = false;
/* Translate guest address to host address */
vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start));
if (kvm_is_error_hva(vmaddr))
return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) {
if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE)) if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE))
return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
@ -1159,19 +1110,17 @@ static int handle_pfmf(struct kvm_vcpu *vcpu)
if (rc) if (rc)
return rc; return rc;
mmap_read_lock(current->mm); scoped_guard(read_lock, &vcpu->kvm->mmu_lock) {
rc = cond_set_guest_storage_key(current->mm, vmaddr, rc = dat_cond_set_storage_key(vcpu->arch.mc, vcpu->arch.gmap->asce,
key, NULL, nq, mr, mc); gpa_to_gfn(start), key,
if (rc < 0) { NULL, nq, mr, mc);
rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked);
rc = !rc ? -EAGAIN : rc;
} }
mmap_read_unlock(current->mm); if (rc > 1)
if (rc == -EFAULT) return kvm_s390_inject_program_int(vcpu, rc);
return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); if (rc == -ENOMEM) {
if (rc == -EAGAIN) kvm_s390_mmu_cache_topup(vcpu->arch.mc);
continue; continue;
}
if (rc < 0) if (rc < 0)
return rc; return rc;
} }
@ -1195,8 +1144,10 @@ static int handle_pfmf(struct kvm_vcpu *vcpu)
static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc) static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc)
{ {
int r1, r2, nappended, entries; int r1, r2, nappended, entries;
unsigned long gfn, hva, res, pgstev, ptev; union essa_state state;
unsigned long *cbrlo; unsigned long *cbrlo;
unsigned long gfn;
bool dirtied;
/* /*
* We don't need to set SD.FPF.SK to 1 here, because if we have a * We don't need to set SD.FPF.SK to 1 here, because if we have a
@ -1205,33 +1156,12 @@ static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc)
kvm_s390_get_regs_rre(vcpu, &r1, &r2); kvm_s390_get_regs_rre(vcpu, &r1, &r2);
gfn = vcpu->run->s.regs.gprs[r2] >> PAGE_SHIFT; gfn = vcpu->run->s.regs.gprs[r2] >> PAGE_SHIFT;
hva = gfn_to_hva(vcpu->kvm, gfn);
entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3;
if (kvm_is_error_hva(hva)) nappended = dat_perform_essa(vcpu->arch.gmap->asce, gfn, orc, &state, &dirtied);
return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); vcpu->run->s.regs.gprs[r1] = state.val;
if (nappended < 0)
nappended = pgste_perform_essa(vcpu->kvm->mm, hva, orc, &ptev, &pgstev);
if (nappended < 0) {
res = orc ? 0x10 : 0;
vcpu->run->s.regs.gprs[r1] = res; /* Exception Indication */
return 0; return 0;
}
res = (pgstev & _PGSTE_GPS_USAGE_MASK) >> 22;
/*
* Set the block-content state part of the result. 0 means resident, so
* nothing to do if the page is valid. 2 is for preserved pages
* (non-present and non-zero), and 3 for zero pages (non-present and
* zero).
*/
if (ptev & _PAGE_INVALID) {
res |= 2;
if (pgstev & _PGSTE_GPS_ZERO)
res |= 1;
}
if (pgstev & _PGSTE_GPS_NODAT)
res |= 0x20;
vcpu->run->s.regs.gprs[r1] = res;
/* /*
* It is possible that all the normal 511 slots were full, in which case * It is possible that all the normal 511 slots were full, in which case
* we will now write in the 512th slot, which is reserved for host use. * we will now write in the 512th slot, which is reserved for host use.
@ -1243,17 +1173,34 @@ static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc)
cbrlo[entries] = gfn << PAGE_SHIFT; cbrlo[entries] = gfn << PAGE_SHIFT;
} }
if (orc) { if (dirtied)
struct kvm_memory_slot *ms = gfn_to_memslot(vcpu->kvm, gfn); atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages);
/* Increment only if we are really flipping the bit */
if (ms && !test_and_set_bit(gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages);
}
return nappended; return nappended;
} }
static void _essa_clear_cbrl(struct kvm_vcpu *vcpu, unsigned long *cbrl, int len)
{
union crste *crstep;
union pgste pgste;
union pte *ptep;
int i;
lockdep_assert_held(&vcpu->kvm->mmu_lock);
for (i = 0; i < len; i++) {
if (dat_entry_walk(NULL, gpa_to_gfn(cbrl[i]), vcpu->arch.gmap->asce,
0, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep))
continue;
if (!ptep || ptep->s.pr)
continue;
pgste = pgste_get_lock(ptep);
if (pgste.usage == PGSTE_GPS_USAGE_UNUSED || pgste.zero)
gmap_helper_zap_one_page(vcpu->kvm->mm, cbrl[i]);
pgste_set_unlock(ptep, pgste);
}
}
static int handle_essa(struct kvm_vcpu *vcpu) static int handle_essa(struct kvm_vcpu *vcpu)
{ {
lockdep_assert_held(&vcpu->kvm->srcu); lockdep_assert_held(&vcpu->kvm->srcu);
@ -1261,11 +1208,9 @@ static int handle_essa(struct kvm_vcpu *vcpu)
/* entries expected to be 1FF */ /* entries expected to be 1FF */
int entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; int entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3;
unsigned long *cbrlo; unsigned long *cbrlo;
struct gmap *gmap;
int i, orc; int i, orc;
VCPU_EVENT(vcpu, 4, "ESSA: release %d pages", entries); VCPU_EVENT(vcpu, 4, "ESSA: release %d pages", entries);
gmap = vcpu->arch.gmap;
vcpu->stat.instruction_essa++; vcpu->stat.instruction_essa++;
if (!vcpu->kvm->arch.use_cmma) if (!vcpu->kvm->arch.use_cmma)
return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); return kvm_s390_inject_program_int(vcpu, PGM_OPERATION);
@ -1289,11 +1234,7 @@ static int handle_essa(struct kvm_vcpu *vcpu)
* value really needs to be written to; if the value is * value really needs to be written to; if the value is
* already correct, we do nothing and avoid the lock. * already correct, we do nothing and avoid the lock.
*/ */
if (vcpu->kvm->mm->context.uses_cmm == 0) { set_bit(GMAP_FLAG_USES_CMM, &vcpu->arch.gmap->flags);
mmap_write_lock(vcpu->kvm->mm);
vcpu->kvm->mm->context.uses_cmm = 1;
mmap_write_unlock(vcpu->kvm->mm);
}
/* /*
* If we are here, we are supposed to have CMMA enabled in * If we are here, we are supposed to have CMMA enabled in
* the SIE block. Enabling CMMA works on a per-CPU basis, * the SIE block. Enabling CMMA works on a per-CPU basis,
@ -1307,20 +1248,22 @@ static int handle_essa(struct kvm_vcpu *vcpu)
/* Retry the ESSA instruction */ /* Retry the ESSA instruction */
kvm_s390_retry_instr(vcpu); kvm_s390_retry_instr(vcpu);
} else { } else {
mmap_read_lock(vcpu->kvm->mm); scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
i = __do_essa(vcpu, orc); i = __do_essa(vcpu, orc);
mmap_read_unlock(vcpu->kvm->mm);
if (i < 0) if (i < 0)
return i; return i;
/* Account for the possible extra cbrl entry */ /* Account for the possible extra cbrl entry */
entries += i; entries += i;
} }
vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */ /* reset nceo */
vcpu->arch.sie_block->cbrlo &= PAGE_MASK;
cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo); cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo);
mmap_read_lock(gmap->mm);
for (i = 0; i < entries; ++i) mmap_read_lock(vcpu->kvm->mm);
__gmap_zap(gmap, cbrlo[i]); scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
mmap_read_unlock(gmap->mm); _essa_clear_cbrl(vcpu, cbrlo, entries);
mmap_read_unlock(vcpu->kvm->mm);
return 0; return 0;
} }

174
arch/s390/kvm/pv.c
View File

@ -12,13 +12,16 @@
#include <linux/minmax.h> #include <linux/minmax.h>
#include <linux/pagemap.h> #include <linux/pagemap.h>
#include <linux/sched/signal.h> #include <linux/sched/signal.h>
#include <asm/gmap.h>
#include <asm/uv.h> #include <asm/uv.h>
#include <asm/mman.h> #include <asm/mman.h>
#include <linux/pagewalk.h> #include <linux/pagewalk.h>
#include <linux/sched/mm.h> #include <linux/sched/mm.h>
#include <linux/mmu_notifier.h> #include <linux/mmu_notifier.h>
#include "kvm-s390.h" #include "kvm-s390.h"
#include "dat.h"
#include "gaccess.h"
#include "gmap.h"
#include "faultin.h"
bool kvm_s390_pv_is_protected(struct kvm *kvm) bool kvm_s390_pv_is_protected(struct kvm *kvm)
{ {
@ -34,6 +37,85 @@ bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu)
} }
EXPORT_SYMBOL_GPL(kvm_s390_pv_cpu_is_protected); EXPORT_SYMBOL_GPL(kvm_s390_pv_cpu_is_protected);
/**
* should_export_before_import() - Determine whether an export is needed
* before an import-like operation.
* @uvcb: The Ultravisor control block of the UVC to be performed.
* @mm: The mm of the process.
*
* Returns whether an export is needed before every import-like operation.
* This is needed for shared pages, which don't trigger a secure storage
* exception when accessed from a different guest.
*
* Although considered as one, the Unpin Page UVC is not an actual import,
* so it is not affected.
*
* No export is needed also when there is only one protected VM, because the
* page cannot belong to the wrong VM in that case (there is no "other VM"
* it can belong to).
*
* Return: %true if an export is needed before every import, otherwise %false.
*/
static bool should_export_before_import(struct uv_cb_header *uvcb, struct mm_struct *mm)
{
/*
* The misc feature indicates, among other things, that importing a
* shared page from a different protected VM will automatically also
* transfer its ownership.
*/
if (uv_has_feature(BIT_UV_FEAT_MISC))
return false;
if (uvcb->cmd == UVC_CMD_UNPIN_PAGE_SHARED)
return false;
return atomic_read(&mm->context.protected_count) > 1;
}
struct pv_make_secure {
void *uvcb;
struct folio *folio;
int rc;
bool needs_export;
};
static int __kvm_s390_pv_make_secure(struct guest_fault *f, struct folio *folio)
{
struct pv_make_secure *priv = f->priv;
int rc;
if (priv->needs_export)
uv_convert_from_secure(folio_to_phys(folio));
if (folio_test_hugetlb(folio))
return -EFAULT;
if (folio_test_large(folio))
return -E2BIG;
if (!f->page)
folio_get(folio);
rc = __make_folio_secure(folio, priv->uvcb);
if (!f->page)
folio_put(folio);
return rc;
}
static void _kvm_s390_pv_make_secure(struct guest_fault *f)
{
struct pv_make_secure *priv = f->priv;
struct folio *folio;
folio = pfn_folio(f->pfn);
priv->rc = -EAGAIN;
if (folio_trylock(folio)) {
priv->rc = __kvm_s390_pv_make_secure(f, folio);
if (priv->rc == -E2BIG || priv->rc == -EBUSY) {
priv->folio = folio;
folio_get(folio);
}
folio_unlock(folio);
}
}
/** /**
* kvm_s390_pv_make_secure() - make one guest page secure * kvm_s390_pv_make_secure() - make one guest page secure
* @kvm: the guest * @kvm: the guest
@ -45,14 +127,34 @@ EXPORT_SYMBOL_GPL(kvm_s390_pv_cpu_is_protected);
*/ */
int kvm_s390_pv_make_secure(struct kvm *kvm, unsigned long gaddr, void *uvcb) int kvm_s390_pv_make_secure(struct kvm *kvm, unsigned long gaddr, void *uvcb)
{ {
unsigned long vmaddr; struct pv_make_secure priv = { .uvcb = uvcb };
struct guest_fault f = {
.write_attempt = true,
.gfn = gpa_to_gfn(gaddr),
.callback = _kvm_s390_pv_make_secure,
.priv = &priv,
};
int rc;
lockdep_assert_held(&kvm->srcu); lockdep_assert_held(&kvm->srcu);
vmaddr = gfn_to_hva(kvm, gpa_to_gfn(gaddr)); priv.needs_export = should_export_before_import(uvcb, kvm->mm);
if (kvm_is_error_hva(vmaddr))
return -EFAULT; scoped_guard(mutex, &kvm->arch.pv.import_lock) {
return make_hva_secure(kvm->mm, vmaddr, uvcb); rc = kvm_s390_faultin_gfn(NULL, kvm, &f);
if (!rc) {
rc = priv.rc;
if (priv.folio) {
rc = s390_wiggle_split_folio(kvm->mm, priv.folio);
if (!rc)
rc = -EAGAIN;
}
}
}
if (priv.folio)
folio_put(priv.folio);
return rc;
} }
int kvm_s390_pv_convert_to_secure(struct kvm *kvm, unsigned long gaddr) int kvm_s390_pv_convert_to_secure(struct kvm *kvm, unsigned long gaddr)
@ -299,35 +401,6 @@ static int kvm_s390_pv_dispose_one_leftover(struct kvm *kvm,
return 0; return 0;
} }
/**
* kvm_s390_destroy_lower_2g - Destroy the first 2GB of protected guest memory.
* @kvm: the VM whose memory is to be cleared.
*
* Destroy the first 2GB of guest memory, to avoid prefix issues after reboot.
* The CPUs of the protected VM need to be destroyed beforehand.
*/
static void kvm_s390_destroy_lower_2g(struct kvm *kvm)
{
const unsigned long pages_2g = SZ_2G / PAGE_SIZE;
struct kvm_memory_slot *slot;
unsigned long len;
int srcu_idx;
srcu_idx = srcu_read_lock(&kvm->srcu);
/* Take the memslot containing guest absolute address 0 */
slot = gfn_to_memslot(kvm, 0);
/* Clear all slots or parts thereof that are below 2GB */
while (slot && slot->base_gfn < pages_2g) {
len = min_t(u64, slot->npages, pages_2g - slot->base_gfn) * PAGE_SIZE;
s390_uv_destroy_range(kvm->mm, slot->userspace_addr, slot->userspace_addr + len);
/* Take the next memslot */
slot = gfn_to_memslot(kvm, slot->base_gfn + slot->npages);
}
srcu_read_unlock(&kvm->srcu, srcu_idx);
}
static int kvm_s390_pv_deinit_vm_fast(struct kvm *kvm, u16 *rc, u16 *rrc) static int kvm_s390_pv_deinit_vm_fast(struct kvm *kvm, u16 *rc, u16 *rrc)
{ {
struct uv_cb_destroy_fast uvcb = { struct uv_cb_destroy_fast uvcb = {
@ -342,7 +415,6 @@ static int kvm_s390_pv_deinit_vm_fast(struct kvm *kvm, u16 *rc, u16 *rrc)
*rc = uvcb.header.rc; *rc = uvcb.header.rc;
if (rrc) if (rrc)
*rrc = uvcb.header.rrc; *rrc = uvcb.header.rrc;
WRITE_ONCE(kvm->arch.gmap->guest_handle, 0);
KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY VM FAST: rc %x rrc %x", KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY VM FAST: rc %x rrc %x",
uvcb.header.rc, uvcb.header.rrc); uvcb.header.rc, uvcb.header.rrc);
WARN_ONCE(cc && uvcb.header.rc != 0x104, WARN_ONCE(cc && uvcb.header.rc != 0x104,
@ -391,7 +463,7 @@ int kvm_s390_pv_set_aside(struct kvm *kvm, u16 *rc, u16 *rrc)
return -EINVAL; return -EINVAL;
/* Guest with segment type ASCE, refuse to destroy asynchronously */ /* Guest with segment type ASCE, refuse to destroy asynchronously */
if ((kvm->arch.gmap->asce & _ASCE_TYPE_MASK) == _ASCE_TYPE_SEGMENT) if (kvm->arch.gmap->asce.dt == TABLE_TYPE_SEGMENT)
return -EINVAL; return -EINVAL;
priv = kzalloc(sizeof(*priv), GFP_KERNEL); priv = kzalloc(sizeof(*priv), GFP_KERNEL);
@ -404,8 +476,7 @@ int kvm_s390_pv_set_aside(struct kvm *kvm, u16 *rc, u16 *rrc)
priv->stor_var = kvm->arch.pv.stor_var; priv->stor_var = kvm->arch.pv.stor_var;
priv->stor_base = kvm->arch.pv.stor_base; priv->stor_base = kvm->arch.pv.stor_base;
priv->handle = kvm_s390_pv_get_handle(kvm); priv->handle = kvm_s390_pv_get_handle(kvm);
priv->old_gmap_table = (unsigned long)kvm->arch.gmap->table; priv->old_gmap_table = (unsigned long)dereference_asce(kvm->arch.gmap->asce);
WRITE_ONCE(kvm->arch.gmap->guest_handle, 0);
if (s390_replace_asce(kvm->arch.gmap)) if (s390_replace_asce(kvm->arch.gmap))
res = -ENOMEM; res = -ENOMEM;
} }
@ -415,7 +486,7 @@ int kvm_s390_pv_set_aside(struct kvm *kvm, u16 *rc, u16 *rrc)
return res; return res;
} }
kvm_s390_destroy_lower_2g(kvm); gmap_pv_destroy_range(kvm->arch.gmap, 0, gpa_to_gfn(SZ_2G), false);
kvm_s390_clear_pv_state(kvm); kvm_s390_clear_pv_state(kvm);
kvm->arch.pv.set_aside = priv; kvm->arch.pv.set_aside = priv;
@ -449,7 +520,6 @@ int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
cc = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm), cc = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
UVC_CMD_DESTROY_SEC_CONF, rc, rrc); UVC_CMD_DESTROY_SEC_CONF, rc, rrc);
WRITE_ONCE(kvm->arch.gmap->guest_handle, 0);
if (!cc) { if (!cc) {
atomic_dec(&kvm->mm->context.protected_count); atomic_dec(&kvm->mm->context.protected_count);
kvm_s390_pv_dealloc_vm(kvm); kvm_s390_pv_dealloc_vm(kvm);
@ -532,7 +602,7 @@ int kvm_s390_pv_deinit_cleanup_all(struct kvm *kvm, u16 *rc, u16 *rrc)
* cleanup has been performed. * cleanup has been performed.
*/ */
if (need_zap && mmget_not_zero(kvm->mm)) { if (need_zap && mmget_not_zero(kvm->mm)) {
s390_uv_destroy_range(kvm->mm, 0, TASK_SIZE); gmap_pv_destroy_range(kvm->arch.gmap, 0, asce_end(kvm->arch.gmap->asce), false);
mmput(kvm->mm); mmput(kvm->mm);
} }
@ -570,7 +640,7 @@ int kvm_s390_pv_deinit_aside_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
return -EINVAL; return -EINVAL;
/* When a fatal signal is received, stop immediately */ /* When a fatal signal is received, stop immediately */
if (s390_uv_destroy_range_interruptible(kvm->mm, 0, TASK_SIZE_MAX)) if (gmap_pv_destroy_range(kvm->arch.gmap, 0, asce_end(kvm->arch.gmap->asce), true))
goto done; goto done;
if (kvm_s390_pv_dispose_one_leftover(kvm, p, rc, rrc)) if (kvm_s390_pv_dispose_one_leftover(kvm, p, rc, rrc))
ret = -EIO; ret = -EIO;
@ -609,6 +679,7 @@ static void kvm_s390_pv_mmu_notifier_release(struct mmu_notifier *subscription,
r = kvm_s390_cpus_from_pv(kvm, &dummy, &dummy); r = kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
if (!r && is_destroy_fast_available() && kvm_s390_pv_get_handle(kvm)) if (!r && is_destroy_fast_available() && kvm_s390_pv_get_handle(kvm))
kvm_s390_pv_deinit_vm_fast(kvm, &dummy, &dummy); kvm_s390_pv_deinit_vm_fast(kvm, &dummy, &dummy);
set_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &kvm->arch.gmap->flags);
} }
static const struct mmu_notifier_ops kvm_s390_pv_mmu_notifier_ops = { static const struct mmu_notifier_ops kvm_s390_pv_mmu_notifier_ops = {
@ -642,7 +713,7 @@ int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
/* Inputs */ /* Inputs */
uvcb.guest_stor_origin = 0; /* MSO is 0 for KVM */ uvcb.guest_stor_origin = 0; /* MSO is 0 for KVM */
uvcb.guest_stor_len = kvm->arch.pv.guest_len; uvcb.guest_stor_len = kvm->arch.pv.guest_len;
uvcb.guest_asce = kvm->arch.gmap->asce; uvcb.guest_asce = kvm->arch.gmap->asce.val;
uvcb.guest_sca = virt_to_phys(kvm->arch.sca); uvcb.guest_sca = virt_to_phys(kvm->arch.sca);
uvcb.conf_base_stor_origin = uvcb.conf_base_stor_origin =
virt_to_phys((void *)kvm->arch.pv.stor_base); virt_to_phys((void *)kvm->arch.pv.stor_base);
@ -669,7 +740,6 @@ int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
} }
return -EIO; return -EIO;
} }
kvm->arch.gmap->guest_handle = uvcb.guest_handle;
return 0; return 0;
} }
@ -704,26 +774,14 @@ static int unpack_one(struct kvm *kvm, unsigned long addr, u64 tweak,
.tweak[1] = offset, .tweak[1] = offset,
}; };
int ret = kvm_s390_pv_make_secure(kvm, addr, &uvcb); int ret = kvm_s390_pv_make_secure(kvm, addr, &uvcb);
unsigned long vmaddr;
bool unlocked;
*rc = uvcb.header.rc; *rc = uvcb.header.rc;
*rrc = uvcb.header.rrc; *rrc = uvcb.header.rrc;
if (ret == -ENXIO) { if (ret == -ENXIO) {
mmap_read_lock(kvm->mm); ret = kvm_s390_faultin_gfn_simple(NULL, kvm, gpa_to_gfn(addr), true);
vmaddr = gfn_to_hva(kvm, gpa_to_gfn(addr));
if (kvm_is_error_hva(vmaddr)) {
ret = -EFAULT;
} else {
ret = fixup_user_fault(kvm->mm, vmaddr, FAULT_FLAG_WRITE, &unlocked);
if (!ret)
ret = __gmap_link(kvm->arch.gmap, addr, vmaddr);
}
mmap_read_unlock(kvm->mm);
if (!ret) if (!ret)
return -EAGAIN; return -EAGAIN;
return ret;
} }
if (ret && ret != -EAGAIN) if (ret && ret != -EAGAIN)

168
arch/s390/kvm/vsie.c
View File

@ -15,7 +15,6 @@
#include <linux/io.h> #include <linux/io.h>
#include <linux/mman.h> #include <linux/mman.h>
#include <asm/gmap.h>
#include <asm/mmu_context.h> #include <asm/mmu_context.h>
#include <asm/sclp.h> #include <asm/sclp.h>
#include <asm/nmi.h> #include <asm/nmi.h>
@ -23,6 +22,7 @@
#include <asm/facility.h> #include <asm/facility.h>
#include "kvm-s390.h" #include "kvm-s390.h"
#include "gaccess.h" #include "gaccess.h"
#include "gmap.h"
enum vsie_page_flags { enum vsie_page_flags {
VSIE_PAGE_IN_USE = 0, VSIE_PAGE_IN_USE = 0,
@ -41,8 +41,11 @@ struct vsie_page {
* are reused conditionally, should be accessed via READ_ONCE. * are reused conditionally, should be accessed via READ_ONCE.
*/ */
struct kvm_s390_sie_block *scb_o; /* 0x0218 */ struct kvm_s390_sie_block *scb_o; /* 0x0218 */
/* the shadow gmap in use by the vsie_page */ /*
struct gmap *gmap; /* 0x0220 */ * Flags: must be set/cleared atomically after the vsie page can be
* looked up by other CPUs.
*/
unsigned long flags; /* 0x0220 */
/* address of the last reported fault to guest2 */ /* address of the last reported fault to guest2 */
unsigned long fault_addr; /* 0x0228 */ unsigned long fault_addr; /* 0x0228 */
/* calculated guest addresses of satellite control blocks */ /* calculated guest addresses of satellite control blocks */
@ -57,33 +60,14 @@ struct vsie_page {
* radix tree. * radix tree.
*/ */
gpa_t scb_gpa; /* 0x0258 */ gpa_t scb_gpa; /* 0x0258 */
/* /* the shadow gmap in use by the vsie_page */
* Flags: must be set/cleared atomically after the vsie page can be struct gmap_cache gmap_cache; /* 0x0260 */
* looked up by other CPUs. __u8 reserved[0x0700 - 0x0278]; /* 0x0278 */
*/
unsigned long flags; /* 0x0260 */
__u8 reserved[0x0700 - 0x0268]; /* 0x0268 */
struct kvm_s390_crypto_cb crycb; /* 0x0700 */ struct kvm_s390_crypto_cb crycb; /* 0x0700 */
__u8 fac[S390_ARCH_FAC_LIST_SIZE_BYTE]; /* 0x0800 */ __u8 fac[S390_ARCH_FAC_LIST_SIZE_BYTE]; /* 0x0800 */
}; };
/** static_assert(sizeof(struct vsie_page) == PAGE_SIZE);
* gmap_shadow_valid() - check if a shadow guest address space matches the
* given properties and is still valid
* @sg: pointer to the shadow guest address space structure
* @asce: ASCE for which the shadow table is requested
* @edat_level: edat level to be used for the shadow translation
*
* Returns 1 if the gmap shadow is still valid and matches the given
* properties, the caller can continue using it. Returns 0 otherwise; the
* caller has to request a new shadow gmap in this case.
*/
int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level)
{
if (sg->removed)
return 0;
return sg->orig_asce == asce && sg->edat_level == edat_level;
}
/* trigger a validity icpt for the given scb */ /* trigger a validity icpt for the given scb */
static int set_validity_icpt(struct kvm_s390_sie_block *scb, static int set_validity_icpt(struct kvm_s390_sie_block *scb,
@ -612,26 +596,17 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
return rc; return rc;
} }
void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, unsigned long start, void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, gpa_t start, gpa_t end)
unsigned long end)
{ {
struct kvm *kvm = gmap->private; struct vsie_page *cur, *next;
struct vsie_page *cur;
unsigned long prefix; unsigned long prefix;
int i;
if (!gmap_is_shadow(gmap)) KVM_BUG_ON(!test_bit(GMAP_FLAG_SHADOW, &gmap->flags), gmap->kvm);
return;
/* /*
* Only new shadow blocks are added to the list during runtime, * Only new shadow blocks are added to the list during runtime,
* therefore we can safely reference them all the time. * therefore we can safely reference them all the time.
*/ */
for (i = 0; i < kvm->arch.vsie.page_count; i++) { list_for_each_entry_safe(cur, next, &gmap->scb_users, gmap_cache.list) {
cur = READ_ONCE(kvm->arch.vsie.pages[i]);
if (!cur)
continue;
if (READ_ONCE(cur->gmap) != gmap)
continue;
prefix = cur->scb_s.prefix << GUEST_PREFIX_SHIFT; prefix = cur->scb_s.prefix << GUEST_PREFIX_SHIFT;
/* with mso/msl, the prefix lies at an offset */ /* with mso/msl, the prefix lies at an offset */
prefix += cur->scb_s.mso; prefix += cur->scb_s.mso;
@ -667,9 +642,9 @@ static int map_prefix(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct
/* with mso/msl, the prefix lies at offset *mso* */ /* with mso/msl, the prefix lies at offset *mso* */
prefix += scb_s->mso; prefix += scb_s->mso;
rc = kvm_s390_shadow_fault(vcpu, sg, prefix, NULL); rc = gaccess_shadow_fault(vcpu, sg, prefix, NULL, true);
if (!rc && (scb_s->ecb & ECB_TE)) if (!rc && (scb_s->ecb & ECB_TE))
rc = kvm_s390_shadow_fault(vcpu, sg, prefix + PAGE_SIZE, NULL); rc = gaccess_shadow_fault(vcpu, sg, prefix + PAGE_SIZE, NULL, true);
/* /*
* We don't have to mprotect, we will be called for all unshadows. * We don't have to mprotect, we will be called for all unshadows.
* SIE will detect if protection applies and trigger a validity. * SIE will detect if protection applies and trigger a validity.
@ -952,6 +927,7 @@ static int inject_fault(struct kvm_vcpu *vcpu, __u16 code, __u64 vaddr,
*/ */
static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg) static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
{ {
bool wr = kvm_s390_cur_gmap_fault_is_write();
int rc; int rc;
if ((current->thread.gmap_int_code & PGM_INT_CODE_MASK) == PGM_PROTECTION) if ((current->thread.gmap_int_code & PGM_INT_CODE_MASK) == PGM_PROTECTION)
@ -959,11 +935,10 @@ static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, stru
return inject_fault(vcpu, PGM_PROTECTION, return inject_fault(vcpu, PGM_PROTECTION,
current->thread.gmap_teid.addr * PAGE_SIZE, 1); current->thread.gmap_teid.addr * PAGE_SIZE, 1);
rc = kvm_s390_shadow_fault(vcpu, sg, current->thread.gmap_teid.addr * PAGE_SIZE, NULL); rc = gaccess_shadow_fault(vcpu, sg, current->thread.gmap_teid.addr * PAGE_SIZE, NULL, wr);
if (rc > 0) { if (rc > 0) {
rc = inject_fault(vcpu, rc, rc = inject_fault(vcpu, rc,
current->thread.gmap_teid.addr * PAGE_SIZE, current->thread.gmap_teid.addr * PAGE_SIZE, wr);
kvm_s390_cur_gmap_fault_is_write());
if (rc >= 0) if (rc >= 0)
vsie_page->fault_addr = current->thread.gmap_teid.addr * PAGE_SIZE; vsie_page->fault_addr = current->thread.gmap_teid.addr * PAGE_SIZE;
} }
@ -979,7 +954,7 @@ static int handle_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, stru
static void handle_last_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg) static void handle_last_fault(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
{ {
if (vsie_page->fault_addr) if (vsie_page->fault_addr)
kvm_s390_shadow_fault(vcpu, sg, vsie_page->fault_addr, NULL); gaccess_shadow_fault(vcpu, sg, vsie_page->fault_addr, NULL, true);
vsie_page->fault_addr = 0; vsie_page->fault_addr = 0;
} }
@ -1064,8 +1039,9 @@ static u64 vsie_get_register(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,
static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg) static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struct gmap *sg)
{ {
struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
unsigned long pei_dest, pei_src, src, dest, mask, prefix; unsigned long src, dest, mask, prefix;
u64 *pei_block = &vsie_page->scb_o->mcic; u64 *pei_block = &vsie_page->scb_o->mcic;
union mvpg_pei pei_dest, pei_src;
int edat, rc_dest, rc_src; int edat, rc_dest, rc_src;
union ctlreg0 cr0; union ctlreg0 cr0;
@ -1079,8 +1055,8 @@ static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,
src = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 16) & mask; src = vsie_get_register(vcpu, vsie_page, scb_s->ipb >> 16) & mask;
src = _kvm_s390_real_to_abs(prefix, src) + scb_s->mso; src = _kvm_s390_real_to_abs(prefix, src) + scb_s->mso;
rc_dest = kvm_s390_shadow_fault(vcpu, sg, dest, &pei_dest); rc_dest = gaccess_shadow_fault(vcpu, sg, dest, &pei_dest, true);
rc_src = kvm_s390_shadow_fault(vcpu, sg, src, &pei_src); rc_src = gaccess_shadow_fault(vcpu, sg, src, &pei_src, false);
/* /*
* Either everything went well, or something non-critical went wrong * Either everything went well, or something non-critical went wrong
* e.g. because of a race. In either case, simply retry. * e.g. because of a race. In either case, simply retry.
@ -1115,8 +1091,8 @@ static int vsie_handle_mvpg(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page,
rc_src = rc_src != PGM_PAGE_TRANSLATION ? rc_src : 0; rc_src = rc_src != PGM_PAGE_TRANSLATION ? rc_src : 0;
} }
if (!rc_dest && !rc_src) { if (!rc_dest && !rc_src) {
pei_block[0] = pei_dest; pei_block[0] = pei_dest.val;
pei_block[1] = pei_src; pei_block[1] = pei_src.val;
return 1; return 1;
} }
@ -1187,7 +1163,7 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struc
goto xfer_to_guest_mode_check; goto xfer_to_guest_mode_check;
} }
guest_timing_enter_irqoff(); guest_timing_enter_irqoff();
rc = kvm_s390_enter_exit_sie(scb_s, vcpu->run->s.regs.gprs, sg->asce); rc = kvm_s390_enter_exit_sie(scb_s, vcpu->run->s.regs.gprs, sg->asce.val);
guest_timing_exit_irqoff(); guest_timing_exit_irqoff();
local_irq_enable(); local_irq_enable();
} }
@ -1237,43 +1213,63 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, struc
static void release_gmap_shadow(struct vsie_page *vsie_page) static void release_gmap_shadow(struct vsie_page *vsie_page)
{ {
if (vsie_page->gmap) struct gmap *gmap = vsie_page->gmap_cache.gmap;
gmap_put(vsie_page->gmap);
WRITE_ONCE(vsie_page->gmap, NULL); lockdep_assert_held(&gmap->kvm->arch.gmap->children_lock);
list_del(&vsie_page->gmap_cache.list);
vsie_page->gmap_cache.gmap = NULL;
prefix_unmapped(vsie_page); prefix_unmapped(vsie_page);
if (list_empty(&gmap->scb_users)) {
gmap_remove_child(gmap);
gmap_put(gmap);
}
} }
static int acquire_gmap_shadow(struct kvm_vcpu *vcpu, static struct gmap *acquire_gmap_shadow(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
struct vsie_page *vsie_page)
{ {
unsigned long asce;
union ctlreg0 cr0; union ctlreg0 cr0;
struct gmap *gmap; struct gmap *gmap;
union asce asce;
int edat; int edat;
asce = vcpu->arch.sie_block->gcr[1]; asce.val = vcpu->arch.sie_block->gcr[1];
cr0.val = vcpu->arch.sie_block->gcr[0]; cr0.val = vcpu->arch.sie_block->gcr[0];
edat = cr0.edat && test_kvm_facility(vcpu->kvm, 8); edat = cr0.edat && test_kvm_facility(vcpu->kvm, 8);
edat += edat && test_kvm_facility(vcpu->kvm, 78); edat += edat && test_kvm_facility(vcpu->kvm, 78);
/* scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) {
* ASCE or EDAT could have changed since last icpt, or the gmap gmap = vsie_page->gmap_cache.gmap;
* we're holding has been unshadowed. If the gmap is still valid, if (gmap) {
* we can safely reuse it. /*
*/ * ASCE or EDAT could have changed since last icpt, or the gmap
if (vsie_page->gmap && gmap_shadow_valid(vsie_page->gmap, asce, edat)) { * we're holding has been unshadowed. If the gmap is still valid,
vcpu->kvm->stat.gmap_shadow_reuse++; * we can safely reuse it.
return 0; */
if (gmap_is_shadow_valid(gmap, asce, edat)) {
vcpu->kvm->stat.gmap_shadow_reuse++;
gmap_get(gmap);
return gmap;
}
/* release the old shadow and mark the prefix as unmapped */
release_gmap_shadow(vsie_page);
}
} }
gmap = gmap_create_shadow(vcpu->arch.mc, vcpu->kvm->arch.gmap, asce, edat);
/* release the old shadow - if any, and mark the prefix as unmapped */
release_gmap_shadow(vsie_page);
gmap = gmap_shadow(vcpu->arch.gmap, asce, edat);
if (IS_ERR(gmap)) if (IS_ERR(gmap))
return PTR_ERR(gmap); return gmap;
vcpu->kvm->stat.gmap_shadow_create++; scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) {
WRITE_ONCE(vsie_page->gmap, gmap); /* unlikely race condition, remove the previous shadow */
return 0; if (vsie_page->gmap_cache.gmap)
release_gmap_shadow(vsie_page);
vcpu->kvm->stat.gmap_shadow_create++;
list_add(&vsie_page->gmap_cache.list, &gmap->scb_users);
vsie_page->gmap_cache.gmap = gmap;
prefix_unmapped(vsie_page);
gmap_get(gmap);
}
return gmap;
} }
/* /*
@ -1330,8 +1326,11 @@ static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
int rc = 0; int rc = 0;
while (1) { while (1) {
rc = acquire_gmap_shadow(vcpu, vsie_page); sg = acquire_gmap_shadow(vcpu, vsie_page);
sg = vsie_page->gmap; if (IS_ERR(sg)) {
rc = PTR_ERR(sg);
sg = NULL;
}
if (!rc) if (!rc)
rc = map_prefix(vcpu, vsie_page, sg); rc = map_prefix(vcpu, vsie_page, sg);
if (!rc) { if (!rc) {
@ -1359,6 +1358,9 @@ static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
kvm_s390_rewind_psw(vcpu, 4); kvm_s390_rewind_psw(vcpu, 4);
break; break;
} }
if (sg)
sg = gmap_put(sg);
cond_resched();
} }
if (rc == -EFAULT) { if (rc == -EFAULT) {
@ -1455,8 +1457,7 @@ static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr)
vsie_page->scb_gpa = ULONG_MAX; vsie_page->scb_gpa = ULONG_MAX;
/* Double use of the same address or allocation failure. */ /* Double use of the same address or allocation failure. */
if (radix_tree_insert(&kvm->arch.vsie.addr_to_page, addr >> 9, if (radix_tree_insert(&kvm->arch.vsie.addr_to_page, addr >> 9, vsie_page)) {
vsie_page)) {
put_vsie_page(vsie_page); put_vsie_page(vsie_page);
mutex_unlock(&kvm->arch.vsie.mutex); mutex_unlock(&kvm->arch.vsie.mutex);
return NULL; return NULL;
@ -1465,7 +1466,12 @@ static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr)
mutex_unlock(&kvm->arch.vsie.mutex); mutex_unlock(&kvm->arch.vsie.mutex);
memset(&vsie_page->scb_s, 0, sizeof(struct kvm_s390_sie_block)); memset(&vsie_page->scb_s, 0, sizeof(struct kvm_s390_sie_block));
release_gmap_shadow(vsie_page); if (vsie_page->gmap_cache.gmap) {
scoped_guard(spinlock, &kvm->arch.gmap->children_lock)
if (vsie_page->gmap_cache.gmap)
release_gmap_shadow(vsie_page);
}
prefix_unmapped(vsie_page);
vsie_page->fault_addr = 0; vsie_page->fault_addr = 0;
vsie_page->scb_s.ihcpu = 0xffffU; vsie_page->scb_s.ihcpu = 0xffffU;
return vsie_page; return vsie_page;
@ -1543,8 +1549,10 @@ void kvm_s390_vsie_destroy(struct kvm *kvm)
mutex_lock(&kvm->arch.vsie.mutex); mutex_lock(&kvm->arch.vsie.mutex);
for (i = 0; i < kvm->arch.vsie.page_count; i++) { for (i = 0; i < kvm->arch.vsie.page_count; i++) {
vsie_page = kvm->arch.vsie.pages[i]; vsie_page = kvm->arch.vsie.pages[i];
scoped_guard(spinlock, &kvm->arch.gmap->children_lock)
if (vsie_page->gmap_cache.gmap)
release_gmap_shadow(vsie_page);
kvm->arch.vsie.pages[i] = NULL; kvm->arch.vsie.pages[i] = NULL;
release_gmap_shadow(vsie_page);
/* free the radix tree entry */ /* free the radix tree entry */
if (vsie_page->scb_gpa != ULONG_MAX) if (vsie_page->scb_gpa != ULONG_MAX)
radix_tree_delete(&kvm->arch.vsie.addr_to_page, radix_tree_delete(&kvm->arch.vsie.addr_to_page,

184
arch/s390/lib/uaccess.c
View File

@ -34,136 +34,19 @@ void debug_user_asce(int exit)
} }
#endif /*CONFIG_DEBUG_ENTRY */ #endif /*CONFIG_DEBUG_ENTRY */
union oac {
unsigned int val;
struct {
struct {
unsigned short key : 4;
unsigned short : 4;
unsigned short as : 2;
unsigned short : 4;
unsigned short k : 1;
unsigned short a : 1;
} oac1;
struct {
unsigned short key : 4;
unsigned short : 4;
unsigned short as : 2;
unsigned short : 4;
unsigned short k : 1;
unsigned short a : 1;
} oac2;
};
};
static uaccess_kmsan_or_inline __must_check unsigned long
raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key)
{
unsigned long osize;
union oac spec = {
.oac2.key = key,
.oac2.as = PSW_BITS_AS_SECONDARY,
.oac2.k = 1,
.oac2.a = 1,
};
int cc;
while (1) {
osize = size;
asm_inline volatile(
" lr %%r0,%[spec]\n"
"0: mvcos %[to],%[from],%[size]\n"
"1: nopr %%r7\n"
CC_IPM(cc)
EX_TABLE_UA_MVCOS_FROM(0b, 0b)
EX_TABLE_UA_MVCOS_FROM(1b, 0b)
: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to)
: [spec] "d" (spec.val), [from] "Q" (*(const char __user *)from)
: CC_CLOBBER_LIST("memory", "0"));
if (CC_TRANSFORM(cc) == 0)
return osize - size;
size -= 4096;
to += 4096;
from += 4096;
}
}
unsigned long _copy_from_user_key(void *to, const void __user *from,
unsigned long n, unsigned long key)
{
unsigned long res = n;
might_fault();
if (!should_fail_usercopy()) {
instrument_copy_from_user_before(to, from, n);
res = raw_copy_from_user_key(to, from, n, key);
instrument_copy_from_user_after(to, from, n, res);
}
if (unlikely(res))
memset(to + (n - res), 0, res);
return res;
}
EXPORT_SYMBOL(_copy_from_user_key);
static uaccess_kmsan_or_inline __must_check unsigned long
raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key)
{
unsigned long osize;
union oac spec = {
.oac1.key = key,
.oac1.as = PSW_BITS_AS_SECONDARY,
.oac1.k = 1,
.oac1.a = 1,
};
int cc;
while (1) {
osize = size;
asm_inline volatile(
" lr %%r0,%[spec]\n"
"0: mvcos %[to],%[from],%[size]\n"
"1: nopr %%r7\n"
CC_IPM(cc)
EX_TABLE_UA_MVCOS_TO(0b, 0b)
EX_TABLE_UA_MVCOS_TO(1b, 0b)
: CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to)
: [spec] "d" (spec.val), [from] "Q" (*(const char *)from)
: CC_CLOBBER_LIST("memory", "0"));
if (CC_TRANSFORM(cc) == 0)
return osize - size;
size -= 4096;
to += 4096;
from += 4096;
}
}
unsigned long _copy_to_user_key(void __user *to, const void *from,
unsigned long n, unsigned long key)
{
might_fault();
if (should_fail_usercopy())
return n;
instrument_copy_to_user(to, from, n);
return raw_copy_to_user_key(to, from, n, key);
}
EXPORT_SYMBOL(_copy_to_user_key);
#define CMPXCHG_USER_KEY_MAX_LOOPS 128 #define CMPXCHG_USER_KEY_MAX_LOOPS 128
static nokprobe_inline int __cmpxchg_user_key_small(unsigned long address, unsigned int *uval, static nokprobe_inline int __cmpxchg_key_small(void *address, unsigned int *uval,
unsigned int old, unsigned int new, unsigned int old, unsigned int new,
unsigned int mask, unsigned long key) unsigned int mask, unsigned long key)
{ {
unsigned long count; unsigned long count;
unsigned int prev; unsigned int prev;
bool sacf_flag;
int rc = 0; int rc = 0;
skey_regions_initialize(); skey_regions_initialize();
sacf_flag = enable_sacf_uaccess();
asm_inline volatile( asm_inline volatile(
"20: spka 0(%[key])\n" "20: spka 0(%[key])\n"
" sacf 256\n"
" llill %[count],%[max_loops]\n" " llill %[count],%[max_loops]\n"
"0: l %[prev],%[address]\n" "0: l %[prev],%[address]\n"
"1: nr %[prev],%[mask]\n" "1: nr %[prev],%[mask]\n"
@ -178,8 +61,7 @@ static nokprobe_inline int __cmpxchg_user_key_small(unsigned long address, unsig
" nr %[tmp],%[mask]\n" " nr %[tmp],%[mask]\n"
" jnz 5f\n" " jnz 5f\n"
" brct %[count],2b\n" " brct %[count],2b\n"
"5: sacf 768\n" "5: spka %[default_key]\n"
" spka %[default_key]\n"
"21:\n" "21:\n"
EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
@ -197,16 +79,16 @@ static nokprobe_inline int __cmpxchg_user_key_small(unsigned long address, unsig
[default_key] "J" (PAGE_DEFAULT_KEY), [default_key] "J" (PAGE_DEFAULT_KEY),
[max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS) [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS)
: "memory", "cc"); : "memory", "cc");
disable_sacf_uaccess(sacf_flag);
*uval = prev; *uval = prev;
if (!count) if (!count)
rc = -EAGAIN; rc = -EAGAIN;
return rc; return rc;
} }
int __kprobes __cmpxchg_user_key1(unsigned long address, unsigned char *uval, int __kprobes __cmpxchg_key1(void *addr, unsigned char *uval, unsigned char old,
unsigned char old, unsigned char new, unsigned long key) unsigned char new, unsigned long key)
{ {
unsigned long address = (unsigned long)addr;
unsigned int prev, shift, mask, _old, _new; unsigned int prev, shift, mask, _old, _new;
int rc; int rc;
@ -215,15 +97,16 @@ int __kprobes __cmpxchg_user_key1(unsigned long address, unsigned char *uval,
_old = (unsigned int)old << shift; _old = (unsigned int)old << shift;
_new = (unsigned int)new << shift; _new = (unsigned int)new << shift;
mask = ~(0xff << shift); mask = ~(0xff << shift);
rc = __cmpxchg_user_key_small(address, &prev, _old, _new, mask, key); rc = __cmpxchg_key_small((void *)address, &prev, _old, _new, mask, key);
*uval = prev >> shift; *uval = prev >> shift;
return rc; return rc;
} }
EXPORT_SYMBOL(__cmpxchg_user_key1); EXPORT_SYMBOL(__cmpxchg_key1);
int __kprobes __cmpxchg_user_key2(unsigned long address, unsigned short *uval, int __kprobes __cmpxchg_key2(void *addr, unsigned short *uval, unsigned short old,
unsigned short old, unsigned short new, unsigned long key) unsigned short new, unsigned long key)
{ {
unsigned long address = (unsigned long)addr;
unsigned int prev, shift, mask, _old, _new; unsigned int prev, shift, mask, _old, _new;
int rc; int rc;
@ -232,27 +115,23 @@ int __kprobes __cmpxchg_user_key2(unsigned long address, unsigned short *uval,
_old = (unsigned int)old << shift; _old = (unsigned int)old << shift;
_new = (unsigned int)new << shift; _new = (unsigned int)new << shift;
mask = ~(0xffff << shift); mask = ~(0xffff << shift);
rc = __cmpxchg_user_key_small(address, &prev, _old, _new, mask, key); rc = __cmpxchg_key_small((void *)address, &prev, _old, _new, mask, key);
*uval = prev >> shift; *uval = prev >> shift;
return rc; return rc;
} }
EXPORT_SYMBOL(__cmpxchg_user_key2); EXPORT_SYMBOL(__cmpxchg_key2);
int __kprobes __cmpxchg_user_key4(unsigned long address, unsigned int *uval, int __kprobes __cmpxchg_key4(void *address, unsigned int *uval, unsigned int old,
unsigned int old, unsigned int new, unsigned long key) unsigned int new, unsigned long key)
{ {
unsigned int prev = old; unsigned int prev = old;
bool sacf_flag;
int rc = 0; int rc = 0;
skey_regions_initialize(); skey_regions_initialize();
sacf_flag = enable_sacf_uaccess();
asm_inline volatile( asm_inline volatile(
"20: spka 0(%[key])\n" "20: spka 0(%[key])\n"
" sacf 256\n"
"0: cs %[prev],%[new],%[address]\n" "0: cs %[prev],%[new],%[address]\n"
"1: sacf 768\n" "1: spka %[default_key]\n"
" spka %[default_key]\n"
"21:\n" "21:\n"
EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
@ -264,27 +143,22 @@ int __kprobes __cmpxchg_user_key4(unsigned long address, unsigned int *uval,
[key] "a" (key << 4), [key] "a" (key << 4),
[default_key] "J" (PAGE_DEFAULT_KEY) [default_key] "J" (PAGE_DEFAULT_KEY)
: "memory", "cc"); : "memory", "cc");
disable_sacf_uaccess(sacf_flag);
*uval = prev; *uval = prev;
return rc; return rc;
} }
EXPORT_SYMBOL(__cmpxchg_user_key4); EXPORT_SYMBOL(__cmpxchg_key4);
int __kprobes __cmpxchg_user_key8(unsigned long address, unsigned long *uval, int __kprobes __cmpxchg_key8(void *address, unsigned long *uval, unsigned long old,
unsigned long old, unsigned long new, unsigned long key) unsigned long new, unsigned long key)
{ {
unsigned long prev = old; unsigned long prev = old;
bool sacf_flag;
int rc = 0; int rc = 0;
skey_regions_initialize(); skey_regions_initialize();
sacf_flag = enable_sacf_uaccess();
asm_inline volatile( asm_inline volatile(
"20: spka 0(%[key])\n" "20: spka 0(%[key])\n"
" sacf 256\n"
"0: csg %[prev],%[new],%[address]\n" "0: csg %[prev],%[new],%[address]\n"
"1: sacf 768\n" "1: spka %[default_key]\n"
" spka %[default_key]\n"
"21:\n" "21:\n"
EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
@ -296,27 +170,22 @@ int __kprobes __cmpxchg_user_key8(unsigned long address, unsigned long *uval,
[key] "a" (key << 4), [key] "a" (key << 4),
[default_key] "J" (PAGE_DEFAULT_KEY) [default_key] "J" (PAGE_DEFAULT_KEY)
: "memory", "cc"); : "memory", "cc");
disable_sacf_uaccess(sacf_flag);
*uval = prev; *uval = prev;
return rc; return rc;
} }
EXPORT_SYMBOL(__cmpxchg_user_key8); EXPORT_SYMBOL(__cmpxchg_key8);
int __kprobes __cmpxchg_user_key16(unsigned long address, __uint128_t *uval, int __kprobes __cmpxchg_key16(void *address, __uint128_t *uval, __uint128_t old,
__uint128_t old, __uint128_t new, unsigned long key) __uint128_t new, unsigned long key)
{ {
__uint128_t prev = old; __uint128_t prev = old;
bool sacf_flag;
int rc = 0; int rc = 0;
skey_regions_initialize(); skey_regions_initialize();
sacf_flag = enable_sacf_uaccess();
asm_inline volatile( asm_inline volatile(
"20: spka 0(%[key])\n" "20: spka 0(%[key])\n"
" sacf 256\n"
"0: cdsg %[prev],%[new],%[address]\n" "0: cdsg %[prev],%[new],%[address]\n"
"1: sacf 768\n" "1: spka %[default_key]\n"
" spka %[default_key]\n"
"21:\n" "21:\n"
EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev])
EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev])
@ -328,8 +197,7 @@ int __kprobes __cmpxchg_user_key16(unsigned long address, __uint128_t *uval,
[key] "a" (key << 4), [key] "a" (key << 4),
[default_key] "J" (PAGE_DEFAULT_KEY) [default_key] "J" (PAGE_DEFAULT_KEY)
: "memory", "cc"); : "memory", "cc");
disable_sacf_uaccess(sacf_flag);
*uval = prev; *uval = prev;
return rc; return rc;
} }
EXPORT_SYMBOL(__cmpxchg_user_key16); EXPORT_SYMBOL(__cmpxchg_key16);

38
arch/s390/mm/gmap_helpers.c
View File

@ -34,28 +34,6 @@ static void ptep_zap_softleaf_entry(struct mm_struct *mm, softleaf_t entry)
free_swap_and_cache(entry); free_swap_and_cache(entry);
} }
static inline pgste_t pgste_get_lock(pte_t *ptep)
{
unsigned long value = 0;
#ifdef CONFIG_PGSTE
unsigned long *ptr = (unsigned long *)(ptep + PTRS_PER_PTE);
do {
value = __atomic64_or_barrier(PGSTE_PCL_BIT, ptr);
} while (value & PGSTE_PCL_BIT);
value |= PGSTE_PCL_BIT;
#endif
return __pgste(value);
}
static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
{
#ifdef CONFIG_PGSTE
barrier();
WRITE_ONCE(*(unsigned long *)(ptep + PTRS_PER_PTE), pgste_val(pgste) & ~PGSTE_PCL_BIT);
#endif
}
/** /**
* gmap_helper_zap_one_page() - discard a page if it was swapped. * gmap_helper_zap_one_page() - discard a page if it was swapped.
* @mm: the mm * @mm: the mm
@ -68,9 +46,7 @@ static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr) void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr)
{ {
struct vm_area_struct *vma; struct vm_area_struct *vma;
unsigned long pgstev;
spinlock_t *ptl; spinlock_t *ptl;
pgste_t pgste;
pte_t *ptep; pte_t *ptep;
mmap_assert_locked(mm); mmap_assert_locked(mm);
@ -85,18 +61,8 @@ void gmap_helper_zap_one_page(struct mm_struct *mm, unsigned long vmaddr)
if (unlikely(!ptep)) if (unlikely(!ptep))
return; return;
if (pte_swap(*ptep)) { if (pte_swap(*ptep)) {
preempt_disable(); ptep_zap_softleaf_entry(mm, softleaf_from_pte(*ptep));
pgste = pgste_get_lock(ptep); pte_clear(mm, vmaddr, ptep);
pgstev = pgste_val(pgste);
if ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED ||
(pgstev & _PGSTE_GPS_ZERO)) {
ptep_zap_softleaf_entry(mm, softleaf_from_pte(*ptep));
pte_clear(mm, vmaddr, ptep);
}
pgste_set_unlock(ptep, pgste);
preempt_enable();
} }
pte_unmap_unlock(ptep, ptl); pte_unmap_unlock(ptep, ptl);
} }