mirror of
https://github.com/torvalds/linux.git
synced 2026-11-07 06:17:02 +01:00
Merge branch kvm-arm64/pre-faulting into kvmarm-master/next
* kvm-arm64/pre-faulting: : \ : Enable stage-2 pre-faulting in the canonical IPA space, implementing the : KVM_PRE_FAULT_MEMORY API. Patches courtesy of Lorenzo Stoakes, and based : on an initial work by Jack Thomson. : / KVM: selftests: Add nested pre-fault test for arm64 KVM: selftests: Add option for different backing in pre-fault tests KVM: selftests: Enable pre_fault_memory_test for arm64 Documentation: KVM: document arm64 KVM_PRE_FAULT_MEMORY KVM: arm64: Implement KVM_PRE_FAULT_MEMORY KVM: arm64: Pass walk flags to kvm_pgtable_get_leaf() KVM: arm64: Propagate EHWPOISON in kvm_s2_fault_pin_pfn() KVM: arm64: Size the stage-2 memcache from the fault MMU KVM: arm64: Propagate and use kvm_s2_fault_result on S2 fault KVM: arm64: Propagate and use mmu in s2fd when handling guest aborts KVM: arm64: Propagate and use esr in s2fd when handling guest aborts KVM: arm64: Use ESR helpers in guest abort handling arm64: Add ESR fault helpers KVM: Allow architectures to disallow pre-fault Signed-off-by: Marc Zyngier <maz@kernel.org>
This commit is contained in:
Marc Zyngier
commit
62449bbaef
|
|
@ -6489,7 +6489,7 @@ See KVM_SET_USER_MEMORY_REGION2 for additional details.
|
|||
---------------------------
|
||||
|
||||
:Capability: KVM_CAP_PRE_FAULT_MEMORY
|
||||
:Architectures: none
|
||||
:Architectures: x86, s390, arm64
|
||||
:Type: vcpu ioctl
|
||||
:Parameters: struct kvm_pre_fault_memory (in/out)
|
||||
:Returns: 0 if at least one page is processed, < 0 on error
|
||||
|
|
@ -6497,12 +6497,15 @@ See KVM_SET_USER_MEMORY_REGION2 for additional details.
|
|||
Errors:
|
||||
|
||||
========== ===============================================================
|
||||
EAGAIN A race occurred before progress was made, but a retry may succeed.
|
||||
EINVAL The specified `gpa` and `size` were invalid (e.g. not
|
||||
page aligned, causes an overflow, or size is zero), or the VM
|
||||
is UCONTROL (s390).
|
||||
ENOENT The specified `gpa` is outside defined memslots.
|
||||
ENOEXEC The vCPU has not been initialised (arm64).
|
||||
EINTR An unmasked signal is pending and no page was processed.
|
||||
EFAULT The parameter address was invalid.
|
||||
EHWPOISON A poisoned host page was encountered.
|
||||
EOPNOTSUPP Mapping memory for a GPA is unsupported by the
|
||||
hypervisor, and/or for the current vCPU state/mode.
|
||||
EIO unexpected error conditions (also causes a WARN)
|
||||
|
|
@ -6522,7 +6525,17 @@ Errors:
|
|||
KVM_PRE_FAULT_MEMORY populates KVM's stage-2 page tables used to map memory
|
||||
for the current vCPU state. KVM maps memory as if the vCPU generated a
|
||||
stage-2 read page fault, e.g. faults in memory as needed, but doesn't break
|
||||
CoW. On x86, KVM does not mark any newly created stage-2 PTE as Accessed.
|
||||
CoW. On arm64, KVM marks newly created stage-2 PTEs as Accessed, as it
|
||||
does for any stage-2 fault, but leaves the Accessed state of existing PTEs
|
||||
unchanged. On x86, KVM does not mark any newly created stage-2 PTE as
|
||||
Accessed, and for s390 it is not applicable.
|
||||
|
||||
On arm64, a GPA is interpreted as an IPA, and never interpreted as the IPA
|
||||
of a nested guest. Pre-faulting only populates canonical stage-2 page
|
||||
tables.
|
||||
|
||||
The feature is not supported on arm64 if the protected KVM (pKVM) feature
|
||||
is enabled.
|
||||
|
||||
In the case of confidential VM types where there is an initial set up of
|
||||
private guest memory before the guest is 'finalized'/measured, this ioctl
|
||||
|
|
@ -6537,7 +6550,7 @@ When the ioctl returns, the input values are updated to point to the
|
|||
remaining range. If `size` > 0 on return, the caller can just issue
|
||||
the ioctl again with the same `struct kvm_map_memory` argument.
|
||||
|
||||
Shadow page tables cannot support this ioctl because they
|
||||
On x86, shadow page tables cannot support this ioctl because they
|
||||
are indexed by virtual address or nested guest physical address.
|
||||
Calling this ioctl when the guest is using shadow page tables (for
|
||||
example because it is running a nested guest with nested page tables)
|
||||
|
|
|
|||
|
|
@ -437,6 +437,35 @@
|
|||
#ifndef __ASSEMBLER__
|
||||
#include <asm/types.h>
|
||||
|
||||
static __always_inline bool esr_trap_is_iabt(unsigned long esr)
|
||||
{
|
||||
return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
|
||||
}
|
||||
|
||||
/* Always check for S1PTW *before* using this. */
|
||||
static __always_inline bool esr_dabt_is_write(unsigned long esr)
|
||||
{
|
||||
return esr & ESR_ELx_WNR;
|
||||
}
|
||||
|
||||
static __always_inline bool esr_dabt_is_cm(unsigned long esr)
|
||||
{
|
||||
return esr & ESR_ELx_CM;
|
||||
}
|
||||
|
||||
static __always_inline bool esr_abt_is_sea(unsigned long esr)
|
||||
{
|
||||
switch (esr & ESR_ELx_FSC) {
|
||||
case ESR_ELx_FSC_EXTABT:
|
||||
case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3):
|
||||
case ESR_ELx_FSC_SECC:
|
||||
case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3):
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static inline unsigned long esr_brk_comment(unsigned long esr)
|
||||
{
|
||||
return esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
|
||||
|
|
|
|||
|
|
@ -336,6 +336,16 @@ static __always_inline u64 kvm_vcpu_get_esr(const struct kvm_vcpu *vcpu)
|
|||
return vcpu->arch.fault.esr_el2;
|
||||
}
|
||||
|
||||
static __always_inline bool esr_abt_is_s1ptw(unsigned long esr)
|
||||
{
|
||||
return esr & ESR_ELx_S1PTW;
|
||||
}
|
||||
|
||||
static __always_inline bool esr_abt_is_exec_fault(unsigned long esr)
|
||||
{
|
||||
return esr_trap_is_iabt(esr) && !esr_abt_is_s1ptw(esr);
|
||||
}
|
||||
|
||||
static inline bool guest_hyp_wfx_traps_enabled(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u64 esr = kvm_vcpu_get_esr(vcpu);
|
||||
|
|
@ -411,18 +421,13 @@ static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
|
|||
|
||||
static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW);
|
||||
return esr_abt_is_s1ptw(kvm_vcpu_get_esr(vcpu));
|
||||
}
|
||||
|
||||
/* Always check for S1PTW *before* using this. */
|
||||
static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR;
|
||||
}
|
||||
|
||||
static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_CM);
|
||||
return esr_dabt_is_write(kvm_vcpu_get_esr(vcpu));
|
||||
}
|
||||
|
||||
static __always_inline unsigned int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
|
||||
|
|
@ -443,12 +448,7 @@ static __always_inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
|
|||
|
||||
static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
|
||||
}
|
||||
|
||||
static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu);
|
||||
return esr_trap_is_iabt(kvm_vcpu_get_esr(vcpu));
|
||||
}
|
||||
|
||||
static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
|
||||
|
|
@ -468,26 +468,9 @@ bool kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu *vcpu)
|
|||
return esr_fsc_is_translation_fault(kvm_vcpu_get_esr(vcpu));
|
||||
}
|
||||
|
||||
static inline
|
||||
u64 kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
unsigned long esr = kvm_vcpu_get_esr(vcpu);
|
||||
|
||||
BUG_ON(!esr_fsc_is_permission_fault(esr));
|
||||
return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(esr & ESR_ELx_FSC_LEVEL));
|
||||
}
|
||||
|
||||
static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
switch (kvm_vcpu_trap_get_fault(vcpu)) {
|
||||
case ESR_ELx_FSC_EXTABT:
|
||||
case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3):
|
||||
case ESR_ELx_FSC_SECC:
|
||||
case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3):
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
return esr_abt_is_sea(kvm_vcpu_get_esr(vcpu));
|
||||
}
|
||||
|
||||
static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
|
||||
|
|
@ -496,9 +479,9 @@ static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
|
|||
return ESR_ELx_SYS64_ISS_RT(esr);
|
||||
}
|
||||
|
||||
static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
|
||||
static inline bool esr_abt_is_write_fault(unsigned long esr)
|
||||
{
|
||||
if (kvm_vcpu_abt_iss1tw(vcpu)) {
|
||||
if (esr_abt_is_s1ptw(esr)) {
|
||||
/*
|
||||
* Only a permission fault on a S1PTW should be
|
||||
* considered as a write. Otherwise, page tables baked
|
||||
|
|
@ -511,13 +494,18 @@ static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
|
|||
* first), then a permission fault to allow the flags
|
||||
* to be set.
|
||||
*/
|
||||
return kvm_vcpu_trap_is_permission_fault(vcpu);
|
||||
return esr_fsc_is_permission_fault(esr);
|
||||
}
|
||||
|
||||
if (kvm_vcpu_trap_is_iabt(vcpu))
|
||||
if (esr_trap_is_iabt(esr))
|
||||
return false;
|
||||
|
||||
return kvm_vcpu_dabt_iswrite(vcpu);
|
||||
return esr_dabt_is_write(esr);
|
||||
}
|
||||
|
||||
static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return esr_abt_is_write_fault(kvm_vcpu_get_esr(vcpu));
|
||||
}
|
||||
|
||||
static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
|
||||
|
|
|
|||
|
|
@ -859,6 +859,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
|
|||
* @addr: Input address for the start of the walk.
|
||||
* @ptep: Pointer to storage for the retrieved PTE.
|
||||
* @level: Pointer to storage for the level of the retrieved PTE.
|
||||
* @flags: Flags to control the page-table walk
|
||||
* (see struct kvm_pgtable_visit_ctx).
|
||||
*
|
||||
* The offset of @addr within a page is ignored.
|
||||
*
|
||||
|
|
@ -869,7 +871,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
|
|||
* Return: 0 on success, negative error code on failure.
|
||||
*/
|
||||
int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,
|
||||
kvm_pte_t *ptep, s8 *level);
|
||||
kvm_pte_t *ptep, s8 *level,
|
||||
enum kvm_pgtable_walk_flags flags);
|
||||
|
||||
/**
|
||||
* kvm_pgtable_stage2_pte_prot() - Retrieve the protection attributes of a
|
||||
|
|
|
|||
|
|
@ -44,9 +44,9 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext)
|
|||
case KVM_CAP_ARM_PTRAUTH_GENERIC:
|
||||
return true;
|
||||
case KVM_CAP_ARM_MTE:
|
||||
return false;
|
||||
case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
|
||||
case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES:
|
||||
case KVM_CAP_PRE_FAULT_MEMORY:
|
||||
return false;
|
||||
default:
|
||||
return !kvm || !kvm_vm_is_protected(kvm);
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ menuconfig KVM
|
|||
select SCHED_INFO
|
||||
select GUEST_PERF_EVENTS if PERF_EVENTS
|
||||
select KVM_GUEST_MEMFD
|
||||
select KVM_GENERIC_PRE_FAULT_MEMORY
|
||||
help
|
||||
Support hosting virtualized guest machines.
|
||||
|
||||
|
|
|
|||
|
|
@ -410,6 +410,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
|
|||
case KVM_CAP_COUNTER_OFFSET:
|
||||
case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
|
||||
case KVM_CAP_ARM_SEA_TO_USER:
|
||||
case KVM_CAP_PRE_FAULT_MEMORY:
|
||||
r = 1;
|
||||
break;
|
||||
case KVM_CAP_SET_GUEST_DEBUG2:
|
||||
|
|
|
|||
|
|
@ -540,7 +540,7 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range)
|
|||
int ret;
|
||||
|
||||
hyp_assert_lock_held(&host_mmu.lock);
|
||||
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level);
|
||||
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level, 0);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
|
|
@ -930,7 +930,7 @@ static int get_valid_guest_pte(struct pkvm_hyp_vm *vm, u64 ipa, kvm_pte_t *ptep,
|
|||
s8 level;
|
||||
int ret;
|
||||
|
||||
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
|
||||
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0);
|
||||
if (ret)
|
||||
return ret;
|
||||
if (guest_pte_is_poisoned(pte))
|
||||
|
|
@ -979,7 +979,7 @@ int __pkvm_vcpu_in_poison_fault(struct pkvm_hyp_vcpu *hyp_vcpu)
|
|||
ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(&hyp_vcpu->vcpu));
|
||||
|
||||
guest_lock_component(vm);
|
||||
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
|
||||
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0);
|
||||
if (ret)
|
||||
goto unlock;
|
||||
|
||||
|
|
@ -1335,7 +1335,7 @@ static int host_stage2_get_guest_info(phys_addr_t phys, struct pkvm_hyp_vm **vm,
|
|||
return -EPERM;
|
||||
}
|
||||
|
||||
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level);
|
||||
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level, 0);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
|
|
@ -1564,7 +1564,7 @@ static int __check_host_shared_guest(struct pkvm_hyp_vm *vm, u64 *__phys, u64 ip
|
|||
s8 level;
|
||||
int ret;
|
||||
|
||||
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
|
||||
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0);
|
||||
if (ret)
|
||||
return ret;
|
||||
if (!kvm_pte_valid(pte))
|
||||
|
|
|
|||
|
|
@ -486,7 +486,7 @@ static int check_page_ownership(phys_addr_t phys)
|
|||
return -EPERM;
|
||||
}
|
||||
|
||||
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL);
|
||||
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL, 0);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
|
|
|
|||
|
|
@ -298,12 +298,13 @@ static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx,
|
|||
}
|
||||
|
||||
int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,
|
||||
kvm_pte_t *ptep, s8 *level)
|
||||
kvm_pte_t *ptep, s8 *level,
|
||||
enum kvm_pgtable_walk_flags flags)
|
||||
{
|
||||
struct leaf_walk_data data;
|
||||
struct kvm_pgtable_walker walker = {
|
||||
.cb = leaf_walker,
|
||||
.flags = KVM_PGTABLE_WALK_LEAF,
|
||||
.flags = flags | KVM_PGTABLE_WALK_LEAF,
|
||||
.arg = &data,
|
||||
};
|
||||
int ret;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
|
||||
#include <linux/acpi.h>
|
||||
#include <linux/cleanup.h>
|
||||
#include <linux/mman.h>
|
||||
#include <linux/kvm_host.h>
|
||||
#include <linux/interval_tree.h>
|
||||
|
|
@ -895,7 +896,7 @@ static int get_user_mapping_size(struct kvm *kvm, u64 addr)
|
|||
* IPI-ing threads).
|
||||
*/
|
||||
local_irq_save(flags);
|
||||
ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level);
|
||||
ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level, 0);
|
||||
local_irq_restore(flags);
|
||||
|
||||
if (ret)
|
||||
|
|
@ -1606,9 +1607,9 @@ static void *get_mmu_memcache(struct kvm_vcpu *vcpu)
|
|||
return &vcpu->arch.pkvm_memcache;
|
||||
}
|
||||
|
||||
static int topup_mmu_memcache(struct kvm_vcpu *vcpu, void *memcache)
|
||||
static int topup_mmu_memcache(struct kvm_s2_mmu *mmu, void *memcache)
|
||||
{
|
||||
int min_pages = kvm_mmu_cache_min_pages(vcpu->arch.hw_mmu);
|
||||
int min_pages = kvm_mmu_cache_min_pages(mmu);
|
||||
|
||||
if (!is_protected_kvm_enabled())
|
||||
return kvm_mmu_topup_memory_cache(memcache, min_pages);
|
||||
|
|
@ -1655,15 +1656,47 @@ struct kvm_s2_fault_desc {
|
|||
struct kvm_s2_trans *nested;
|
||||
struct kvm_memory_slot *memslot;
|
||||
unsigned long hva;
|
||||
unsigned long esr;
|
||||
struct kvm_s2_mmu *mmu;
|
||||
};
|
||||
|
||||
static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
|
||||
struct kvm_s2_fault_result {
|
||||
unsigned long mapping_size;
|
||||
};
|
||||
|
||||
static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd)
|
||||
{
|
||||
return esr_fsc_is_permission_fault(s2fd->esr);
|
||||
}
|
||||
|
||||
static bool kvm_s2_fault_is_exec(const struct kvm_s2_fault_desc *s2fd)
|
||||
{
|
||||
return esr_abt_is_exec_fault(s2fd->esr);
|
||||
}
|
||||
|
||||
static bool kvm_s2_fault_is_write(const struct kvm_s2_fault_desc *s2fd)
|
||||
{
|
||||
return esr_abt_is_write_fault(s2fd->esr);
|
||||
}
|
||||
|
||||
static u64 kvm_s2_perm_fault_granule(const struct kvm_s2_fault_desc *s2fd)
|
||||
{
|
||||
u64 level;
|
||||
|
||||
if (!kvm_s2_fault_is_perm(s2fd))
|
||||
return 0;
|
||||
level = s2fd->esr & ESR_ELx_FSC_LEVEL;
|
||||
return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level));
|
||||
}
|
||||
|
||||
static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
|
||||
struct kvm_s2_fault_result *result)
|
||||
{
|
||||
bool write_fault, exec_fault;
|
||||
bool perm_fault = kvm_vcpu_trap_is_permission_fault(s2fd->vcpu);
|
||||
bool perm_fault = kvm_s2_fault_is_perm(s2fd);
|
||||
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED;
|
||||
enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
|
||||
struct kvm_pgtable *pgt = s2fd->vcpu->arch.hw_mmu->pgt;
|
||||
struct kvm_pgtable *pgt = s2fd->mmu->pgt;
|
||||
struct kvm_guest_s2_mapping *mapping = NULL;
|
||||
unsigned long mmu_seq;
|
||||
struct page *page;
|
||||
|
|
@ -1675,7 +1708,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
|
|||
|
||||
if (!perm_fault) {
|
||||
memcache = get_mmu_memcache(s2fd->vcpu);
|
||||
ret = topup_mmu_memcache(s2fd->vcpu, memcache);
|
||||
ret = topup_mmu_memcache(s2fd->mmu, memcache);
|
||||
if (ret)
|
||||
return ret;
|
||||
if (kvm_is_nested_s2_mmu(kvm, pgt->mmu)) {
|
||||
|
|
@ -1690,8 +1723,8 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
|
|||
else
|
||||
gfn = s2fd->fault_ipa >> PAGE_SHIFT;
|
||||
|
||||
write_fault = kvm_is_write_fault(s2fd->vcpu);
|
||||
exec_fault = kvm_vcpu_trap_is_exec_fault(s2fd->vcpu);
|
||||
write_fault = kvm_s2_fault_is_write(s2fd);
|
||||
exec_fault = kvm_s2_fault_is_exec(s2fd);
|
||||
|
||||
VM_WARN_ON_ONCE(write_fault && exec_fault);
|
||||
|
||||
|
|
@ -1701,8 +1734,10 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
|
|||
|
||||
ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, &page, NULL);
|
||||
if (ret) {
|
||||
kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE,
|
||||
write_fault, exec_fault, false);
|
||||
/* If result is non-NULL this is a synthetic fault. */
|
||||
if (!result)
|
||||
kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE,
|
||||
write_fault, exec_fault, false);
|
||||
kfree(mapping);
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -1757,7 +1792,13 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
|
|||
if ((prot & KVM_PGTABLE_PROT_W) && !ret)
|
||||
mark_page_dirty_in_slot(kvm, s2fd->memslot, gfn);
|
||||
|
||||
return ret != -EAGAIN ? ret : 0;
|
||||
if (ret == -EAGAIN)
|
||||
return result ? ret : 0;
|
||||
|
||||
if (result && !ret)
|
||||
result->mapping_size = PAGE_SIZE;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
struct kvm_s2_fault_vma_info {
|
||||
|
|
@ -1779,7 +1820,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
|
|||
{
|
||||
unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE;
|
||||
struct kvm_vcpu *vcpu = s2fd->vcpu;
|
||||
struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt;
|
||||
struct kvm_pgtable *pgt = s2fd->mmu->pgt;
|
||||
struct mm_struct *mm = current->mm;
|
||||
struct kvm *kvm = vcpu->kvm;
|
||||
void *hyp_memcache;
|
||||
|
|
@ -1787,7 +1828,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
|
|||
int ret;
|
||||
|
||||
hyp_memcache = get_mmu_memcache(vcpu);
|
||||
ret = topup_mmu_memcache(vcpu, hyp_memcache);
|
||||
ret = topup_mmu_memcache(s2fd->mmu, hyp_memcache);
|
||||
if (ret)
|
||||
return -ENOMEM;
|
||||
|
||||
|
|
@ -1910,11 +1951,6 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd,
|
|||
return vma_shift;
|
||||
}
|
||||
|
||||
static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd)
|
||||
{
|
||||
return kvm_vcpu_trap_is_permission_fault(s2fd->vcpu);
|
||||
}
|
||||
|
||||
static int kvm_s2_fault_get_vma_info(const struct kvm_s2_fault_desc *s2fd,
|
||||
struct kvm_s2_fault_vma_info *s2vi)
|
||||
{
|
||||
|
|
@ -1980,13 +2016,11 @@ static int kvm_s2_fault_pin_pfn(const struct kvm_s2_fault_desc *s2fd,
|
|||
return ret;
|
||||
|
||||
s2vi->pfn = __kvm_faultin_pfn(s2fd->memslot, get_canonical_gfn(s2fd, s2vi),
|
||||
kvm_is_write_fault(s2fd->vcpu) ? FOLL_WRITE : 0,
|
||||
kvm_s2_fault_is_write(s2fd) ? FOLL_WRITE : 0,
|
||||
&s2vi->map_writable, &s2vi->page);
|
||||
if (unlikely(is_error_noslot_pfn(s2vi->pfn))) {
|
||||
if (s2vi->pfn == KVM_PFN_ERR_HWPOISON) {
|
||||
kvm_send_hwpoison_signal(s2fd->hva, __ffs(s2vi->vma_pagesize));
|
||||
return 0;
|
||||
}
|
||||
if (s2vi->pfn == KVM_PFN_ERR_HWPOISON)
|
||||
return -EHWPOISON;
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
|
|
@ -2038,7 +2072,7 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd,
|
|||
{
|
||||
struct kvm *kvm = s2fd->vcpu->kvm;
|
||||
|
||||
if (kvm_vcpu_trap_is_exec_fault(s2fd->vcpu) && s2vi->map_non_cacheable)
|
||||
if (kvm_s2_fault_is_exec(s2fd) && s2vi->map_non_cacheable)
|
||||
return -ENOEXEC;
|
||||
|
||||
/*
|
||||
|
|
@ -2047,7 +2081,7 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd,
|
|||
* and trigger the exception here. Since the memslot is valid, inject
|
||||
* the fault back to the guest.
|
||||
*/
|
||||
if (esr_fsc_is_excl_atomic_fault(kvm_vcpu_get_esr(s2fd->vcpu))) {
|
||||
if (esr_fsc_is_excl_atomic_fault(s2fd->esr)) {
|
||||
kvm_inject_dabt_excl_atomic(s2fd->vcpu, kvm_vcpu_get_hfar(s2fd->vcpu));
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -2056,13 +2090,13 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd,
|
|||
|
||||
if (s2vi->map_writable && (s2vi->device ||
|
||||
!memslot_is_logging(s2fd->memslot) ||
|
||||
kvm_is_write_fault(s2fd->vcpu)))
|
||||
kvm_s2_fault_is_write(s2fd)))
|
||||
*prot |= KVM_PGTABLE_PROT_W;
|
||||
|
||||
if (s2fd->nested)
|
||||
*prot = adjust_nested_fault_perms(s2fd->nested, *prot);
|
||||
|
||||
if (kvm_vcpu_trap_is_exec_fault(s2fd->vcpu))
|
||||
if (kvm_s2_fault_is_exec(s2fd))
|
||||
*prot |= KVM_PGTABLE_PROT_X;
|
||||
|
||||
if (s2vi->map_non_cacheable)
|
||||
|
|
@ -2086,7 +2120,8 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd,
|
|||
static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd,
|
||||
const struct kvm_s2_fault_vma_info *s2vi,
|
||||
enum kvm_pgtable_prot prot,
|
||||
void *memcache)
|
||||
void *memcache,
|
||||
struct kvm_s2_fault_result *result)
|
||||
{
|
||||
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED;
|
||||
struct kvm_guest_s2_mapping *mapping = NULL;
|
||||
|
|
@ -2100,7 +2135,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd,
|
|||
gfn_t gfn;
|
||||
int ret;
|
||||
|
||||
if (kvm_is_nested_s2_mmu(kvm, s2fd->vcpu->arch.hw_mmu)) {
|
||||
if (kvm_is_nested_s2_mmu(kvm, s2fd->mmu)) {
|
||||
mapping = kmalloc_obj(struct kvm_guest_s2_mapping,
|
||||
GFP_KERNEL_ACCOUNT);
|
||||
if (!mapping) {
|
||||
|
|
@ -2110,13 +2145,12 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd,
|
|||
}
|
||||
|
||||
kvm_fault_lock(kvm);
|
||||
pgt = s2fd->vcpu->arch.hw_mmu->pgt;
|
||||
pgt = s2fd->mmu->pgt;
|
||||
ret = -EAGAIN;
|
||||
if (mmu_invalidate_retry(kvm, s2vi->mmu_seq))
|
||||
goto out_unlock;
|
||||
|
||||
perm_fault_granule = (kvm_s2_fault_is_perm(s2fd) ?
|
||||
kvm_vcpu_trap_get_perm_fault_granule(s2fd->vcpu) : 0);
|
||||
perm_fault_granule = kvm_s2_perm_fault_granule(s2fd);
|
||||
mapping_size = s2vi->vma_pagesize;
|
||||
pfn = s2vi->pfn;
|
||||
gfn = s2vi->gfn;
|
||||
|
|
@ -2188,14 +2222,19 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd,
|
|||
mark_page_dirty_in_slot(kvm, s2fd->memslot,
|
||||
gpa_to_gfn(canonical_ipa));
|
||||
|
||||
if (ret != -EAGAIN)
|
||||
return ret;
|
||||
return 0;
|
||||
if (ret == -EAGAIN)
|
||||
return result ? ret : 0;
|
||||
|
||||
if (result && !ret)
|
||||
result->mapping_size = mapping_size;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd)
|
||||
static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd,
|
||||
struct kvm_s2_fault_result *result)
|
||||
{
|
||||
bool perm_fault = kvm_vcpu_trap_is_permission_fault(s2fd->vcpu);
|
||||
bool perm_fault = kvm_s2_fault_is_perm(s2fd);
|
||||
struct kvm_s2_fault_vma_info s2vi = {};
|
||||
enum kvm_pgtable_prot prot;
|
||||
void *memcache;
|
||||
|
|
@ -2213,7 +2252,7 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd)
|
|||
memcache = get_mmu_memcache(s2fd->vcpu);
|
||||
if (!perm_fault || memslot_is_logging(s2fd->memslot) ||
|
||||
is_protected_kvm_enabled()) {
|
||||
ret = topup_mmu_memcache(s2fd->vcpu, memcache);
|
||||
ret = topup_mmu_memcache(s2fd->mmu, memcache);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -2223,6 +2262,13 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd)
|
|||
* get block mapping for device MMIO region.
|
||||
*/
|
||||
ret = kvm_s2_fault_pin_pfn(s2fd, &s2vi);
|
||||
if (ret == -EHWPOISON) {
|
||||
/* If result is specified, let the caller handle this. */
|
||||
if (result)
|
||||
return -EHWPOISON;
|
||||
kvm_send_hwpoison_signal(s2fd->hva, __ffs(s2vi.vma_pagesize));
|
||||
return 0;
|
||||
}
|
||||
if (ret != 1)
|
||||
return ret;
|
||||
|
||||
|
|
@ -2232,7 +2278,7 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd)
|
|||
return ret;
|
||||
}
|
||||
|
||||
return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache);
|
||||
return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache, result);
|
||||
}
|
||||
|
||||
/* Resolve the access fault by making the page young again. */
|
||||
|
|
@ -2342,7 +2388,8 @@ int kvm_handle_guest_sea(struct kvm_vcpu *vcpu)
|
|||
int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
struct kvm_s2_trans nested_trans, *nested = NULL;
|
||||
unsigned long esr;
|
||||
unsigned long esr = kvm_vcpu_get_esr(vcpu);
|
||||
struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu;
|
||||
phys_addr_t fault_ipa; /* The address we faulted on */
|
||||
phys_addr_t ipa; /* Always the IPA in the L1 guest phys space */
|
||||
struct kvm_memory_slot *memslot;
|
||||
|
|
@ -2351,11 +2398,9 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
gfn_t gfn;
|
||||
int ret, idx;
|
||||
|
||||
if (kvm_vcpu_abt_issea(vcpu))
|
||||
if (esr_abt_is_sea(esr))
|
||||
return kvm_handle_guest_sea(vcpu);
|
||||
|
||||
esr = kvm_vcpu_get_esr(vcpu);
|
||||
|
||||
/*
|
||||
* The fault IPA should be reliable at this point as we're not dealing
|
||||
* with an SEA.
|
||||
|
|
@ -2364,7 +2409,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
if (KVM_BUG_ON(ipa == INVALID_GPA, vcpu->kvm))
|
||||
return -EFAULT;
|
||||
|
||||
is_iabt = kvm_vcpu_trap_is_iabt(vcpu);
|
||||
is_iabt = esr_trap_is_iabt(esr);
|
||||
|
||||
if (esr_fsc_is_translation_fault(esr)) {
|
||||
/* Beyond sanitised PARange (which is the IPA limit) */
|
||||
|
|
@ -2374,14 +2419,14 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
}
|
||||
|
||||
/* Falls between the IPA range and the PARange? */
|
||||
if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(vcpu->arch.hw_mmu->vtcr))) {
|
||||
if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(mmu->vtcr))) {
|
||||
fault_ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(vcpu));
|
||||
|
||||
return kvm_inject_sea(vcpu, is_iabt, fault_ipa);
|
||||
}
|
||||
}
|
||||
|
||||
trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu),
|
||||
trace_kvm_guest_fault(*vcpu_pc(vcpu), esr,
|
||||
kvm_vcpu_get_hfar(vcpu), fault_ipa);
|
||||
|
||||
/* Check the stage-2 fault is trans. fault or write fault */
|
||||
|
|
@ -2389,10 +2434,10 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
!esr_fsc_is_permission_fault(esr) &&
|
||||
!esr_fsc_is_access_flag_fault(esr) &&
|
||||
!esr_fsc_is_excl_atomic_fault(esr)) {
|
||||
kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
|
||||
kvm_vcpu_trap_get_class(vcpu),
|
||||
(unsigned long)kvm_vcpu_trap_get_fault(vcpu),
|
||||
(unsigned long)kvm_vcpu_get_esr(vcpu));
|
||||
kvm_err("Unsupported FSC: EC=%#lx xFSC=%#lx ESR_EL2=%#lx\n",
|
||||
ESR_ELx_EC(esr),
|
||||
(unsigned long)(esr & ESR_ELx_FSC),
|
||||
(unsigned long)esr);
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
|
|
@ -2411,8 +2456,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
* nothing to walk and we treat it as a 1:1 before going through the
|
||||
* canonical translation.
|
||||
*/
|
||||
if (kvm_is_nested_s2_mmu(vcpu->kvm,vcpu->arch.hw_mmu) &&
|
||||
vcpu->arch.hw_mmu->nested_stage2_enabled) {
|
||||
if (kvm_is_nested_s2_mmu(vcpu->kvm, mmu) &&
|
||||
mmu->nested_stage2_enabled) {
|
||||
u32 esr;
|
||||
|
||||
ret = kvm_walk_nested_s2(vcpu, fault_ipa, &nested_trans);
|
||||
|
|
@ -2441,7 +2486,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
gfn = ipa >> PAGE_SHIFT;
|
||||
memslot = gfn_to_memslot(vcpu->kvm, gfn);
|
||||
hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
|
||||
write_fault = kvm_is_write_fault(vcpu);
|
||||
write_fault = esr_abt_is_write_fault(esr);
|
||||
if (kvm_is_error_hva(hva) || (write_fault && !writable)) {
|
||||
/*
|
||||
* The guest has put either its instructions or its page-tables
|
||||
|
|
@ -2454,7 +2499,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
goto out;
|
||||
}
|
||||
|
||||
if (kvm_vcpu_abt_iss1tw(vcpu)) {
|
||||
if (esr_abt_is_s1ptw(esr)) {
|
||||
ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
|
||||
goto out_unlock;
|
||||
}
|
||||
|
|
@ -2469,7 +2514,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
* So let's assume that the guest is just being
|
||||
* cautious, and skip the instruction.
|
||||
*/
|
||||
if (kvm_is_error_hva(hva) && kvm_vcpu_dabt_is_cm(vcpu)) {
|
||||
if (kvm_is_error_hva(hva) && esr_dabt_is_cm(esr)) {
|
||||
kvm_incr_pc(vcpu);
|
||||
ret = 1;
|
||||
goto out_unlock;
|
||||
|
|
@ -2487,7 +2532,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
}
|
||||
|
||||
/* Userspace should not be able to register out-of-bounds IPAs */
|
||||
VM_BUG_ON(ipa >= kvm_phys_size(vcpu->arch.hw_mmu));
|
||||
VM_BUG_ON(ipa >= kvm_phys_size(mmu));
|
||||
|
||||
if (esr_fsc_is_access_flag_fault(esr)) {
|
||||
handle_access_fault(vcpu, fault_ipa);
|
||||
|
|
@ -2501,19 +2546,20 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
.nested = nested,
|
||||
.memslot = memslot,
|
||||
.hva = hva,
|
||||
.esr = esr,
|
||||
.mmu = mmu,
|
||||
};
|
||||
|
||||
if (kvm_vm_is_protected(vcpu->kvm)) {
|
||||
ret = pkvm_mem_abort(&s2fd);
|
||||
} else {
|
||||
VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) &&
|
||||
!write_fault &&
|
||||
!kvm_vcpu_trap_is_exec_fault(vcpu));
|
||||
VM_WARN_ON_ONCE(kvm_s2_fault_is_perm(&s2fd) && !write_fault &&
|
||||
!kvm_s2_fault_is_exec(&s2fd));
|
||||
|
||||
if (kvm_slot_has_gmem(memslot))
|
||||
ret = gmem_abort(&s2fd);
|
||||
ret = gmem_abort(&s2fd, NULL);
|
||||
else
|
||||
ret = user_mem_abort(&s2fd);
|
||||
ret = user_mem_abort(&s2fd, NULL);
|
||||
}
|
||||
|
||||
if (ret == 0)
|
||||
|
|
@ -2890,3 +2936,146 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
|
|||
|
||||
trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled);
|
||||
}
|
||||
|
||||
/*
|
||||
* Try to walk to the specified GPA in canonical mmu - if unmapped returns 0, if
|
||||
* mapped returns the granule size, otherwise returns an error.
|
||||
*/
|
||||
static long kvm_walk_s2(struct kvm_pgtable *pgt,
|
||||
gpa_t gpa, s8 *level)
|
||||
{
|
||||
struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
|
||||
kvm_pte_t pte;
|
||||
long ret;
|
||||
|
||||
guard(read_lock)(&kvm->mmu_lock);
|
||||
|
||||
ret = kvm_pgtable_get_leaf(pgt, gpa, &pte, level,
|
||||
KVM_PGTABLE_WALK_SHARED);
|
||||
if (ret)
|
||||
return ret;
|
||||
/* Unpopulated, must fault. */
|
||||
if (!kvm_pte_valid(pte))
|
||||
return 0;
|
||||
return kvm_granule_size(*level);
|
||||
}
|
||||
|
||||
/* Synthesised data abort at specified page table level. */
|
||||
#define PRE_FAULT_ESR(level) \
|
||||
((ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT) | \
|
||||
ESR_ELx_IL | ESR_ELx_FSC_FAULT_L(level))
|
||||
|
||||
/* Retrieve either a read-only or a read/write hva. */
|
||||
static hva_t gfn_to_hva_memslot_read(struct kvm_memory_slot *slot, gfn_t gfn)
|
||||
{
|
||||
return gfn_to_hva_memslot_prot(slot, gfn, /*writable=*/NULL);
|
||||
}
|
||||
|
||||
static long __pre_fault_s2(struct kvm_s2_mmu *mmu, struct kvm_vcpu *vcpu,
|
||||
gpa_t gpa, struct kvm_memory_slot *memslot, s8 level)
|
||||
{
|
||||
const bool is_gmem = kvm_slot_has_gmem(memslot);
|
||||
const gfn_t gfn = gpa_to_gfn(gpa);
|
||||
const hva_t hva = is_gmem ? 0 : gfn_to_hva_memslot_read(memslot, gfn);
|
||||
const struct kvm_s2_fault_desc s2fd = {
|
||||
.vcpu = vcpu,
|
||||
.fault_ipa = gpa,
|
||||
.nested = NULL,
|
||||
.memslot = memslot,
|
||||
.hva = hva,
|
||||
.esr = PRE_FAULT_ESR(level),
|
||||
.mmu = mmu,
|
||||
};
|
||||
struct kvm_s2_fault_result result = {};
|
||||
long ret;
|
||||
|
||||
if (kvm_is_error_hva(hva))
|
||||
return -EFAULT;
|
||||
|
||||
if (is_gmem)
|
||||
ret = gmem_abort(&s2fd, &result);
|
||||
else
|
||||
ret = user_mem_abort(&s2fd, &result);
|
||||
if (IS_ERR_VALUE(ret))
|
||||
return ret;
|
||||
return result.mapping_size;
|
||||
}
|
||||
|
||||
static long pre_fault_s2(struct kvm_s2_mmu *mmu, struct kvm_vcpu *vcpu,
|
||||
gpa_t gpa, struct kvm_memory_slot *memslot)
|
||||
{
|
||||
s8 level;
|
||||
long ret;
|
||||
|
||||
/* Try a walk first. */
|
||||
ret = kvm_walk_s2(mmu->pgt, gpa, &level);
|
||||
if (ret)
|
||||
return ret;
|
||||
/* OK, have to fault page in. */
|
||||
return __pre_fault_s2(mmu, vcpu, gpa, memslot, level);
|
||||
}
|
||||
|
||||
static unsigned long
|
||||
pre_fault_bytes_consumed(gpa_t gpa, unsigned long granule_size,
|
||||
unsigned long bytes_remaining)
|
||||
{
|
||||
/* Granules are always a power-of-2. */
|
||||
const unsigned long granule_bytes_remaining =
|
||||
granule_size - (gpa % granule_size);
|
||||
|
||||
return min(granule_bytes_remaining, bytes_remaining);
|
||||
}
|
||||
|
||||
/* If you lose the race this many times, time to give up. */
|
||||
#define MAX_PRE_FAULT_RETRIES 3
|
||||
|
||||
int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (is_protected_kvm_enabled())
|
||||
return -EOPNOTSUPP;
|
||||
if (!kvm_vcpu_initialized(vcpu))
|
||||
return -ENOEXEC;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* kvm_arch_vcpu_pre_fault_memory - pre-fault stage-2 page tables for the
|
||||
* specified GPA.
|
||||
* @vcpu: The VCPU pointer
|
||||
* @range: {gpa, size, flags} tuple
|
||||
*
|
||||
* The mapping performed is always best-effort - faulting in is necessarily
|
||||
* racey. The ranges faulted in are canonical, nested page tables are ignored.
|
||||
*
|
||||
* @range->gpa specifies the GPA to pre-fault, @range->size specifies how many
|
||||
* bytes remain to be pre-faulted and @range->flags is reserved and must be 0.
|
||||
*
|
||||
* Returns: the number of bytes the pre-fault consumed, or an error.
|
||||
*/
|
||||
long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
|
||||
struct kvm_pre_fault_memory *range)
|
||||
{
|
||||
struct kvm *kvm = vcpu->kvm;
|
||||
const u64 bytes_remaining = range->size;
|
||||
struct kvm_s2_mmu *mmu = &kvm->arch.mmu; /* Canonical. */
|
||||
struct kvm_memory_slot *memslot;
|
||||
const gpa_t gpa = range->gpa;
|
||||
int num_retries = 0;
|
||||
long ret;
|
||||
|
||||
memslot = gfn_to_memslot(kvm, gpa_to_gfn(gpa));
|
||||
if (!memslot)
|
||||
return -ENOENT;
|
||||
/* SRCU must be released for progress and only userland can do that. */
|
||||
if (memslot->flags & KVM_MEMSLOT_INVALID)
|
||||
return -EAGAIN;
|
||||
|
||||
do {
|
||||
ret = pre_fault_s2(mmu, vcpu, gpa, memslot);
|
||||
} while (ret == -EAGAIN && num_retries++ < MAX_PRE_FAULT_RETRIES);
|
||||
|
||||
if (IS_ERR_VALUE(ret))
|
||||
return ret;
|
||||
return pre_fault_bytes_consumed(gpa, ret, bytes_remaining);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -665,7 +665,7 @@ static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
|
|||
return 0;
|
||||
|
||||
tmp &= ~(sz - 1);
|
||||
if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))
|
||||
if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL, 0))
|
||||
goto again;
|
||||
if (!(pte & PTE_VALID))
|
||||
goto again;
|
||||
|
|
|
|||
|
|
@ -5784,6 +5784,14 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old,
|
|||
s390_kvm_mmu_commit_memory_region(kvm, old, new, change);
|
||||
}
|
||||
|
||||
int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (kvm_is_ucontrol(vcpu->kvm))
|
||||
return -EINVAL;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables
|
||||
* @vcpu: the vcpu that shall appear to have generated the fault-in.
|
||||
|
|
@ -5810,9 +5818,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_
|
|||
gpa_t end;
|
||||
int rc;
|
||||
|
||||
if (kvm_is_ucontrol(vcpu->kvm))
|
||||
return -EINVAL;
|
||||
|
||||
rc = kvm_s390_faultin_gfn(vcpu, NULL, &f);
|
||||
if (rc == PGM_ADDRESSING)
|
||||
return -ENOENT;
|
||||
|
|
|
|||
|
|
@ -5086,6 +5086,14 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa,
|
|||
}
|
||||
}
|
||||
|
||||
int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (!vcpu->kvm->arch.pre_fault_allowed)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
|
||||
struct kvm_pre_fault_memory *range)
|
||||
{
|
||||
|
|
@ -5095,9 +5103,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
|
|||
u64 end;
|
||||
int r;
|
||||
|
||||
if (!vcpu->kvm->arch.pre_fault_allowed)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa)))
|
||||
return -EINVAL;
|
||||
|
||||
|
|
|
|||
|
|
@ -1708,7 +1708,7 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve
|
|||
WARN_ON_ONCE((new_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK);
|
||||
|
||||
/*
|
||||
* Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory()
|
||||
* Ensure pre_fault_allowed is read by kvm_arch_pre_fault_allowed()
|
||||
* before kvm_tdx->state. Userspace must not be allowed to pre-fault
|
||||
* arbitrary memory until the initial memory image is finalized. Pairs
|
||||
* with the smp_wmb() in tdx_td_finalize().
|
||||
|
|
|
|||
|
|
@ -2643,6 +2643,7 @@ void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range
|
|||
#endif
|
||||
|
||||
#ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY
|
||||
int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu);
|
||||
long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
|
||||
struct kvm_pre_fault_memory *range);
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -178,6 +178,7 @@ TEST_GEN_PROGS_arm64 += arm64/debug-exceptions
|
|||
TEST_GEN_PROGS_arm64 += arm64/hello_el2
|
||||
TEST_GEN_PROGS_arm64 += arm64/host_sve
|
||||
TEST_GEN_PROGS_arm64 += arm64/hypercalls
|
||||
TEST_GEN_PROGS_arm64 += arm64/nv_pre_fault_memory_test
|
||||
TEST_GEN_PROGS_arm64 += arm64/external_aborts
|
||||
TEST_GEN_PROGS_arm64 += arm64/mmio_sign_ext
|
||||
TEST_GEN_PROGS_arm64 += arm64/page_fault_test
|
||||
|
|
@ -205,6 +206,7 @@ TEST_GEN_PROGS_arm64 += guest_memfd_test
|
|||
TEST_GEN_PROGS_arm64 += mmu_stress_test
|
||||
TEST_GEN_PROGS_arm64 += rseq_test
|
||||
TEST_GEN_PROGS_arm64 += steal_time
|
||||
TEST_GEN_PROGS_arm64 += pre_fault_memory_test
|
||||
|
||||
TEST_GEN_PROGS_s390 = $(TEST_GEN_PROGS_COMMON)
|
||||
TEST_GEN_PROGS_s390 += s390/memop
|
||||
|
|
|
|||
158
tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c
Normal file
158
tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* nv_pre_fault_memory_test - Test KVM_PRE_FAULT_MEMORY on a vCPU whose
|
||||
* last-run context is nested.
|
||||
*
|
||||
* The guest enters vEL2, sets up its EL2 translation configuration into the
|
||||
* real EL1 registers then ERETs to vEL1 and exits to userspace so its vCPU
|
||||
* last-run context is nested backed by a shadow stage 2 MMU.
|
||||
*
|
||||
* Assert that pre-faulting ignores that and targets the canonical stage-2
|
||||
* page tables only.
|
||||
*/
|
||||
#include "kvm_util.h"
|
||||
#include "processor.h"
|
||||
#include "test_util.h"
|
||||
#include "ucall.h"
|
||||
|
||||
#include <asm/sysreg.h>
|
||||
#include <linux/sizes.h>
|
||||
|
||||
#define TEST_MEM_SLOT 10
|
||||
#define TEST_MEM_SIZE SZ_2M
|
||||
#define TEST_MEM_GPA SZ_1G
|
||||
|
||||
static void guest_el1_code(void)
|
||||
{
|
||||
u64 offset;
|
||||
|
||||
GUEST_ASSERT_EQ(get_current_el(), 1);
|
||||
|
||||
/* Exit to userspace with the vEL1 (nested) context live. */
|
||||
GUEST_SYNC(1);
|
||||
|
||||
/*
|
||||
* Touch the prefaulted range. vstage-2 is disabled, so the shadow
|
||||
* stage-2 is a 1:1 view of the canonical IPA space.
|
||||
*/
|
||||
for (offset = 0; offset < TEST_MEM_SIZE; offset += SZ_4K)
|
||||
READ_ONCE(*(u64 *)(TEST_MEM_GPA + offset));
|
||||
|
||||
GUEST_DONE();
|
||||
}
|
||||
|
||||
static void guest_code(void)
|
||||
{
|
||||
u64 sp;
|
||||
|
||||
GUEST_ASSERT_EQ(get_current_el(), 2);
|
||||
|
||||
/*
|
||||
* Mirror the EL2 translation regime into the real EL1 registers so
|
||||
* that vEL1 runs on the test's stage-1 page tables. With E2H=1, the
|
||||
* _EL1 accessors read the EL2 registers, and the _EL12 accessors
|
||||
* write the real EL1 registers.
|
||||
*/
|
||||
write_sysreg_s(read_sysreg(sctlr_el1), SYS_SCTLR_EL12);
|
||||
write_sysreg_s(read_sysreg(tcr_el1), SYS_TCR_EL12);
|
||||
write_sysreg_s(read_sysreg(ttbr0_el1), SYS_TTBR0_EL12);
|
||||
write_sysreg_s(read_sysreg(mair_el1), SYS_MAIR_EL12);
|
||||
write_sysreg_s(read_sysreg(cpacr_el1), SYS_CPACR_EL12);
|
||||
|
||||
/* Run vEL1 on the same stack. */
|
||||
asm volatile("mov %0, sp" : "=r"(sp));
|
||||
write_sysreg(sp, sp_el1);
|
||||
|
||||
/*
|
||||
* Drop TGE so that vEL1 is a nested context rather than host EL0.
|
||||
* KVM backs it with a shadow stage-2 MMU even though vstage-2 is
|
||||
* disabled (HCR_EL2.VM=0).
|
||||
*/
|
||||
write_sysreg(read_sysreg(hcr_el2) & ~HCR_EL2_TGE, hcr_el2);
|
||||
isb();
|
||||
|
||||
write_sysreg(PSR_MODE_EL1h | PSR_F_BIT | PSR_I_BIT | PSR_A_BIT |
|
||||
PSR_D_BIT, spsr_el2);
|
||||
write_sysreg((u64)guest_el1_code, elr_el2);
|
||||
asm volatile("eret");
|
||||
|
||||
GUEST_ASSERT(false);
|
||||
}
|
||||
|
||||
static void pre_fault(struct kvm_vcpu *vcpu, u64 gpa, u64 size)
|
||||
{
|
||||
struct kvm_pre_fault_memory range = {
|
||||
.gpa = gpa,
|
||||
.size = size,
|
||||
};
|
||||
int ret;
|
||||
|
||||
do {
|
||||
ret = __vcpu_ioctl(vcpu, KVM_PRE_FAULT_MEMORY, &range);
|
||||
} while ((!ret && range.size) ||
|
||||
(ret < 0 && (errno == EINTR || errno == EAGAIN)));
|
||||
|
||||
TEST_ASSERT(!ret, "KVM_PRE_FAULT_MEMORY failed, ret: %d errno: %d",
|
||||
ret, errno);
|
||||
TEST_ASSERT_EQ(range.size, 0);
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
struct kvm_vcpu_init init;
|
||||
struct kvm_vcpu *vcpu;
|
||||
struct kvm_vm *vm;
|
||||
struct ucall uc;
|
||||
u64 npages;
|
||||
|
||||
TEST_REQUIRE(test_supports_el2());
|
||||
TEST_REQUIRE(kvm_check_cap(KVM_CAP_PRE_FAULT_MEMORY));
|
||||
|
||||
vm = vm_create(1);
|
||||
|
||||
kvm_get_default_vcpu_target(vm, &init);
|
||||
init.features[0] |= BIT(KVM_ARM_VCPU_HAS_EL2);
|
||||
vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code);
|
||||
kvm_arch_vm_finalize_vcpus(vm);
|
||||
|
||||
npages = TEST_MEM_SIZE / vm->page_size;
|
||||
vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, TEST_MEM_GPA,
|
||||
TEST_MEM_SLOT, npages, 0);
|
||||
virt_map(vm, TEST_MEM_GPA, TEST_MEM_GPA, npages);
|
||||
|
||||
/* Run the guest until it has ERET'd from vEL2 to vEL1. */
|
||||
vcpu_run(vcpu);
|
||||
switch (get_ucall(vcpu, &uc)) {
|
||||
case UCALL_SYNC:
|
||||
TEST_ASSERT_EQ(uc.args[1], 1);
|
||||
break;
|
||||
case UCALL_ABORT:
|
||||
REPORT_GUEST_ASSERT(uc);
|
||||
break;
|
||||
default:
|
||||
TEST_FAIL("Unhandled ucall: %ld", uc.cmd);
|
||||
}
|
||||
|
||||
/*
|
||||
* The vCPU's last-run context is vEL1, so its hw_mmu is a shadow
|
||||
* stage-2 MMU.
|
||||
*
|
||||
* Pre-faulting must ignore that and populate the canonical stage-2.
|
||||
*/
|
||||
pre_fault(vcpu, TEST_MEM_GPA, TEST_MEM_SIZE);
|
||||
|
||||
/* Resume at vEL1 and touch the prefaulted range. */
|
||||
vcpu_run(vcpu);
|
||||
switch (get_ucall(vcpu, &uc)) {
|
||||
case UCALL_DONE:
|
||||
break;
|
||||
case UCALL_ABORT:
|
||||
REPORT_GUEST_ASSERT(uc);
|
||||
break;
|
||||
default:
|
||||
TEST_FAIL("Unhandled ucall: %ld", uc.cmd);
|
||||
}
|
||||
|
||||
kvm_vm_free(vm);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -12,19 +12,29 @@
|
|||
#include <processor.h>
|
||||
#include <pthread.h>
|
||||
#include <ucall_common.h>
|
||||
#include <guest_modes.h>
|
||||
|
||||
/* Arbitrarily chosen values */
|
||||
#define TEST_SIZE (SZ_2M + PAGE_SIZE)
|
||||
#define TEST_NPAGES (TEST_SIZE / PAGE_SIZE)
|
||||
#define TEST_BASE_SIZE SZ_2M
|
||||
#define TEST_SLOT 10
|
||||
|
||||
/* Storage of test info to share with guest code */
|
||||
struct test_config {
|
||||
u64 page_size;
|
||||
u64 test_size;
|
||||
u64 test_num_pages;
|
||||
};
|
||||
|
||||
static struct test_config test_config;
|
||||
|
||||
static void guest_code(u64 base_gva)
|
||||
{
|
||||
volatile u64 val __used;
|
||||
struct test_config *config = &test_config;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < TEST_NPAGES; i++) {
|
||||
u64 *src = (u64 *)(base_gva + i * PAGE_SIZE);
|
||||
for (i = 0; i < config->test_num_pages; i++) {
|
||||
u64 *src = (u64 *)(base_gva + i * config->page_size);
|
||||
|
||||
val = *src;
|
||||
}
|
||||
|
|
@ -36,6 +46,7 @@ struct slot_worker_data {
|
|||
struct kvm_vm *vm;
|
||||
gpa_t gpa;
|
||||
u32 flags;
|
||||
enum vm_mem_backing_src_type mem_backing_src;
|
||||
bool worker_ready;
|
||||
bool prefault_ready;
|
||||
bool recreate_slot;
|
||||
|
|
@ -56,14 +67,16 @@ static void *delete_slot_worker(void *__data)
|
|||
while (!READ_ONCE(data->recreate_slot))
|
||||
cpu_relax();
|
||||
|
||||
vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, data->gpa,
|
||||
TEST_SLOT, TEST_NPAGES, data->flags);
|
||||
vm_userspace_mem_region_add(vm, data->mem_backing_src, data->gpa,
|
||||
TEST_SLOT, test_config.test_num_pages, data->flags);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset,
|
||||
u64 size, u64 expected_left, bool private)
|
||||
u64 size, u64 expected_left,
|
||||
enum vm_mem_backing_src_type mem_backing_src,
|
||||
bool private)
|
||||
{
|
||||
struct kvm_pre_fault_memory range = {
|
||||
.gpa = base_gpa + offset,
|
||||
|
|
@ -74,6 +87,7 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset,
|
|||
.vm = vcpu->vm,
|
||||
.gpa = base_gpa,
|
||||
.flags = private ? KVM_MEM_GUEST_MEMFD : 0,
|
||||
.mem_backing_src = mem_backing_src,
|
||||
};
|
||||
bool slot_recreated = false;
|
||||
pthread_t slot_worker;
|
||||
|
|
@ -150,8 +164,8 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset,
|
|||
/*
|
||||
* Assert success if prefaulting the entire range should succeed, i.e.
|
||||
* complete with no bytes remaining. Otherwise prefaulting should have
|
||||
* failed due to ENOENT (due to RET_PF_EMULATE for emulated MMIO when
|
||||
* no memslot exists).
|
||||
* failed due to ENOENT (no memslot exists for the GPA; on x86 this
|
||||
* surfaces via RET_PF_EMULATE).
|
||||
*/
|
||||
if (!expected_left)
|
||||
TEST_ASSERT_VM_VCPU_IOCTL(!ret, KVM_PRE_FAULT_MEMORY, ret, vcpu->vm);
|
||||
|
|
@ -160,39 +174,82 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset,
|
|||
KVM_PRE_FAULT_MEMORY, ret, vcpu->vm);
|
||||
}
|
||||
|
||||
static void __test_pre_fault_memory(unsigned long vm_type, bool private)
|
||||
struct test_params {
|
||||
unsigned long vm_type;
|
||||
bool private;
|
||||
enum vm_mem_backing_src_type mem_backing_src;
|
||||
};
|
||||
|
||||
static void __test_pre_fault_memory(enum vm_guest_mode guest_mode, void *arg)
|
||||
{
|
||||
gpa_t gpa, gva, alignment, guest_page_size;
|
||||
gpa_t gpa, gva, alignment, guest_page_size, host_page_size;
|
||||
gpa_t backing_src_pagesz, mem_page_size;
|
||||
struct test_params *p = arg;
|
||||
const struct vm_shape shape = {
|
||||
.mode = VM_MODE_DEFAULT,
|
||||
.type = vm_type,
|
||||
.mode = guest_mode,
|
||||
.type = p->vm_type,
|
||||
};
|
||||
struct kvm_vcpu *vcpu;
|
||||
struct kvm_run *run;
|
||||
struct kvm_vm *vm;
|
||||
struct ucall uc;
|
||||
|
||||
pr_info("Testing guest mode: %s\n", vm_guest_mode_string(guest_mode));
|
||||
pr_info("Testing memory backing src type: %s\n",
|
||||
vm_mem_backing_src_alias(p->mem_backing_src)->name);
|
||||
|
||||
vm = vm_create_shape_with_one_vcpu(shape, &vcpu, guest_code);
|
||||
|
||||
alignment = guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
|
||||
gpa = (vm->max_gfn - TEST_NPAGES) * guest_page_size;
|
||||
guest_page_size = vm_guest_mode_params[guest_mode].page_size;
|
||||
host_page_size = getpagesize();
|
||||
backing_src_pagesz = get_backing_src_pagesz(p->mem_backing_src);
|
||||
mem_page_size = max(host_page_size, backing_src_pagesz);
|
||||
|
||||
test_config.page_size = guest_page_size;
|
||||
test_config.test_size = align_up(TEST_BASE_SIZE + test_config.page_size,
|
||||
mem_page_size);
|
||||
test_config.test_num_pages = vm_calc_num_guest_pages(vm->mode, test_config.test_size);
|
||||
|
||||
gpa = (vm->max_gfn - test_config.test_num_pages) * test_config.page_size;
|
||||
alignment = SZ_2M;
|
||||
alignment = max(alignment, mem_page_size);
|
||||
gpa = align_down(gpa, alignment);
|
||||
gva = gpa & ((1ULL << (vm->va_bits - 1)) - 1);
|
||||
|
||||
vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa, TEST_SLOT,
|
||||
TEST_NPAGES, private ? KVM_MEM_GUEST_MEMFD : 0);
|
||||
virt_map(vm, gva, gpa, TEST_NPAGES);
|
||||
vm_userspace_mem_region_add(vm, p->mem_backing_src,
|
||||
gpa, TEST_SLOT, test_config.test_num_pages,
|
||||
p->private ? KVM_MEM_GUEST_MEMFD : 0);
|
||||
virt_map(vm, gva, gpa, test_config.test_num_pages);
|
||||
|
||||
if (private)
|
||||
vm_mem_set_private(vm, gpa, TEST_SIZE);
|
||||
if (p->private)
|
||||
vm_mem_set_private(vm, gpa, test_config.test_size);
|
||||
|
||||
pre_fault_memory(vcpu, gpa, 0, SZ_2M, 0, private);
|
||||
pre_fault_memory(vcpu, gpa, SZ_2M, PAGE_SIZE * 2, PAGE_SIZE, private);
|
||||
pre_fault_memory(vcpu, gpa, TEST_SIZE, PAGE_SIZE, PAGE_SIZE, private);
|
||||
pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0,
|
||||
p->mem_backing_src, p->private);
|
||||
/* Retry the same range after the first prefault attempt. */
|
||||
pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0,
|
||||
p->mem_backing_src, p->private);
|
||||
pre_fault_memory(vcpu, gpa,
|
||||
test_config.test_size - host_page_size,
|
||||
host_page_size * 2, host_page_size,
|
||||
p->mem_backing_src, p->private);
|
||||
pre_fault_memory(vcpu, gpa, test_config.test_size,
|
||||
host_page_size, host_page_size,
|
||||
p->mem_backing_src, p->private);
|
||||
|
||||
vcpu_args_set(vcpu, 1, gva);
|
||||
|
||||
/* Export the shared variables to the guest. */
|
||||
sync_global_to_guest(vm, test_config);
|
||||
|
||||
vcpu_run(vcpu);
|
||||
|
||||
run = vcpu->run;
|
||||
TEST_ASSERT(run->exit_reason == UCALL_EXIT_REASON,
|
||||
"Wanted %s, got exit reason: %u (%s)",
|
||||
exit_reason_str(UCALL_EXIT_REASON),
|
||||
run->exit_reason, exit_reason_str(run->exit_reason));
|
||||
|
||||
switch (get_ucall(vcpu, &uc)) {
|
||||
case UCALL_ABORT:
|
||||
REPORT_GUEST_ASSERT(uc);
|
||||
|
|
@ -207,24 +264,61 @@ static void __test_pre_fault_memory(unsigned long vm_type, bool private)
|
|||
kvm_vm_free(vm);
|
||||
}
|
||||
|
||||
static void test_pre_fault_memory(unsigned long vm_type, bool private)
|
||||
static void test_pre_fault_memory(unsigned long vm_type, enum vm_mem_backing_src_type backing_src,
|
||||
bool private)
|
||||
{
|
||||
struct test_params p = {
|
||||
.vm_type = vm_type,
|
||||
.private = private,
|
||||
.mem_backing_src = backing_src,
|
||||
};
|
||||
|
||||
if (vm_type && !(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(vm_type))) {
|
||||
pr_info("Skipping tests for vm_type 0x%lx\n", vm_type);
|
||||
return;
|
||||
}
|
||||
|
||||
__test_pre_fault_memory(vm_type, private);
|
||||
for_each_guest_mode(__test_pre_fault_memory, &p);
|
||||
}
|
||||
|
||||
static void help(char *name)
|
||||
{
|
||||
puts("");
|
||||
printf("usage: %s [-h] [-m mode] [-s mem-type]\n", name);
|
||||
puts("");
|
||||
guest_modes_help();
|
||||
backing_src_help("-s");
|
||||
puts("");
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
enum vm_mem_backing_src_type backing = DEFAULT_VM_MEM_SRC;
|
||||
int opt;
|
||||
|
||||
guest_modes_append_default();
|
||||
|
||||
while ((opt = getopt(argc, argv, "hm:s:")) != -1) {
|
||||
switch (opt) {
|
||||
case 'm':
|
||||
guest_modes_cmdline(optarg);
|
||||
break;
|
||||
case 's':
|
||||
backing = parse_backing_src_type(optarg);
|
||||
break;
|
||||
case 'h':
|
||||
default:
|
||||
help(argv[0]);
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_REQUIRE(kvm_check_cap(KVM_CAP_PRE_FAULT_MEMORY));
|
||||
|
||||
test_pre_fault_memory(0, false);
|
||||
test_pre_fault_memory(0, backing, false);
|
||||
#ifdef __x86_64__
|
||||
test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, false);
|
||||
test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, true);
|
||||
test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, backing, false);
|
||||
test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, backing, true);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4353,7 +4353,7 @@ static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu)
|
|||
static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
|
||||
struct kvm_pre_fault_memory *range)
|
||||
{
|
||||
int idx;
|
||||
int idx, err;
|
||||
long r;
|
||||
u64 full_size;
|
||||
|
||||
|
|
@ -4365,6 +4365,14 @@ static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
|
|||
range->gpa + range->size <= range->gpa)
|
||||
return -EINVAL;
|
||||
|
||||
/*
|
||||
* Certain architectures (e.g. arm64) need to reject the ioctl 'early'
|
||||
* before vcpu_load().
|
||||
*/
|
||||
err = kvm_arch_pre_fault_allowed(vcpu);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
vcpu_load(vcpu);
|
||||
idx = srcu_read_lock(&vcpu->kvm->srcu);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user