mirror of
https://github.com/torvalds/linux.git
synced 2026-11-07 06:17:02 +01:00
Merge branch kvm-arm64/cca-7.4 into kvmarm-master/next
* kvm-arm64/cca-7.4: : \ : Preliminary changes that pave the way for CCA in a future revision : of Linux, introducing some required abstractions (vcpu ops, S2 ops) : and making some concepts common with pKVM. : : Patches courtesy of Suzuki Poulose. : / KVM: arm64: Prevent unsupported vcpu features for VM types KVM: arm64: Mandate VGIC v3 for pKVM VMs and Realms KVM: arm64: Abstract out memory abort handling KVM: arm64: Use a local kvm pointer in kvm_handle_guest_abort() KVM: arm64: Add VM specific callback for S2 MMU operations KVM: arm64: Consolidate stage2 unmap range into kvm_stage2_unmap_range KVM: arm64: Add vcpu load/put call backs for flavors KVM: arm64: Refactor the vcpu_load to allow for VM specific callbacks KVM: arm64: Don't call vcpu_set_pauth_traps for pKVM host KVM: arm64: Track the type of VM in kvm_arch KVM: arm64: Avoid including linux/kvm_host.h in kvm_pgtable.h KVM: arm64: Include kvm_emulate.h in kvm/arm_psci.h KVM: arm64: protected VM: Handle user writes to CNTVCT_EL0/CNTPCT_EL0 Signed-off-by: Marc Zyngier <maz@kernel.org>
This commit is contained in:
Marc Zyngier
commit
dd0bc00e3e
|
|
@ -221,6 +221,26 @@ struct kvm_guest_s2_mapping {
|
|||
struct interval_tree_node nested;
|
||||
};
|
||||
|
||||
struct kvm_vcpu_ops {
|
||||
void (*vcpu_load)(struct kvm_vcpu *vcpu);
|
||||
void (*vcpu_put)(struct kvm_vcpu *vcpu);
|
||||
};
|
||||
|
||||
struct kvm_gfn_range;
|
||||
struct kvm_s2_fault_desc;
|
||||
|
||||
struct kvm_vm_s2_ops {
|
||||
bool (*vm_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range);
|
||||
bool (*vm_test_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range);
|
||||
int (*vm_flush_remote_tlbs)(struct kvm *kvm);
|
||||
int (*vm_flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn,
|
||||
u64 nr_pages);
|
||||
void (*vm_stage2_unmap_range)(struct kvm_s2_mmu *mmu,
|
||||
phys_addr_t start, u64 size,
|
||||
bool may_block);
|
||||
int (*vm_mem_abort)(const struct kvm_s2_fault_desc *s2fd);
|
||||
};
|
||||
|
||||
struct kvm_s2_mmu {
|
||||
struct kvm_vmid vmid;
|
||||
|
||||
|
|
@ -327,7 +347,6 @@ struct kvm_smccc_features {
|
|||
struct kvm_protected_vm {
|
||||
pkvm_handle_t handle;
|
||||
struct kvm_hyp_memcache stage2_teardown_mc;
|
||||
bool is_protected;
|
||||
bool is_created;
|
||||
|
||||
/*
|
||||
|
|
@ -376,8 +395,24 @@ enum fgt_group_id {
|
|||
__NR_FGT_GROUP_IDS__
|
||||
};
|
||||
|
||||
enum kvm_arm_vm_flavor {
|
||||
VM_NVHE,
|
||||
VM_VHE,
|
||||
/* VMs running on a hyp that doesn't trust */
|
||||
MARKER(__VM_DISTRUSTING_HYP),
|
||||
VM_PKVM, /* Normal guests on pKVM */
|
||||
MARKER(__VM_PROTECTED),
|
||||
VM_PROTECTED_PKVM, /* Protected VM */
|
||||
VM_FLAVOR_MAX
|
||||
};
|
||||
|
||||
struct kvm_arch {
|
||||
struct kvm_s2_mmu mmu;
|
||||
const struct kvm_vm_s2_ops *vm_s2_ops;
|
||||
|
||||
enum kvm_arm_vm_flavor vm_flavor;
|
||||
/* Mandated version of PSCI */
|
||||
u32 psci_version;
|
||||
|
||||
/*
|
||||
* Fine-Grained UNDEF, mimicking the FGT layout defined by the
|
||||
|
|
@ -408,9 +443,6 @@ struct kvm_arch {
|
|||
/* Timers */
|
||||
struct arch_timer_vm_data timer_data;
|
||||
|
||||
/* Mandated version of PSCI */
|
||||
u32 psci_version;
|
||||
|
||||
/* Protects VM-scoped configuration data */
|
||||
struct mutex config_lock;
|
||||
|
||||
|
|
@ -886,6 +918,7 @@ struct vncr_tlb;
|
|||
|
||||
struct kvm_vcpu_arch {
|
||||
struct kvm_cpu_context ctxt;
|
||||
const struct kvm_vcpu_ops *vcpu_ops;
|
||||
|
||||
/*
|
||||
* Guest floating point state
|
||||
|
|
@ -1532,10 +1565,31 @@ struct kvm *kvm_arch_alloc_vm(void);
|
|||
|
||||
#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE
|
||||
|
||||
#define kvm_vm_is_protected(kvm) (is_protected_kvm_enabled() && (kvm)->arch.pkvm.is_protected)
|
||||
#ifdef __KVM_NVHE_HYPERVISOR__
|
||||
|
||||
#define kvm_vm_is_protected(kvm) \
|
||||
(is_protected_kvm_enabled() && ((kvm)->arch.vm_flavor == VM_PROTECTED_PKVM))
|
||||
/*
|
||||
* Accessing vcpu->kvm from nVHE hyp stub is tricky, as we need to convert the
|
||||
* pointer to the hyp VA. With pKVM, the nVHE code runs with the hyp_vcpu,
|
||||
* which is populated correctly and is gated on is_protected_kvm_enabled().
|
||||
*/
|
||||
#define vcpu_is_protected(vcpu) \
|
||||
({ \
|
||||
struct kvm *__kvm = READ_ONCE((vcpu)->kvm); \
|
||||
\
|
||||
(__kvm && kvm_vm_is_protected(__kvm)); \
|
||||
})
|
||||
|
||||
#else
|
||||
|
||||
#define kvm_vm_is_protected(kvm) ((kvm)->arch.vm_flavor >= __VM_PROTECTED)
|
||||
#define vcpu_is_protected(vcpu) kvm_vm_is_protected((vcpu)->kvm)
|
||||
|
||||
#define kvm_vm_hyp_is_distrusting(kvm) ((kvm)->arch.vm_flavor >= __VM_DISTRUSTING_HYP)
|
||||
|
||||
#endif /* __KVM_NVHE_HYPERVISOR__ */
|
||||
|
||||
#define kvm_has_mte(kvm) \
|
||||
(system_supports_mte() && \
|
||||
test_bit(KVM_ARCH_FLAG_MTE_ENABLED, &(kvm)->arch.flags))
|
||||
|
|
|
|||
|
|
@ -7,10 +7,18 @@
|
|||
#ifndef __ARM64_KVM_PGTABLE_H__
|
||||
#define __ARM64_KVM_PGTABLE_H__
|
||||
|
||||
#include <linux/bitfield.h>
|
||||
#include <linux/bits.h>
|
||||
#include <linux/kvm_host.h>
|
||||
#include <linux/kvm_types.h>
|
||||
#include <linux/rbtree_types.h>
|
||||
#include <linux/rcupdate.h>
|
||||
#include <linux/types.h>
|
||||
|
||||
#include <asm/memory.h>
|
||||
#include <asm/page.h>
|
||||
|
||||
struct kvm_s2_mmu;
|
||||
|
||||
#define KVM_PGTABLE_FIRST_LEVEL -1
|
||||
#define KVM_PGTABLE_LAST_LEVEL 3
|
||||
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@
|
|||
#define __ARM64_KVM_PKVM_H__
|
||||
|
||||
#include <linux/arm_ffa.h>
|
||||
#include <linux/kvm_host.h>
|
||||
#include <linux/memblock.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <asm/kvm_host.h>
|
||||
#include <asm/kvm_pgtable.h>
|
||||
|
||||
/* Maximum number of VMs that can co-exist under pKVM. */
|
||||
|
|
@ -18,7 +18,7 @@
|
|||
#define HYP_MEMBLOCK_REGIONS 128
|
||||
|
||||
void pkvm_selftests(void);
|
||||
int pkvm_init_host_vm(struct kvm *kvm, unsigned long type);
|
||||
int pkvm_init_host_vm(struct kvm *kvm);
|
||||
int pkvm_create_hyp_vm(struct kvm *kvm);
|
||||
bool pkvm_hyp_vm_is_created(struct kvm *kvm);
|
||||
void pkvm_destroy_hyp_vm(struct kvm *kvm);
|
||||
|
|
@ -59,7 +59,7 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext)
|
|||
case KVM_CAP_PRE_FAULT_MEMORY:
|
||||
return false;
|
||||
default:
|
||||
return !kvm || !kvm_vm_is_protected(kvm);
|
||||
return !kvm || (kvm->arch.vm_flavor == VM_PKVM);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1107,8 +1107,7 @@ void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
|
|||
timer_context_init(vcpu, i);
|
||||
|
||||
/* Synchronize offsets across timers of a VM if not already provided */
|
||||
if (!vcpu_is_protected(vcpu) &&
|
||||
!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) {
|
||||
if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) {
|
||||
timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read());
|
||||
timer_set_offset(vcpu_ptimer(vcpu), 0);
|
||||
}
|
||||
|
|
@ -1130,6 +1129,15 @@ void kvm_timer_init_vm(struct kvm *kvm)
|
|||
*/
|
||||
for (int i = 0; i < NR_KVM_TIMERS; i++)
|
||||
kvm->arch.timer_data.ppi[i] = get_vgic_ppi(kvm, default_ppi[i]);
|
||||
|
||||
/*
|
||||
* Protected VMs don't allow the userspace to set counter offsets,
|
||||
* either set via counter register writes or the dedicated ioctls.
|
||||
* Pretend the offset has already been set and rely on the default
|
||||
* offset being 0.
|
||||
*/
|
||||
if (kvm_vm_is_protected(kvm))
|
||||
set_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &kvm->arch.flags);
|
||||
}
|
||||
|
||||
void kvm_timer_cpu_up(void)
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ static const struct kvm_ioctl_cap_map vm_ioctl_caps[] = {
|
|||
{ KVM_ARM_PREFERRED_TARGET, KVM_CAP_ARM_BASIC },
|
||||
};
|
||||
|
||||
static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu);
|
||||
/*
|
||||
* Set *ext to the capability.
|
||||
* Return 0 if found, or -EINVAL if no IOCTL matches.
|
||||
|
|
@ -214,6 +215,26 @@ static int kvm_arm_default_max_vcpus(void)
|
|||
return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
|
||||
}
|
||||
|
||||
static int kvm_init_vm_flavor(struct kvm *kvm, unsigned long type)
|
||||
{
|
||||
bool protected = type & KVM_VM_TYPE_ARM_PROTECTED;
|
||||
|
||||
if (is_protected_kvm_enabled()) {
|
||||
if (protected)
|
||||
kvm->arch.vm_flavor = VM_PROTECTED_PKVM;
|
||||
else
|
||||
kvm->arch.vm_flavor = VM_PKVM;
|
||||
} else if (protected) {
|
||||
return -EINVAL;
|
||||
} else if (has_vhe()) {
|
||||
kvm->arch.vm_flavor = VM_VHE;
|
||||
} else {
|
||||
kvm->arch.vm_flavor = VM_NVHE;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* kvm_arch_init_vm - initializes a VM data structure
|
||||
* @kvm: pointer to the KVM struct
|
||||
|
|
@ -236,6 +257,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
|
|||
mutex_unlock(&kvm->lock);
|
||||
#endif
|
||||
|
||||
ret = kvm_init_vm_flavor(kvm, type);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ret = kvm_share_hyp(kvm, kvm + 1);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
|
@ -259,12 +284,9 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
|
|||
* If any failures occur after this is successful, make sure to
|
||||
* call __pkvm_unreserve_vm to unreserve the VM in hyp.
|
||||
*/
|
||||
ret = pkvm_init_host_vm(kvm, type);
|
||||
ret = pkvm_init_host_vm(kvm);
|
||||
if (ret)
|
||||
goto err_uninit_mmu;
|
||||
} else if (type & KVM_VM_TYPE_ARM_PROTECTED) {
|
||||
ret = -EINVAL;
|
||||
goto err_uninit_mmu;
|
||||
}
|
||||
|
||||
kvm_vgic_early_init(kvm);
|
||||
|
|
@ -553,6 +575,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
|
|||
mutex_unlock(&vcpu->mutex);
|
||||
#endif
|
||||
|
||||
kvm_init_vcpu_ops(vcpu);
|
||||
|
||||
/* Force users to call KVM_ARM_VCPU_INIT */
|
||||
vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
|
||||
|
||||
|
|
@ -615,33 +639,34 @@ void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
|
|||
|
||||
static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) {
|
||||
/*
|
||||
* Either we're running an L2 guest, and the API/APK bits come
|
||||
* from L1's HCR_EL2, or API/APK are both set.
|
||||
*/
|
||||
if (unlikely(is_nested_ctxt(vcpu))) {
|
||||
u64 val;
|
||||
if (!vcpu_has_ptrauth(vcpu))
|
||||
return;
|
||||
|
||||
val = __vcpu_sys_reg(vcpu, HCR_EL2);
|
||||
val &= (HCR_API | HCR_APK);
|
||||
vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
|
||||
vcpu->arch.hcr_el2 |= val;
|
||||
} else {
|
||||
vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
|
||||
}
|
||||
/*
|
||||
* Either we're running an L2 guest, and the API/APK bits come
|
||||
* from L1's HCR_EL2, or API/APK are both set.
|
||||
*/
|
||||
if (unlikely(is_nested_ctxt(vcpu))) {
|
||||
u64 val;
|
||||
|
||||
/*
|
||||
* Save the host keys if there is any chance for the guest
|
||||
* to use pauth, as the entry code will reload the guest
|
||||
* keys in that case.
|
||||
*/
|
||||
if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) {
|
||||
struct kvm_cpu_context *ctxt;
|
||||
val = __vcpu_sys_reg(vcpu, HCR_EL2);
|
||||
val &= (HCR_API | HCR_APK);
|
||||
vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
|
||||
vcpu->arch.hcr_el2 |= val;
|
||||
} else {
|
||||
vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
|
||||
}
|
||||
|
||||
ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt);
|
||||
ptrauth_save_keys(ctxt);
|
||||
}
|
||||
/*
|
||||
* Save the host keys if there is any chance for the guest
|
||||
* to use pauth, as the entry code will reload the guest
|
||||
* keys in that case.
|
||||
*/
|
||||
if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) {
|
||||
struct kvm_cpu_context *ctxt;
|
||||
|
||||
ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt);
|
||||
ptrauth_save_keys(ctxt);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -667,14 +692,11 @@ static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu)
|
|||
return single_task_running();
|
||||
}
|
||||
|
||||
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
|
||||
static void vcpu_prepare_mmu(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
struct kvm_s2_mmu *mmu;
|
||||
int *last_ran;
|
||||
|
||||
if (is_protected_kvm_enabled())
|
||||
goto nommu;
|
||||
|
||||
if (vcpu_has_nv(vcpu))
|
||||
kvm_vcpu_load_hw_mmu(vcpu);
|
||||
|
||||
|
|
@ -704,25 +726,15 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
|
|||
kvm_call_hyp(__kvm_flush_cpu_context, mmu);
|
||||
*last_ran = vcpu->vcpu_idx;
|
||||
}
|
||||
}
|
||||
|
||||
nommu:
|
||||
vcpu->cpu = cpu;
|
||||
|
||||
/*
|
||||
* The timer must be loaded before the vgic to correctly set up physical
|
||||
* interrupt deactivation in nested state (e.g. timer interrupt).
|
||||
*/
|
||||
kvm_timer_vcpu_load(vcpu);
|
||||
kvm_vgic_load(vcpu);
|
||||
kvm_vcpu_load_debug(vcpu);
|
||||
kvm_vcpu_load_fgt(vcpu);
|
||||
if (has_vhe())
|
||||
kvm_vcpu_load_vhe(vcpu);
|
||||
kvm_arch_vcpu_load_fp(vcpu);
|
||||
kvm_vcpu_pmu_restore_guest(vcpu);
|
||||
if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
|
||||
kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
|
||||
static void vcpu_put_mmu(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
kvm_arm_vmid_clear_active();
|
||||
}
|
||||
|
||||
static void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (kvm_vcpu_should_clear_twe(vcpu))
|
||||
vcpu->arch.hcr_el2 &= ~HCR_TWE;
|
||||
else
|
||||
|
|
@ -732,16 +744,73 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
|
|||
vcpu->arch.hcr_el2 &= ~HCR_TWI;
|
||||
else
|
||||
vcpu->arch.hcr_el2 |= HCR_TWI;
|
||||
}
|
||||
|
||||
static void vcpu_load_pvtime(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
|
||||
kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
|
||||
}
|
||||
|
||||
static void vhe_vcpu_load(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
vcpu_prepare_mmu(vcpu);
|
||||
/*
|
||||
* For VHE, the timer must be loaded before the vgic to correctly
|
||||
* set up physical interrupt deactivation in nested state (e.g. timer
|
||||
* interrupt).
|
||||
*/
|
||||
kvm_timer_vcpu_load(vcpu);
|
||||
kvm_vgic_load(vcpu);
|
||||
kvm_vcpu_load_debug(vcpu);
|
||||
kvm_vcpu_load_fgt(vcpu);
|
||||
kvm_vcpu_load_vhe(vcpu);
|
||||
kvm_arch_vcpu_load_fp(vcpu);
|
||||
kvm_vcpu_pmu_restore_guest(vcpu);
|
||||
|
||||
vcpu_load_pvtime(vcpu);
|
||||
vcpu_set_wfx_traps(vcpu);
|
||||
vcpu_set_pauth_traps(vcpu);
|
||||
}
|
||||
|
||||
if (is_protected_kvm_enabled()) {
|
||||
kvm_call_hyp_nvhe(__pkvm_vcpu_load,
|
||||
vcpu->kvm->arch.pkvm.handle,
|
||||
vcpu->vcpu_idx, vcpu->arch.hcr_el2);
|
||||
kvm_call_hyp(__vgic_v3_restore_vmcr_aprs,
|
||||
&vcpu->arch.vgic_cpu.vgic_v3);
|
||||
}
|
||||
static void nvhe_vcpu_load(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
vcpu_prepare_mmu(vcpu);
|
||||
kvm_timer_vcpu_load(vcpu);
|
||||
kvm_vgic_load(vcpu);
|
||||
kvm_vcpu_load_debug(vcpu);
|
||||
kvm_vcpu_load_fgt(vcpu);
|
||||
kvm_arch_vcpu_load_fp(vcpu);
|
||||
kvm_vcpu_pmu_restore_guest(vcpu);
|
||||
|
||||
vcpu_load_pvtime(vcpu);
|
||||
vcpu_set_wfx_traps(vcpu);
|
||||
vcpu_set_pauth_traps(vcpu);
|
||||
}
|
||||
|
||||
static void pkvm_vcpu_load(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
kvm_timer_vcpu_load(vcpu);
|
||||
kvm_vgic_load(vcpu);
|
||||
kvm_vcpu_load_debug(vcpu);
|
||||
kvm_vcpu_load_fgt(vcpu);
|
||||
kvm_arch_vcpu_load_fp(vcpu);
|
||||
kvm_vcpu_pmu_restore_guest(vcpu);
|
||||
|
||||
vcpu_load_pvtime(vcpu);
|
||||
vcpu_set_wfx_traps(vcpu);
|
||||
|
||||
kvm_call_hyp_nvhe(__pkvm_vcpu_load,
|
||||
vcpu->kvm->arch.pkvm.handle,
|
||||
vcpu->vcpu_idx, vcpu->arch.hcr_el2);
|
||||
kvm_call_hyp_nvhe(__vgic_v3_restore_vmcr_aprs,
|
||||
&vcpu->arch.vgic_cpu.vgic_v3);
|
||||
}
|
||||
|
||||
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
|
||||
{
|
||||
vcpu->cpu = cpu;
|
||||
vcpu->arch.vcpu_ops->vcpu_load(vcpu);
|
||||
|
||||
if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus))
|
||||
vcpu_set_on_unsupported_cpu(vcpu);
|
||||
|
|
@ -749,28 +818,50 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
|
|||
vcpu->arch.pid = pid_nr(vcpu->pid);
|
||||
}
|
||||
|
||||
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
|
||||
static void vhe_vcpu_put(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (is_protected_kvm_enabled()) {
|
||||
kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3);
|
||||
kvm_call_hyp_nvhe(__pkvm_vcpu_put);
|
||||
|
||||
/* __pkvm_vcpu_put implies a sync of the state */
|
||||
if (!kvm_vm_is_protected(vcpu->kvm))
|
||||
vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY);
|
||||
}
|
||||
|
||||
kvm_vcpu_put_debug(vcpu);
|
||||
kvm_arch_vcpu_put_fp(vcpu);
|
||||
if (has_vhe())
|
||||
kvm_vcpu_put_vhe(vcpu);
|
||||
kvm_vcpu_put_vhe(vcpu);
|
||||
kvm_timer_vcpu_put(vcpu);
|
||||
kvm_vgic_put(vcpu);
|
||||
kvm_vcpu_pmu_restore_host(vcpu);
|
||||
|
||||
if (vcpu_has_nv(vcpu))
|
||||
kvm_vcpu_put_hw_mmu(vcpu);
|
||||
kvm_arm_vmid_clear_active();
|
||||
|
||||
vcpu_put_mmu(vcpu);
|
||||
}
|
||||
|
||||
static void nvhe_vcpu_put(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
kvm_vcpu_put_debug(vcpu);
|
||||
kvm_arch_vcpu_put_fp(vcpu);
|
||||
kvm_timer_vcpu_put(vcpu);
|
||||
kvm_vgic_put(vcpu);
|
||||
kvm_vcpu_pmu_restore_host(vcpu);
|
||||
vcpu_put_mmu(vcpu);
|
||||
}
|
||||
|
||||
static void pkvm_vcpu_put(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
kvm_call_hyp_nvhe(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3);
|
||||
kvm_call_hyp_nvhe(__pkvm_vcpu_put);
|
||||
|
||||
/* __pkvm_vcpu_put implies a sync of the state */
|
||||
if (vcpu->kvm->arch.vm_flavor == VM_PKVM)
|
||||
vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY);
|
||||
|
||||
kvm_vcpu_put_debug(vcpu);
|
||||
kvm_arch_vcpu_put_fp(vcpu);
|
||||
kvm_timer_vcpu_put(vcpu);
|
||||
kvm_vgic_put(vcpu);
|
||||
kvm_vcpu_pmu_restore_host(vcpu);
|
||||
}
|
||||
|
||||
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
vcpu->arch.vcpu_ops->vcpu_put(vcpu);
|
||||
vcpu_clear_on_unsupported_cpu(vcpu);
|
||||
vcpu->cpu = -1;
|
||||
}
|
||||
|
|
@ -1013,7 +1104,7 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
|
|||
|
||||
if (is_protected_kvm_enabled()) {
|
||||
/* Start with the vcpu in a dirty state */
|
||||
if (!kvm_vm_is_protected(vcpu->kvm))
|
||||
if (vcpu->kvm->arch.vm_flavor == VM_PKVM)
|
||||
vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY);
|
||||
ret = pkvm_create_hyp_vm(kvm);
|
||||
if (ret)
|
||||
|
|
@ -1613,10 +1704,13 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
|
|||
return -EINVAL;
|
||||
}
|
||||
|
||||
static unsigned long system_supported_vcpu_features(void)
|
||||
static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
unsigned long features = KVM_VCPU_VALID_FEATURES;
|
||||
|
||||
if (vcpu->kvm->arch.vm_flavor == VM_PROTECTED_PKVM)
|
||||
kvm_pkvm_vcpu_allowed_features(vcpu->kvm, &features);
|
||||
|
||||
if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1))
|
||||
clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features);
|
||||
|
||||
|
|
@ -1654,21 +1748,13 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
|
|||
return -ENOENT;
|
||||
}
|
||||
|
||||
if (features & ~system_supported_vcpu_features())
|
||||
if (features & ~system_supported_vcpu_features(vcpu))
|
||||
return -EINVAL;
|
||||
|
||||
/* Reject features EL2 would drop when it creates the hyp VM. */
|
||||
if (vcpu_is_protected(vcpu)) {
|
||||
DECLARE_BITMAP(allowed, KVM_VCPU_MAX_FEATURES);
|
||||
|
||||
kvm_pkvm_vcpu_allowed_features(vcpu->kvm, allowed);
|
||||
if (!bitmap_subset(&features, allowed, KVM_VCPU_MAX_FEATURES))
|
||||
return -EINVAL;
|
||||
|
||||
/* EL2 implements PSCI 1.1; the host must not dispatch as 0.1. */
|
||||
if (!test_bit(KVM_ARM_VCPU_PSCI_0_2, &features))
|
||||
return -EINVAL;
|
||||
}
|
||||
/* EL2 implements PSCI 1.1; the host must not dispatch as 0.1. */
|
||||
if (vcpu_is_protected(vcpu) &&
|
||||
!test_bit(KVM_ARM_VCPU_PSCI_0_2, &features))
|
||||
return -EINVAL;
|
||||
|
||||
/*
|
||||
* For now make sure that both address/generic pointer authentication
|
||||
|
|
@ -2179,6 +2265,37 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
|
|||
}
|
||||
}
|
||||
|
||||
static const struct kvm_vcpu_ops vhe_vcpu_ops = {
|
||||
.vcpu_load = vhe_vcpu_load,
|
||||
.vcpu_put = vhe_vcpu_put,
|
||||
};
|
||||
|
||||
static const struct kvm_vcpu_ops nvhe_vcpu_ops = {
|
||||
.vcpu_load = nvhe_vcpu_load,
|
||||
.vcpu_put = nvhe_vcpu_put,
|
||||
};
|
||||
|
||||
static const struct kvm_vcpu_ops pkvm_vcpu_ops = {
|
||||
.vcpu_load = pkvm_vcpu_load,
|
||||
.vcpu_put = pkvm_vcpu_put,
|
||||
};
|
||||
|
||||
#define KVM_VCPU_OPS(flavor, ops) \
|
||||
[(flavor)] = &(ops)
|
||||
|
||||
static const struct kvm_vcpu_ops *arm64_vcpu_ops[] = {
|
||||
KVM_VCPU_OPS(VM_NVHE, nvhe_vcpu_ops),
|
||||
KVM_VCPU_OPS(VM_VHE, vhe_vcpu_ops),
|
||||
KVM_VCPU_OPS(VM_PKVM, pkvm_vcpu_ops),
|
||||
KVM_VCPU_OPS(VM_PROTECTED_PKVM, pkvm_vcpu_ops),
|
||||
};
|
||||
|
||||
static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
BUILD_BUG_ON(ARRAY_SIZE(arm64_vcpu_ops) != VM_FLAVOR_MAX);
|
||||
vcpu->arch.vcpu_ops = arm64_vcpu_ops[vcpu->kvm->arch.vm_flavor];
|
||||
}
|
||||
|
||||
static unsigned long nvhe_percpu_size(void)
|
||||
{
|
||||
return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
|
||||
|
|
|
|||
|
|
@ -498,7 +498,11 @@ static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm,
|
|||
|
||||
hyp_vm->host_kvm = host_kvm;
|
||||
hyp_vm->kvm.created_vcpus = nr_vcpus;
|
||||
hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected);
|
||||
if (READ_ONCE(host_kvm->arch.vm_flavor) == VM_PROTECTED_PKVM)
|
||||
hyp_vm->kvm.arch.vm_flavor = VM_PROTECTED_PKVM;
|
||||
else
|
||||
hyp_vm->kvm.arch.vm_flavor = VM_PKVM;
|
||||
|
||||
hyp_vm->kvm.arch.flags = 0;
|
||||
pkvm_init_features_from_host(hyp_vm, host_kvm);
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
*/
|
||||
|
||||
#include <linux/bitfield.h>
|
||||
#include <linux/kvm_host.h>
|
||||
#include <asm/kvm_pgtable.h>
|
||||
#include <asm/stage2_pgtable.h>
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
#include <linux/kvm_host.h>
|
||||
#include <asm/kvm_emulate.h>
|
||||
#include <asm/kvm_mmu.h>
|
||||
#include <trace/events/kvm.h>
|
||||
|
||||
#include "trace.h"
|
||||
|
|
|
|||
|
|
@ -39,6 +39,8 @@ static unsigned long __ro_after_init io_map_base;
|
|||
|
||||
#define KVM_PGT_FN(fn) (!is_protected_kvm_enabled() ? fn : p ## fn)
|
||||
|
||||
static int kvm_vm_init_vm_s2_ops(struct kvm *kvm);
|
||||
|
||||
static phys_addr_t __stage2_range_addr_end(phys_addr_t addr, phys_addr_t end,
|
||||
phys_addr_t size)
|
||||
{
|
||||
|
|
@ -177,6 +179,18 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot)
|
|||
return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY);
|
||||
}
|
||||
|
||||
static int pkvm_flush_remote_tlbs(struct kvm *kvm)
|
||||
{
|
||||
kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int kvm_vm_flush_remote_tlbs(struct kvm *kvm)
|
||||
{
|
||||
kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* kvm_arch_flush_remote_tlbs() - flush all VM TLB entries for v7/8
|
||||
* @kvm: pointer to kvm structure.
|
||||
|
|
@ -185,26 +199,31 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot)
|
|||
*/
|
||||
int kvm_arch_flush_remote_tlbs(struct kvm *kvm)
|
||||
{
|
||||
if (is_protected_kvm_enabled())
|
||||
kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle);
|
||||
else
|
||||
kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu);
|
||||
return 0;
|
||||
return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs(kvm);
|
||||
}
|
||||
|
||||
int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm,
|
||||
gfn_t gfn, u64 nr_pages)
|
||||
static int pkvm_flush_remote_tlbs_range(struct kvm *kvm,
|
||||
gfn_t gfn, u64 nr_pages)
|
||||
{
|
||||
return pkvm_flush_remote_tlbs(kvm);
|
||||
}
|
||||
|
||||
static int kvm_vm_flush_remote_tlbs_range(struct kvm *kvm,
|
||||
gfn_t gfn, u64 nr_pages)
|
||||
{
|
||||
u64 size = nr_pages << PAGE_SHIFT;
|
||||
u64 addr = gfn << PAGE_SHIFT;
|
||||
|
||||
if (is_protected_kvm_enabled())
|
||||
kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle);
|
||||
else
|
||||
kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size);
|
||||
kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm,
|
||||
gfn_t gfn, u64 nr_pages)
|
||||
{
|
||||
return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs_range(kvm, gfn, nr_pages);
|
||||
}
|
||||
|
||||
static void *stage2_memcache_zalloc_page(void *arg)
|
||||
{
|
||||
struct kvm_mmu_memory_cache *mc = arg;
|
||||
|
|
@ -300,6 +319,37 @@ static void invalidate_icache_guest_page(void *va, size_t size)
|
|||
__invalidate_icache_guest_page(va, size);
|
||||
}
|
||||
|
||||
static int kvm_pgtable_stage2_unmap_tracked(struct kvm_pgtable *pgt,
|
||||
u64 addr, u64 size);
|
||||
|
||||
static void kvm_vm_stage2_unmap_range(struct kvm_s2_mmu *mmu,
|
||||
phys_addr_t start,
|
||||
u64 size, bool may_block)
|
||||
{
|
||||
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
|
||||
int (*fn)(struct kvm_pgtable *, u64, u64);
|
||||
|
||||
if (kvm_is_nested_s2_mmu(kvm, mmu))
|
||||
fn = kvm_pgtable_stage2_unmap_tracked;
|
||||
else
|
||||
fn = kvm_pgtable_stage2_unmap;
|
||||
|
||||
WARN_ON(stage2_apply_range(mmu, start, start + size, fn, may_block));
|
||||
}
|
||||
|
||||
static void pkvm_stage2_unmap_range(struct kvm_s2_mmu *mmu,
|
||||
phys_addr_t start,
|
||||
u64 size, bool may_block)
|
||||
{
|
||||
WARN_ON(stage2_apply_range(mmu, start, start + size,
|
||||
pkvm_pgtable_stage2_unmap, may_block));
|
||||
}
|
||||
|
||||
static void no_stage2_unmap_range(struct kvm_s2_mmu *mmu,
|
||||
phys_addr_t start, u64 size, bool may_block)
|
||||
{
|
||||
}
|
||||
|
||||
/*
|
||||
* Unmapping vs dcache management:
|
||||
*
|
||||
|
|
@ -338,42 +388,25 @@ static int kvm_pgtable_stage2_unmap_tracked(struct kvm_pgtable *pgt, u64 addr, u
|
|||
}
|
||||
|
||||
/**
|
||||
* __unmap_stage2_range -- Clear stage2 page table entries to unmap a range
|
||||
* kvm_stage2_unmap_range -- Clear stage2 page table entries to unmap a range
|
||||
* @mmu: The KVM stage-2 MMU pointer
|
||||
* @start: The intermediate physical base address of the range to unmap
|
||||
* @size: The size of the area to unmap
|
||||
* @may_block: Whether or not we are permitted to block
|
||||
*
|
||||
* Clear a range of stage-2 mappings, lowering the various ref-counts. Must
|
||||
* be called while holding mmu_lock (unless for freeing the stage2 pgd before
|
||||
* destroying the VM), otherwise another faulting VCPU may come in and mess
|
||||
* with things behind our backs.
|
||||
* be called while holding mmu_lock otherwise another faulting VCPU may
|
||||
* come in and mess with things behind our backs.
|
||||
*/
|
||||
static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size,
|
||||
bool may_block)
|
||||
void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start,
|
||||
u64 size, bool may_block)
|
||||
{
|
||||
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
|
||||
phys_addr_t end = start + size;
|
||||
int (*fn)(struct kvm_pgtable *, u64, u64);
|
||||
|
||||
lockdep_assert_held_write(&kvm->mmu_lock);
|
||||
WARN_ON(size & ~PAGE_MASK);
|
||||
|
||||
if (kvm_is_nested_s2_mmu(kvm, mmu))
|
||||
fn = kvm_pgtable_stage2_unmap_tracked;
|
||||
else
|
||||
fn = KVM_PGT_FN(kvm_pgtable_stage2_unmap);
|
||||
|
||||
WARN_ON(stage2_apply_range(mmu, start, end, fn, may_block));
|
||||
}
|
||||
|
||||
void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start,
|
||||
u64 size, bool may_block)
|
||||
{
|
||||
if (kvm_vm_is_protected(kvm_s2_mmu_to_kvm(mmu)))
|
||||
return;
|
||||
|
||||
__unmap_stage2_range(mmu, start, size, may_block);
|
||||
kvm->arch.vm_s2_ops->vm_stage2_unmap_range(mmu, start, size, may_block);
|
||||
}
|
||||
|
||||
void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end)
|
||||
|
|
@ -1013,6 +1046,12 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t
|
|||
int cpu, err;
|
||||
struct kvm_pgtable *pgt;
|
||||
|
||||
/* Initialize the VM ops for the VM instance for the first time */
|
||||
if (mmu == &kvm->arch.mmu) {
|
||||
err = kvm_vm_init_vm_s2_ops(kvm);
|
||||
if (err)
|
||||
return err;
|
||||
}
|
||||
/*
|
||||
* If we already have our page tables in place, and that the
|
||||
* MMU context is the canonical one, we have a bug somewhere,
|
||||
|
|
@ -1269,7 +1308,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
|
|||
KVM_PGTABLE_PROT_R |
|
||||
(writable ? KVM_PGTABLE_PROT_W : 0);
|
||||
|
||||
if (is_protected_kvm_enabled())
|
||||
if (kvm_vm_hyp_is_distrusting(kvm))
|
||||
return -EPERM;
|
||||
|
||||
size += offset_in_page(guest_ipa);
|
||||
|
|
@ -1825,7 +1864,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
|
||||
static int protected_pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
|
||||
{
|
||||
unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE;
|
||||
struct kvm_s2_fault_vma_info s2vi = {};
|
||||
|
|
@ -2292,6 +2331,18 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd,
|
|||
return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache, result);
|
||||
}
|
||||
|
||||
static int kvm_vm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
|
||||
{
|
||||
VM_WARN_ON_ONCE(kvm_s2_fault_is_perm(s2fd) &&
|
||||
!esr_abt_is_write_fault(s2fd->esr) &&
|
||||
!kvm_s2_fault_is_exec(s2fd));
|
||||
|
||||
if (kvm_slot_has_gmem(s2fd->memslot))
|
||||
return gmem_abort(s2fd, NULL);
|
||||
else
|
||||
return user_mem_abort(s2fd, NULL);
|
||||
}
|
||||
|
||||
/* Resolve the access fault by making the page young again. */
|
||||
static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
|
||||
{
|
||||
|
|
@ -2401,6 +2452,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
struct kvm_s2_trans nested_trans, *nested = NULL;
|
||||
unsigned long esr = kvm_vcpu_get_esr(vcpu);
|
||||
struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu;
|
||||
struct kvm *kvm = vcpu->kvm;
|
||||
phys_addr_t fault_ipa; /* The address we faulted on */
|
||||
phys_addr_t ipa; /* Always the IPA in the L1 guest phys space */
|
||||
struct kvm_memory_slot *memslot;
|
||||
|
|
@ -2417,7 +2469,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
* with an SEA.
|
||||
*/
|
||||
ipa = fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
|
||||
if (KVM_BUG_ON(ipa == INVALID_GPA, vcpu->kvm))
|
||||
if (KVM_BUG_ON(ipa == INVALID_GPA, kvm))
|
||||
return -EFAULT;
|
||||
|
||||
is_iabt = esr_trap_is_iabt(esr);
|
||||
|
|
@ -2452,7 +2504,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
return -EFAULT;
|
||||
}
|
||||
|
||||
idx = srcu_read_lock(&vcpu->kvm->srcu);
|
||||
idx = srcu_read_lock(&kvm->srcu);
|
||||
|
||||
/*
|
||||
* We may have faulted on a shadow stage 2 page table if we are
|
||||
|
|
@ -2467,8 +2519,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
* nothing to walk and we treat it as a 1:1 before going through the
|
||||
* canonical translation.
|
||||
*/
|
||||
if (kvm_is_nested_s2_mmu(vcpu->kvm, mmu) &&
|
||||
mmu->nested_stage2_enabled) {
|
||||
if (kvm_is_nested_s2_mmu(kvm, mmu) && mmu->nested_stage2_enabled) {
|
||||
u32 esr;
|
||||
|
||||
ret = kvm_walk_nested_s2(vcpu, fault_ipa, &nested_trans);
|
||||
|
|
@ -2495,7 +2546,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
}
|
||||
|
||||
gfn = ipa >> PAGE_SHIFT;
|
||||
memslot = gfn_to_memslot(vcpu->kvm, gfn);
|
||||
memslot = gfn_to_memslot(kvm, gfn);
|
||||
hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
|
||||
write_fault = esr_abt_is_write_fault(esr);
|
||||
if (kvm_is_error_hva(hva) || (write_fault && !writable)) {
|
||||
|
|
@ -2561,25 +2612,14 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
|
|||
.mmu = mmu,
|
||||
};
|
||||
|
||||
if (kvm_vm_is_protected(vcpu->kvm)) {
|
||||
ret = pkvm_mem_abort(&s2fd);
|
||||
} else {
|
||||
VM_WARN_ON_ONCE(kvm_s2_fault_is_perm(&s2fd) && !write_fault &&
|
||||
!kvm_s2_fault_is_exec(&s2fd));
|
||||
|
||||
if (kvm_slot_has_gmem(memslot))
|
||||
ret = gmem_abort(&s2fd, NULL);
|
||||
else
|
||||
ret = user_mem_abort(&s2fd, NULL);
|
||||
}
|
||||
|
||||
ret = kvm->arch.vm_s2_ops->vm_mem_abort(&s2fd);
|
||||
if (ret == 0)
|
||||
ret = 1;
|
||||
out:
|
||||
if (ret == -ENOEXEC)
|
||||
ret = kvm_inject_sea_iabt(vcpu, kvm_vcpu_get_hfar(vcpu));
|
||||
out_unlock:
|
||||
srcu_read_unlock(&vcpu->kvm->srcu, idx);
|
||||
srcu_read_unlock(&kvm->srcu, idx);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
@ -2589,43 +2629,77 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
|
|||
size_t size = (range->end - range->start) << PAGE_SHIFT;
|
||||
bool may_block = range->may_block;
|
||||
|
||||
if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))
|
||||
if (!kvm->arch.mmu.pgt)
|
||||
return false;
|
||||
|
||||
__unmap_stage2_range(&kvm->arch.mmu, gpa, size, may_block);
|
||||
kvm_stage2_unmap_range(&kvm->arch.mmu, gpa, size, may_block);
|
||||
kvm_nested_unmap_cipa_range(kvm, gpa, size, may_block);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
|
||||
static bool kvm_vm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
|
||||
{
|
||||
u64 size = (range->end - range->start) << PAGE_SHIFT;
|
||||
|
||||
if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))
|
||||
return false;
|
||||
|
||||
return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt,
|
||||
return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt,
|
||||
range->start << PAGE_SHIFT,
|
||||
size, true);
|
||||
}
|
||||
|
||||
static bool pkvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
|
||||
{
|
||||
u64 size = (range->end - range->start) << PAGE_SHIFT;
|
||||
|
||||
return pkvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt,
|
||||
range->start << PAGE_SHIFT,
|
||||
size, true);
|
||||
}
|
||||
|
||||
static bool no_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
|
||||
{
|
||||
/* The hypervisor doesn't support aging */
|
||||
return false;
|
||||
}
|
||||
|
||||
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
|
||||
{
|
||||
if (!kvm->arch.mmu.pgt)
|
||||
return false;
|
||||
|
||||
return kvm->arch.vm_s2_ops->vm_age_gfn(kvm, range);
|
||||
/*
|
||||
* TODO: Handle nested_mmu structures here using the reverse mapping in
|
||||
* a later version of patch series.
|
||||
*/
|
||||
}
|
||||
|
||||
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
|
||||
static bool kvm_vm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
|
||||
{
|
||||
u64 size = (range->end - range->start) << PAGE_SHIFT;
|
||||
|
||||
if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))
|
||||
return false;
|
||||
|
||||
return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt,
|
||||
return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt,
|
||||
range->start << PAGE_SHIFT,
|
||||
size, false);
|
||||
}
|
||||
|
||||
static bool pkvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
|
||||
{
|
||||
u64 size = (range->end - range->start) << PAGE_SHIFT;
|
||||
|
||||
return pkvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt,
|
||||
range->start << PAGE_SHIFT,
|
||||
size, false);
|
||||
}
|
||||
|
||||
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
|
||||
{
|
||||
if (!kvm->arch.mmu.pgt)
|
||||
return false;
|
||||
|
||||
return kvm->arch.vm_s2_ops->vm_test_age_gfn(kvm, range);
|
||||
}
|
||||
|
||||
phys_addr_t kvm_mmu_get_httbr(void)
|
||||
{
|
||||
return __pa(hyp_pgtable->pgd);
|
||||
|
|
@ -3090,3 +3164,50 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
|
|||
return ret;
|
||||
return pre_fault_bytes_consumed(gpa, ret, bytes_remaining);
|
||||
}
|
||||
|
||||
static const struct kvm_vm_s2_ops protected_pkvm_vm_s2_ops = {
|
||||
.vm_flush_remote_tlbs = pkvm_flush_remote_tlbs,
|
||||
.vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range,
|
||||
.vm_age_gfn = no_age_gfn,
|
||||
.vm_test_age_gfn = no_age_gfn,
|
||||
.vm_stage2_unmap_range = no_stage2_unmap_range,
|
||||
.vm_mem_abort = protected_pkvm_mem_abort,
|
||||
};
|
||||
|
||||
static const struct kvm_vm_s2_ops pkvm_vm_s2_ops = {
|
||||
.vm_flush_remote_tlbs = pkvm_flush_remote_tlbs,
|
||||
.vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range,
|
||||
.vm_age_gfn = pkvm_age_gfn,
|
||||
.vm_test_age_gfn = pkvm_test_age_gfn,
|
||||
.vm_stage2_unmap_range = pkvm_stage2_unmap_range,
|
||||
.vm_mem_abort = kvm_vm_mem_abort,
|
||||
};
|
||||
|
||||
static const struct kvm_vm_s2_ops kvm_default_vm_s2_ops = {
|
||||
.vm_flush_remote_tlbs = kvm_vm_flush_remote_tlbs,
|
||||
.vm_flush_remote_tlbs_range = kvm_vm_flush_remote_tlbs_range,
|
||||
.vm_age_gfn = kvm_vm_age_gfn,
|
||||
.vm_test_age_gfn = kvm_vm_test_age_gfn,
|
||||
.vm_stage2_unmap_range = kvm_vm_stage2_unmap_range,
|
||||
.vm_mem_abort = kvm_vm_mem_abort,
|
||||
};
|
||||
|
||||
#define KVM_VM_S2_OPS(flavor, ops) \
|
||||
[(flavor)] = &(ops)
|
||||
|
||||
static const struct kvm_vm_s2_ops *arm64_vm_s2_ops[] = {
|
||||
KVM_VM_S2_OPS(VM_VHE, kvm_default_vm_s2_ops),
|
||||
KVM_VM_S2_OPS(VM_NVHE, kvm_default_vm_s2_ops),
|
||||
KVM_VM_S2_OPS(VM_PKVM, pkvm_vm_s2_ops),
|
||||
KVM_VM_S2_OPS(VM_PROTECTED_PKVM, protected_pkvm_vm_s2_ops),
|
||||
};
|
||||
|
||||
static int kvm_vm_init_vm_s2_ops(struct kvm *kvm)
|
||||
{
|
||||
BUILD_BUG_ON(ARRAY_SIZE(arm64_vm_s2_ops) != VM_FLAVOR_MAX);
|
||||
|
||||
kvm->arch.vm_s2_ops = arm64_vm_s2_ops[kvm->arch.vm_flavor];
|
||||
if (WARN_ON(!kvm->arch.vm_s2_ops))
|
||||
return -EINVAL;
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -294,10 +294,9 @@ void pkvm_destroy_hyp_vm(struct kvm *kvm)
|
|||
mutex_unlock(&kvm->arch.config_lock);
|
||||
}
|
||||
|
||||
int pkvm_init_host_vm(struct kvm *kvm, unsigned long type)
|
||||
int pkvm_init_host_vm(struct kvm *kvm)
|
||||
{
|
||||
int ret;
|
||||
bool protected = type & KVM_VM_TYPE_ARM_PROTECTED;
|
||||
|
||||
/* Reserve the VM in hyp and obtain a hyp handle for the VM. */
|
||||
ret = kvm_call_hyp_nvhe(__pkvm_reserve_vm);
|
||||
|
|
@ -305,8 +304,7 @@ int pkvm_init_host_vm(struct kvm *kvm, unsigned long type)
|
|||
return ret;
|
||||
|
||||
kvm->arch.pkvm.handle = ret;
|
||||
kvm->arch.pkvm.is_protected = protected;
|
||||
if (protected) {
|
||||
if (kvm_vm_is_protected(kvm)) {
|
||||
pr_warn_once("kvm: protected VMs are experimental and for development only, tainting kernel\n");
|
||||
add_taint(TAINT_USER, LOCKDEP_STILL_OK);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -84,6 +84,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
|
|||
!kvm_vgic_global_state.can_emulate_gicv2)
|
||||
return -ENODEV;
|
||||
|
||||
if (kvm_vm_hyp_is_distrusting(kvm) && type != KVM_DEV_TYPE_ARM_VGIC_V3)
|
||||
return -ENODEV;
|
||||
/*
|
||||
* Ensure mutual exclusion with vCPU creation and any vCPU ioctls by:
|
||||
*
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@
|
|||
#include <linux/kvm_host.h>
|
||||
#include <uapi/linux/psci.h>
|
||||
|
||||
#include <asm/kvm_emulate.h>
|
||||
|
||||
#define KVM_ARM_PSCI_0_1 PSCI_VERSION(0, 1)
|
||||
#define KVM_ARM_PSCI_0_2 PSCI_VERSION(0, 2)
|
||||
#define KVM_ARM_PSCI_1_0 PSCI_VERSION(1, 0)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user