github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-23 05:04:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'kvm-arm64/feat-nv3' into next

Browse Source 
* kvm-arm64/feat-nv3: (29 commits)
  : Support for FEAT_NV2p1 and FEAT_NV3, courtesy of Marc Zyngier
  :
  : Couple of accelerations for nested virtualization:
  :
  :  - Elide traps for EL1 alias registers when hardware guarantees that
  :    EL2-only bits are stateful (NV2p1)
  :
  :  - Use TGE-aware hardware to avoid unnecesary ERET/TLBI traps that
  :    remain 'in-context' (i.e. L1 kernel/userspace)
  arm64: Add override for ID_AA64MMFR4_EL1.NV_frac
  KVM: arm64: selftest: Add NVHCR_EL2 to get-reg-list
  KVM: arm64: Expose FEAT_NV3 to guests
  KVM: arm64: Add FEAT_NV3 detection
  KVM: arm64: Engage NV3 TLBI trap elision
  KVM: arm64: Engage NV3 ERET trap elision
  KVM: arm64: Add NVHCR_EL2 context switching
  KVM: arm64: Add routing for NVHCR_EL2 trap
  KVM: arm64: Add NVHCR_EL2 handling to the sysreg array
  KVM: arm64: Add sanitisation for NVHCR_EL2
  KVM: arm64: Make HCR_EL2 a non-VNCR register
  KVM: arm64: Add kvm_has_nv{2,3}() predicates
  KVM: arm64: Add NV3 control bits to HCRX_EL2 sanitisation
  KVM: arm64: Split NV-specific exit fixups from the non-NV handling
  arm64: Add ARM64_HAS_NV3 capability
  arm64: sysreg: Add HCRX_EL2 bits related to FEAT_NV3
  arm64: sysreg: Add NVHCR_EL2 description as a mirror of HCR_EL2
  arm64: Add FEAT_NV2p1 detection
  KVM: arm64: Expose FEAT_NV2p1 to NV guests
  KVM: arm64: Relax CNTHCTL_EL2 handling when FEAT_NV2p1 is present
  ...

Signed-off-by: Oliver Upton <oupton@kernel.org>
This commit is contained in:
Oliver Upton 2026-08-19 12:27:53 -07:00
commit 5f44f603b0
21 changed files with 322 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arch/arm64/include/asm/cpufeature.h
View File

@ -968,6 +968,7 @@ struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id);
extern struct arm64_ftr_override id_aa64mmfr0_override;
extern struct arm64_ftr_override id_aa64mmfr1_override;
extern struct arm64_ftr_override id_aa64mmfr2_override;
extern struct arm64_ftr_override id_aa64mmfr4_override;
extern struct arm64_ftr_override id_aa64pfr0_override;
extern struct arm64_ftr_override id_aa64pfr1_override;
extern struct arm64_ftr_override id_aa64zfr0_override;

15
arch/arm64/include/asm/kvm_arm.h
View File

@ -287,21 +287,6 @@
GENMASK(19, 18) | \
GENMASK(15, 0))
/*
* Polarity masks for HCRX_EL2, limited to the bits that we know about
* at this point in time. It doesn't mean that we actually *handle*
* them, but that at least those that are not advertised to a guest
* will be RES0 for that guest.
*/
#define __HCRX_EL2_MASK (BIT_ULL(6))
#define __HCRX_EL2_nMASK (GENMASK_ULL(24, 14) | \
GENMASK_ULL(11, 7) | \
GENMASK_ULL(5, 0))
#define __HCRX_EL2_RES0 ~(__HCRX_EL2_nMASK | __HCRX_EL2_MASK)
#define __HCRX_EL2_RES1 ~(__HCRX_EL2_nMASK | \
__HCRX_EL2_MASK | \
__HCRX_EL2_RES0)
/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
#define HPFAR_MASK (~UL(0xf))
/*

44
arch/arm64/include/asm/kvm_emulate.h
View File

@ -266,6 +266,25 @@ static inline bool vserror_state_is_nested(struct kvm_vcpu *vcpu)
(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA);
}
static inline bool kvm_has_nv2(struct kvm *kvm)
{
return (cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) &&
kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY));
}
static inline bool kvm_has_nv3(struct kvm *kvm)
{
return (cpus_have_final_cap(ARM64_HAS_NV3) &&
kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV3));
}
static inline bool is_nested_nv3_ctxt(struct kvm_vcpu *vcpu)
{
return (has_vhe() && kvm_has_nv3(vcpu->kvm) && is_nested_ctxt(vcpu) &&
(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_NV) &&
(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_NVTGE));
}
/*
* The layout of SPSR for an AArch32 state is different when observed from an
* AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32
@ -623,7 +642,7 @@ static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu)
*/
static inline u64 vcpu_sanitised_cptr_el2(const struct kvm_vcpu *vcpu)
{
u64 cptr = __vcpu_sys_reg(vcpu, CPTR_EL2);
u64 cptr = vcpu_read_sys_reg(vcpu, CPTR_EL2);
if (!vcpu_el2_e2h_is_set(vcpu))
cptr = translate_cptr_el2_to_cpacr_el1(cptr);
@ -692,6 +711,29 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu)
if (kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V))
vcpu->arch.hcrx_el2 |= HCRX_EL2_EnASR;
/*
* NV3 is a host-specific extension, and we always use
* it when present and that the guest uses NV. It may
* be hidden from the guest though.
*/
if (cpus_have_final_cap(ARM64_HAS_NV3) &&
vcpu_has_nv(vcpu) && vcpu_el2_e2h_is_set(vcpu)) {
vcpu->arch.hcrx_el2 |= HCRX_EL2_NVTGE;
/*
* If the guest is NV2-capable, then we need to see
* all the TLBIs, as configured in HCR_EL2.
* Otherwise, relax the TLBI traps to only TGE=0.
*/
if (!kvm_has_nv2(vcpu->kvm)) {
vcpu->arch.hcrx_el2 |= (HCRX_EL2_NVnTTLB |
HCRX_EL2_NVnTTLBIS);
if (kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
vcpu->arch.hcrx_el2 |= HCRX_EL2_NVnTTLBOS;
}
}
}
}

3
arch/arm64/include/asm/kvm_host.h
View File

@ -546,6 +546,7 @@ enum vcpu_sysreg {
MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */
CNTHCTL_EL2, /* Counter-timer Hypervisor Control register */
ZCR_EL2, /* SVE Control Register (EL2) */
HCR_EL2, /* Hypervisor Control Register */
/* Any VNCR-capable reg goes after this point */
MARKER(__VNCR_START__),
@ -574,7 +575,7 @@ enum vcpu_sysreg {
VNCR(TFSR_EL1), /* Tag Fault Status Register (EL1) */
VNCR(VPIDR_EL2),/* Virtualization Processor ID Register */
VNCR(VMPIDR_EL2),/* Virtualization Multiprocessor ID Register */
VNCR(HCR_EL2), /* Hypervisor Configuration Register */
VNCR(NVHCR_EL2),/* NV Hypervisor Configuration Register */
VNCR(HSTR_EL2), /* Hypervisor System Trap Register */
VNCR(VTTBR_EL2),/* Virtualization Translation Table Base Register */
VNCR(VTCR_EL2), /* Virtualization Translation Control Register */

2
arch/arm64/include/asm/vncr_mapping.h
View File

@ -11,7 +11,7 @@
#define VNCR_VTCR_EL2 0x040
#define VNCR_VMPIDR_EL2 0x050
#define VNCR_CNTVOFF_EL2 0x060
#define VNCR_HCR_EL2 0x078
#define VNCR_NVHCR_EL2 0x078
#define VNCR_HSTR_EL2 0x080
#define VNCR_VPIDR_EL2 0x088
#define VNCR_TPIDR_EL2 0x090

18
arch/arm64/kernel/cpufeature.c
View File

@ -785,6 +785,7 @@ static const struct arm64_ftr_bits ftr_raz[] = {
struct arm64_ftr_override __read_mostly id_aa64mmfr0_override;
struct arm64_ftr_override __read_mostly id_aa64mmfr1_override;
struct arm64_ftr_override __read_mostly id_aa64mmfr2_override;
struct arm64_ftr_override __read_mostly id_aa64mmfr4_override;
struct arm64_ftr_override __read_mostly id_aa64pfr0_override;
struct arm64_ftr_override __read_mostly id_aa64pfr1_override;
struct arm64_ftr_override __read_mostly id_aa64zfr0_override;
@ -858,7 +859,8 @@ static const struct __ftr_reg_entry {
ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2,
&id_aa64mmfr2_override),
ARM64_FTR_REG(SYS_ID_AA64MMFR3_EL1, ftr_id_aa64mmfr3),
ARM64_FTR_REG(SYS_ID_AA64MMFR4_EL1, ftr_id_aa64mmfr4),
ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR4_EL1, ftr_id_aa64mmfr4,
&id_aa64mmfr4_override),
/* Op1 = 0, CRn = 10, CRm = 4 */
ARM64_FTR_REG(SYS_MPAMIDR_EL1, ftr_mpamidr),
@ -2620,6 +2622,20 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
{ /* Sentinel */ }
},
},
{
.desc = "FEAT_NV2p1",
.capability = ARM64_HAS_NV2P1,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
ARM64_CPUID_FIELDS(ID_AA64MMFR4_EL1, NV_frac, NV2P1)
},
{
.desc = "FEAT_NV3",
.capability = ARM64_HAS_NV3,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
ARM64_CPUID_FIELDS(ID_AA64MMFR4_EL1, NV_frac, NV3)
},
{
.capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,

1
arch/arm64/kernel/image-vars.h
View File

@ -51,6 +51,7 @@ PI_EXPORT_SYM(id_aa64isar2_override);
PI_EXPORT_SYM(id_aa64mmfr0_override);
PI_EXPORT_SYM(id_aa64mmfr1_override);
PI_EXPORT_SYM(id_aa64mmfr2_override);
PI_EXPORT_SYM(id_aa64mmfr4_override);
PI_EXPORT_SYM(id_aa64pfr0_override);
PI_EXPORT_SYM(id_aa64pfr1_override);
PI_EXPORT_SYM(id_aa64smfr0_override);

10
arch/arm64/kernel/pi/idreg-override.c
View File

@ -106,6 +106,15 @@ static const struct ftr_set_desc mmfr2 __prel64_initconst = {
},
};
static const struct ftr_set_desc mmfr4 __prel64_initconst = {
.name = "id_aa64mmfr4",
.override = &id_aa64mmfr4_override,
.fields = {
FIELD("nv_frac", ID_AA64MMFR4_EL1_NV_frac_SHIFT, NULL),
{}
},
};
static bool __init pfr0_sve_filter(u64 val)
{
/*
@ -220,6 +229,7 @@ PREL64(const struct ftr_set_desc, reg) regs[] __prel64_initconst = {
{ &mmfr0 },
{ &mmfr1 },
{ &mmfr2 },
{ &mmfr4 },
{ &pfr0 },
{ &pfr1 },
{ &isar1 },

10
arch/arm64/kvm/arch_timer.c
View File

@ -876,8 +876,14 @@ static void timer_set_traps(struct kvm_vcpu *vcpu, struct timer_map *map)
assign_clear_set_bit(tvt02, CNTHCTL_EL1NVVCT, clr, set);
assign_clear_set_bit(tpt02, CNTHCTL_EL1NVPCT, clr, set);
/* This only happens on VHE, so use the CNTHCTL_EL2 accessor. */
sysreg_clear_set(cnthctl_el2, clr, set);
/*
* This only happens on VHE, so use the CNTHCTL_EL2 accessor, unless
* we are sure CNTKCTL_EL1 is completely stateful with FEAT_NV2p1.
*/
if (!cpus_have_final_cap(ARM64_HAS_NV2P1))
sysreg_clear_set(cnthctl_el2, clr, set);
else
sysreg_clear_set(cntkctl_el1, clr, set);
}
void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)

27
arch/arm64/kvm/config.c
View File

@ -225,6 +225,7 @@ struct reg_feat_map_desc {
#define FEAT_HCX ID_AA64MMFR1_EL1, HCX, IMP
#define FEAT_S2PIE ID_AA64MMFR3_EL1, S2PIE, IMP
#define FEAT_GCIE ID_AA64PFR2_EL1, GCIE, IMP
#define FEAT_NV3 ID_AA64MMFR4_EL1, NV_frac, NV3
static bool not_feat_aa64el3(struct kvm *kvm)
{
@ -904,6 +905,12 @@ static const DECLARE_FEAT_MAP_FGT(hdfgwtr2_desc, hdfgwtr2_masks,
static const struct reg_bits_to_feat_map hcrx_feat_map[] = {
NEEDS_FEAT(HCRX_EL2_NVTGE |
HCRX_EL2_NVnTTLB |
HCRX_EL2_NVnTTLBIS |
HCRX_EL2_NVnTTLBOS,
FEAT_NV3),
NEEDS_FEAT(HCRX_EL2_SRMASKEn, FEAT_SRMASK),
NEEDS_FEAT(HCRX_EL2_PACMEn, feat_pauth_lr),
NEEDS_FEAT(HCRX_EL2_EnFPM, FEAT_FPMR),
NEEDS_FEAT(HCRX_EL2_GCSEn, FEAT_GCS),
@ -930,10 +937,12 @@ static const struct reg_bits_to_feat_map hcrx_feat_map[] = {
NEEDS_FEAT(HCRX_EL2_EnASR, FEAT_LS64_V),
NEEDS_FEAT(HCRX_EL2_EnALS, FEAT_LS64),
NEEDS_FEAT(HCRX_EL2_EnAS0, FEAT_LS64_ACCDATA),
FORCE_RES0(HCRX_EL2_RES0),
FORCE_RES1(HCRX_EL2_RES1),
};
static const DECLARE_FEAT_MAP(hcrx_desc, __HCRX_EL2,
static const DECLARE_FEAT_MAP(hcrx_desc, HCRX_EL2,
hcrx_feat_map, FEAT_HCX);
static const struct reg_bits_to_feat_map hcr_feat_map[] = {
@ -1010,6 +1019,9 @@ static const struct reg_bits_to_feat_map hcr_feat_map[] = {
static const DECLARE_FEAT_MAP(hcr_desc, HCR_EL2,
hcr_feat_map, FEAT_AA64EL2);
static const DECLARE_FEAT_MAP(nvhcr_desc, NVHCR_EL2,
hcr_feat_map, FEAT_NV3);
static const struct reg_bits_to_feat_map sctlr2_feat_map[] = {
NEEDS_FEAT(SCTLR2_EL1_NMEA |
SCTLR2_EL1_EASE,
@ -1384,6 +1396,7 @@ void __init check_feature_map(void)
check_reg_desc(&hdfgwtr2_desc);
check_reg_desc(&hcrx_desc);
check_reg_desc(&hcr_desc);
check_reg_desc(&nvhcr_desc);
check_reg_desc(&sctlr2_desc);
check_reg_desc(&tcr2_el2_desc);
check_reg_desc(&sctlr_el1_desc);
@ -1579,11 +1592,21 @@ struct resx get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg)
break;
case HCRX_EL2:
resx = compute_reg_resx_bits(kvm, &hcrx_desc, 0, 0);
resx.res1 |= __HCRX_EL2_RES1;
break;
case HCR_EL2:
resx = compute_reg_resx_bits(kvm, &hcr_desc, 0, 0);
break;
case NVHCR_EL2:
/*
* Only apply sanitisation if we do have FEAT_NV3.
* Otherwise, the register aliases with HCR_EL2 in VNCR,
* and we're better off relying on data transfers between
* NVHCR_EL2 and HCR_EL2 to sanitise things.
*/
resx = (kvm_has_nv3(kvm) ?
compute_reg_resx_bits(kvm, &nvhcr_desc, 0, 0) :
(typeof(resx)){});
break;
case SCTLR2_EL1:
case SCTLR2_EL2:
resx = compute_reg_resx_bits(kvm, &sctlr2_desc, 0, 0);

18
arch/arm64/kvm/emulate-nested.c
View File

@ -136,6 +136,8 @@ enum cgt_group_id {
CGT_CPTR_TTA,
CGT_MDCR_HPMN,
CGT_HCR_NV_HCRX_nNVTGE,
/* Must be last */
__NR_CGT_GROUP_IDS__
};
@ -588,6 +590,15 @@ static enum trap_behaviour check_mdcr_hpmn(struct kvm_vcpu *vcpu)
return BEHAVE_HANDLE_LOCALLY;
}
static enum trap_behaviour check_hcr_nv_hcrx_nnvtge(struct kvm_vcpu *vcpu)
{
if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_NV) &&
!(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_NVTGE))
return BEHAVE_FORWARD_RW;
return BEHAVE_HANDLE_LOCALLY;
}
#define CCC(id, fn) \
[id - __COMPLEX_CONDITIONS__] = fn
@ -598,6 +609,7 @@ static const complex_condition_check ccc[] = {
CCC(CGT_CNTHCTL_EL1NVVCT, check_cnthctl_el1nvvct),
CCC(CGT_CPTR_TTA, check_cptr_tta),
CCC(CGT_MDCR_HPMN, check_mdcr_hpmn),
CCC(CGT_HCR_NV_HCRX_nNVTGE, check_hcr_nv_hcrx_nnvtge),
};
/*
@ -853,6 +865,7 @@ static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = {
SR_TRAP(SYS_SCTLR2_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_HCR_EL2,
SYS_HCRX_EL2, CGT_HCR_NV),
SR_TRAP(SYS_NVHCR_EL2, CGT_HCR_NV_HCRX_nNVTGE),
SR_TRAP(SYS_SMPRIMAP_EL2, CGT_HCR_NV),
SR_TRAP(SYS_SMCR_EL2, CGT_HCR_NV),
SR_RANGE_TRAP(SYS_TTBR0_EL2,
@ -2320,7 +2333,6 @@ int __init populate_nv_trap_config(void)
BUILD_BUG_ON(__NR_CGT_GROUP_IDS__ > BIT(TC_CGT_BITS));
BUILD_BUG_ON(__NR_FGT_GROUP_IDS__ > BIT(TC_FGT_BITS));
BUILD_BUG_ON(__NR_FG_FILTER_IDS__ > BIT(TC_FGF_BITS));
BUILD_BUG_ON(__HCRX_EL2_MASK & __HCRX_EL2_nMASK);
for (int i = 0; i < ARRAY_SIZE(encoding_to_cgt); i++) {
const struct encoding_to_trap_config *cgt = &encoding_to_cgt[i];
@ -2346,10 +2358,6 @@ int __init populate_nv_trap_config(void)
}
}
if (__HCRX_EL2_RES0 != HCRX_EL2_RES0)
kvm_info("Sanitised HCR_EL2_RES0 = %016llx, expecting %016llx\n",
__HCRX_EL2_RES0, HCRX_EL2_RES0);
kvm_info("nv: %ld coarse grained trap handlers\n",
ARRAY_SIZE(encoding_to_cgt));

27
arch/arm64/kvm/hyp/include/hyp/switch.h
View File

@ -108,9 +108,10 @@ static inline void __activate_cptr_traps_vhe(struct kvm_vcpu *vcpu)
* The architecture is a bit crap (what a surprise): an EL2 guest
* writing to CPTR_EL2 via CPACR_EL1 can't set any of TCPAC or TTA,
* as they are RES0 in the guest's view. To work around it, trap the
* sucker using the very same bit it can't set...
* sucker using the very same bit it can't set. FEAT_NV2p1 fixes it.
*/
if (vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu))
if (!cpus_have_final_cap(ARM64_HAS_NV2P1) &&
vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu))
val |= CPTR_EL2_TCPAC;
/*
@ -325,6 +326,24 @@ static inline void __deactivate_traps_mpam(void)
write_sysreg_s(MPAMHCR_HOST_FLAGS, SYS_MPAMHCR_EL2);
}
/*
* Just like for HCR_EL2, we can't let the guest mess with some of the
* basics we rely on in HCRX_EL2. However, the major difference is that
* HCRX_EL2 only affects EL1, and never EL2 (sudden outburst of sanity, I
* guess). So it is always the guest inflicting it on its own guestx.
*
* Things we don't want to let the guest control are:
*
* - TMEA: That's for us to decide how an SEA is routed, not the guest.
*
* - PTTWI: Similarly, it is for us to decide whether Reduced Coherency for
* the PTW is a thing. It really isn't.
*
* - EnIDCP128: We don't allow IMPDEF sysregs -- full stop.
*/
#define NV_HCRX_GUEST_EXCLUDE (HCRX_EL2_TMEA | HCRX_EL2_PTTWI | \
HCRX_EL2_EnIDCP128)
static inline void __activate_traps_common(struct kvm_vcpu *vcpu)
{
struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
@ -350,8 +369,8 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu)
u64 hcrx = vcpu->arch.hcrx_el2;
if (is_nested_ctxt(vcpu)) {
u64 val = __vcpu_sys_reg(vcpu, HCRX_EL2);
hcrx |= val & __HCRX_EL2_MASK;
hcrx &= ~(~val & __HCRX_EL2_nMASK);
hcrx |= (val & ~NV_HCRX_GUEST_EXCLUDE);
hcrx &= ~(~val & ~NV_HCRX_GUEST_EXCLUDE);
}
ctxt_sys_reg(hctxt, HCRX_EL2) = read_sysreg_s(SYS_HCRX_EL2);

11
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
View File

@ -172,6 +172,10 @@ static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
if (ctxt_has_sctlr2(ctxt))
ctxt_sys_reg(ctxt, SCTLR2_EL1) = read_sysreg_el1(SYS_SCTLR2);
/* Retrieve L2's HCR_EL2, and save it for future use */
if (is_nested_nv3_ctxt(ctxt_to_vcpu(ctxt)))
ctxt_sys_reg(ctxt, NVHCR_EL2) = read_sysreg_s(SYS_NVHCR_EL2);
}
static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
@ -285,6 +289,13 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt,
if (ctxt_has_sctlr2(ctxt))
write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR2_EL1), SYS_SCTLR2);
/*
* Publish the L2 view of HCR_EL2 to the HW if L1 is using NV3.
* Otherwise, the data is already in place in the L1's own VNCR.
*/
if (is_nested_nv3_ctxt(ctxt_to_vcpu(ctxt)))
write_sysreg_s(ctxt_sys_reg(ctxt, NVHCR_EL2), SYS_NVHCR_EL2);
}
/* Read the VCPU state's PSTATE, but translate (v)EL2 to EL1. */

52
arch/arm64/kvm/hyp/vhe/switch.c
View File

@ -70,6 +70,12 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu)
if (!vcpu_el2_e2h_is_set(vcpu))
hcr |= HCR_NV1;
/* Publish the guest's view of HCR_EL2 to the HW */
if (cpus_have_final_cap(ARM64_HAS_NV3) && vcpu_el2_e2h_is_set(vcpu))
write_sysreg_s(__vcpu_sys_reg(vcpu, HCR_EL2), SYS_NVHCR_EL2);
else
__vcpu_assign_sys_reg(vcpu, NVHCR_EL2, __vcpu_sys_reg(vcpu, HCR_EL2));
/*
* Nothing in HCR_EL2 should impact running in hypervisor
* context, apart from bits we have defined as RESx (E2H,
@ -339,18 +345,24 @@ static bool kvm_hyp_handle_eret(struct kvm_vcpu *vcpu, u64 *exit_code)
u64 esr = kvm_vcpu_get_esr(vcpu);
u64 spsr, elr, mode;
/* With NV3, the fast path is handled in HW */
if (cpus_have_final_cap(ARM64_HAS_NV3) && vcpu_el2_e2h_is_set(vcpu))
return false;
/*
* Going through the whole put/load motions is a waste of time
* if this is a VHE guest hypervisor returning to its own
* userspace, or the hypervisor performing a local exception
* return. No need to save/restore registers, no need to
* switch S2 MMU. Just do the canonical ERET.
* switch S2 MMU. Just do the canonical ERET unless we are in
* nested context.
*
* Unless the trap has to be forwarded further down the line,
* of course...
* Note that this is made possible because KVM itself never traps
* ERET when running an L2. The consequence is that any ERET trap is
* the result of HCR_EL2 or HFGITR_EL2 programming by L1 for its own
* guest, and the exception must be forwarded to L1.
*/
if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) ||
(__vcpu_sys_reg(vcpu, HFGITR_EL2) & HFGITR_EL2_ERET))
if (is_nested_ctxt(vcpu))
return false;
spsr = read_sysreg_el1(SYS_SPSR);
@ -441,6 +453,9 @@ static bool kvm_hyp_handle_cpacr_el1(struct kvm_vcpu *vcpu, u64 *exit_code)
u64 esr = kvm_vcpu_get_esr(vcpu);
int rt;
if (cpus_have_final_cap(ARM64_HAS_NV2P1))
return false;
if (!is_hyp_ctxt(vcpu) || esr_sys64_to_sysreg(esr) != SYS_CPACR_EL1)
return false;
@ -534,19 +549,17 @@ static const exit_handler_fn hyp_exit_handlers[] = {
[0x3F] = kvm_hyp_handle_impdef,
};
static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
static void fixup_nv_guest_exit(struct kvm_vcpu *vcpu)
{
synchronize_vcpu_pstate(vcpu);
/*
* If we were in HYP context on entry, adjust the PSTATE view
* so that the usual helpers work correctly. This enforces our
* invariant that the guest's HYP context status is preserved
* across a run.
*/
if (vcpu_has_nv(vcpu) &&
unlikely(host_data_test_flag(VCPU_IN_HYP_CONTEXT))) {
if (unlikely(host_data_test_flag(VCPU_IN_HYP_CONTEXT))) {
u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT);
u64 hcr;
switch (mode) {
case PSR_MODE_EL1t:
@ -559,11 +572,26 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
*vcpu_cpsr(vcpu) &= ~(PSR_MODE_MASK | PSR_MODE32_BIT);
*vcpu_cpsr(vcpu) |= mode;
/* Publish the latest HCR_EL2 to the emulation */
hcr = (cpus_have_final_cap(ARM64_HAS_NV3) &&
vcpu_el2_e2h_is_set(vcpu)) ?
read_sysreg_s(SYS_NVHCR_EL2) :
__vcpu_sys_reg(vcpu, NVHCR_EL2);
__vcpu_assign_sys_reg(vcpu, HCR_EL2, hcr);
}
/* Apply extreme paranoia! */
BUG_ON(vcpu_has_nv(vcpu) &&
!!host_data_test_flag(VCPU_IN_HYP_CONTEXT) != is_hyp_ctxt(vcpu));
BUG_ON(!!host_data_test_flag(VCPU_IN_HYP_CONTEXT) != is_hyp_ctxt(vcpu));
}
static bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
{
synchronize_vcpu_pstate(vcpu);
if (vcpu_has_nv(vcpu))
fixup_nv_guest_exit(vcpu);
return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers);
}

21
arch/arm64/kvm/hyp/vhe/sysreg-sr.c
View File

@ -42,10 +42,12 @@ static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu)
u64 val;
/*
* We don't save CPTR_EL2, as accesses to CPACR_EL1
* are always trapped, ensuring that the in-memory
* copy is always up-to-date. A small blessing...
* Without FEAT_NV2p1, we don't save CPTR_EL2, as accesses
* to CPACR_EL1 are always trapped, ensuring that the
* in-memory copy is always up-to-date. A small blessing...
*/
if (cpus_have_final_cap(ARM64_HAS_NV2P1))
__vcpu_assign_sys_reg(vcpu, CPTR_EL2, read_sysreg_el1(SYS_CPACR));
__vcpu_assign_sys_reg(vcpu, SCTLR_EL2, read_sysreg_el1(SYS_SCTLR));
__vcpu_assign_sys_reg(vcpu, TTBR0_EL2, read_sysreg_el1(SYS_TTBR0));
__vcpu_assign_sys_reg(vcpu, TTBR1_EL2, read_sysreg_el1(SYS_TTBR1));
@ -67,11 +69,18 @@ static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu)
* The EL1 view of CNTKCTL_EL1 has a bunch of RES0 bits where
* the interesting CNTHCTL_EL2 bits live. So preserve these
* bits when reading back the guest-visible value.
*
* While NV2p1 fixes some of that, it makes CNTHCTL_EL2.ECV
* even more broken than it already was with NV2.
*/
val = read_sysreg_el1(SYS_CNTKCTL);
val &= CNTKCTL_VALID_BITS;
__vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, &=, ~CNTKCTL_VALID_BITS);
__vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, |=, val);
if (!cpus_have_final_cap(ARM64_HAS_NV2P1)) {
val &= CNTKCTL_VALID_BITS;
__vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, &=, ~CNTKCTL_VALID_BITS);
__vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, |=, val);
} else {
__vcpu_assign_sys_reg(vcpu, CNTHCTL_EL2, val);
}
}
__vcpu_assign_sys_reg(vcpu, SP_EL2, read_sysreg(sp_el1));

14
arch/arm64/kvm/nested.c
View File

@ -1728,7 +1728,7 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val)
* You get EITHER
*
* - FEAT_VHE without FEAT_E2H0
* - FEAT_NV limited to FEAT_NV2
* - FEAT_NV limited to FEAT_NV2(p1)/NV3
* - HCR_EL2.NV1 being RES0
*
* OR
@ -1740,7 +1740,13 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val)
if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) {
val = 0;
} else {
val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY);
val &= ID_AA64MMFR4_EL1_NV_frac;
if (cpus_have_final_cap(ARM64_HAS_NV3))
val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV3);
else if (cpus_have_final_cap(ARM64_HAS_NV2P1))
val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV2P1);
else
val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY);
val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, E2H0, NI_NV1);
}
break;
@ -1826,6 +1832,10 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu)
resx = get_reg_fixed_bits(kvm, HCR_EL2);
set_sysreg_masks(kvm, HCR_EL2, resx);
/* NVHCR_EL2 */
resx = get_reg_fixed_bits(kvm, NVHCR_EL2);
set_sysreg_masks(kvm, NVHCR_EL2, resx);
/* HCRX_EL2 */
resx = get_reg_fixed_bits(kvm, HCRX_EL2);
set_sysreg_masks(kvm, HCRX_EL2, resx);

57
arch/arm64/kvm/sys_regs.c
View File

@ -183,8 +183,6 @@ static void locate_register(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg,
switch (reg) {
MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1,
translate_sctlr_el2_to_sctlr_el1 );
MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1,
translate_cptr_el2_to_cpacr_el1 );
MAPPED_EL2_SYSREG(TTBR0_EL2, TTBR0_EL1,
translate_ttbr0_el2_to_ttbr0_el1 );
MAPPED_EL2_SYSREG(TTBR1_EL2, TTBR1_EL1, NULL );
@ -210,6 +208,33 @@ static void locate_register(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg,
loc->loc = ((is_hyp_ctxt(vcpu) && vcpu_el2_e2h_is_set(vcpu)) ?
SR_LOC_SPECIAL : SR_LOC_MEMORY);
break;
case CPTR_EL2:
/*
* CPTR_EL2 is just as special, and needs a certain amount
* of handholding. It always lives in memory, due to being
* heavily trapped thanks to CPACR_EL1.TCPAC being RES0.
* FEAT_NV2p1 fixes this.
*/
locate_mapped_el2_register(vcpu, CPTR_EL2, CPACR_EL1,
translate_cptr_el2_to_cpacr_el1,
loc);
if (is_hyp_ctxt(vcpu) && vcpu_el2_e2h_is_set(vcpu))
loc->loc = SR_LOC_SPECIAL;
break;
case NVHCR_EL2:
/*
* Yes, NVHCR_EL2 maps to itself when loaded in nested
* context. If you feel like the architecture is double
* backing on itself upside down, you're not alone.
*/
WARN_ON_ONCE(!kvm_has_nv3(vcpu->kvm));
if (is_hyp_ctxt(vcpu)) {
loc->loc = SR_LOC_MEMORY;
} else {
loc->loc = SR_LOC_LOADED | SR_LOC_MAPPED;
loc->map_reg = NVHCR_EL2;
}
break;
default:
loc->loc = locate_direct_register(vcpu, reg);
}
@ -249,6 +274,7 @@ static u64 read_sr_from_cpu(enum vcpu_sysreg reg)
case DACR32_EL2: val = read_sysreg_s(SYS_DACR32_EL2); break;
case IFSR32_EL2: val = read_sysreg_s(SYS_IFSR32_EL2); break;
case DBGVCR32_EL2: val = read_sysreg_s(SYS_DBGVCR32_EL2); break;
case NVHCR_EL2: val = read_sysreg_s(SYS_NVHCR_EL2); break;
default: WARN_ON_ONCE(1);
}
@ -287,6 +313,7 @@ static void write_sr_to_cpu(enum vcpu_sysreg reg, u64 val)
case DACR32_EL2: write_sysreg_s(val, SYS_DACR32_EL2); break;
case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break;
case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); break;
case NVHCR_EL2: write_sysreg_s(val, SYS_NVHCR_EL2); break;
default: WARN_ON_ONCE(1);
}
}
@ -311,9 +338,16 @@ u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg)
switch (reg) {
case CNTHCTL_EL2:
val = read_sysreg_el1(SYS_CNTKCTL);
val &= CNTKCTL_VALID_BITS;
val |= __vcpu_sys_reg(vcpu, reg) & ~CNTKCTL_VALID_BITS;
if (!cpus_have_final_cap(ARM64_HAS_NV2P1)) {
val &= CNTKCTL_VALID_BITS;
val |= __vcpu_sys_reg(vcpu, reg) & ~CNTKCTL_VALID_BITS;
}
return val;
case CPTR_EL2:
if (cpus_have_final_cap(ARM64_HAS_NV2P1))
return read_sysreg_el1(SYS_CPACR);
else
return __vcpu_sys_reg(vcpu, reg);
default:
WARN_ON_ONCE(1);
}
@ -359,6 +393,9 @@ void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, enum vcpu_sysreg reg)
*/
write_sysreg_el1(val, SYS_CNTKCTL);
break;
case CPTR_EL2:
write_sysreg_el1(val, SYS_CPACR);
break;
default:
WARN_ON_ONCE(1);
}
@ -2887,6 +2924,16 @@ static unsigned int vncr_el2_visibility(const struct kvm_vcpu *vcpu,
return REG_HIDDEN;
}
static unsigned int nvhcr_el2_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
if (el2_visibility(vcpu, rd) == 0 &&
kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV3))
return 0;
return REG_HIDDEN;
}
static unsigned int sctlr2_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
@ -3801,6 +3848,8 @@ static const struct sys_reg_desc sys_reg_descs[] = {
sve_el2_visibility),
EL2_REG_VNCR(HCRX_EL2, reset_val, 0),
EL2_REG_FILTERED(NVHCR_EL2, undef_access, reset_val, 0,
nvhcr_el2_visibility),
EL2_REG(TTBR0_EL2, access_rw, reset_val, 0),
EL2_REG(TTBR1_EL2, access_rw, reset_val, 0),

2
arch/arm64/tools/cpucaps
View File

@ -51,6 +51,8 @@ HAS_LS64_V
HAS_LSUI
HAS_MOPS
HAS_NESTED_VIRT
HAS_NV2P1
HAS_NV3
HAS_BBML2_NOABORT
HAS_PAN
HAS_PMUV3

8
arch/arm64/tools/gen-sysreg.awk
View File

@ -228,7 +228,7 @@ $1 == "EndSysreg" && block_current() == "Sysreg" {
}
# Currently this is effectivey a comment, in future we may want to emit
# defines for the fields.
# defines for the fields. We do emit RESx and UNKN values in any case.
($1 == "Fields" || $1 == "Mapping") && block_current() == "Sysreg" {
expect_fields(2)
@ -239,9 +239,9 @@ $1 == "EndSysreg" && block_current() == "Sysreg" {
print ""
next_bit = -1
res0 = null
res1 = null
unkn = null
res0 = $2 "_RES0"
res1 = $2 "_RES1"
unkn = $2 "_UNKN"
next
}

44
arch/arm64/tools/sysreg
View File

@ -2386,17 +2386,40 @@ EndEnum
EndSysreg
Sysreg ID_AA64MMFR4_EL1 3 0 0 7 4
Res0 63:48
UnsignedEnum 47:44 SRMASK
UnsignedEnum 63:60 MTEFGT
0b0000 NI
0b0001 IMP
EndEnum
UnsignedEnum 59:56 SCRX
0b0000 NI
0b0001 IMP
EndEnum
UnsignedEnum 55:52 TEV
0b0000 NI
0b0001 IMP
EndEnum
UnsignedEnum 51:48 TPS
0b0000 VAL_0000
0b0001 VAL_0001
0b0010 VAL_0010
EndEnum
UnsignedEnum 47:44 SRMASK
0b0000 NI
0b0001 IMP
0b0010 SRMASK2
EndEnum
UnsignedEnum 43:40 TLBID
0b0000 NI
0b0001 IMP
EndEnum
Res0 43:40
UnsignedEnum 39:36 E3DSE
0b0000 NI
0b0001 IMP
EndEnum
Res0 35:32
UnsignedEnum 35:32 EAESR
0b0000 NI
0b0001 IMP
EndEnum
UnsignedEnum 31:28 RMEGDI
0b0000 NI
0b0001 IMP
@ -2410,6 +2433,7 @@ UnsignedEnum 23:20 NV_frac
0b0000 NV_NV2
0b0001 NV2_ONLY
0b0010 NV2P1
0b0011 NV3
EndEnum
UnsignedEnum 19:16 FGWTE3
0b0000 NI
@ -4242,6 +4266,9 @@ Field 1 E2TRE
Field 0 E0HTRE
EndSysreg
Sysreg NVHCR_EL2 3 4 1 5 0
Mapping HCR_EL2
EndSysreg
Sysreg HDFGRTR2_EL2 3 4 3 1 0
Res0 63:25
@ -4521,7 +4548,14 @@ Fields ZCR_ELx
EndSysreg
Sysreg HCRX_EL2 3 4 1 2 2
Res0 63:25
Res0 63:35
Field 34 NVnTTLBOS
Field 33 NVnTTLBIS
Field 32 NVnTTLB
Res0 31:28
Field 27 NVTGE
Field 26 SRMASKEn
Res0 25
Field 24 PACMEn
Field 23 EnFPM
Field 22 GCSEn

2
tools/testing/selftests/kvm/arm64/get-reg-list.c
View File

@ -67,6 +67,7 @@ static struct feature_id_reg feat_id_regs[] = {
REG_FEAT(VDISR_EL2, ID_AA64PFR0_EL1, RAS, IMP),
REG_FEAT(VSESR_EL2, ID_AA64PFR0_EL1, RAS, IMP),
REG_FEAT(VNCR_EL2, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY),
REG_FEAT(NVHCR_EL2, ID_AA64MMFR4_EL1, NV_frac, NV3),
REG_FEAT(CNTHV_CTL_EL2, ID_AA64MMFR1_EL1, VH, IMP),
REG_FEAT(CNTHV_CVAL_EL2,ID_AA64MMFR1_EL1, VH, IMP),
REG_FEAT(ZCR_EL2, ID_AA64PFR0_EL1, SVE, IMP),
@ -771,6 +772,7 @@ static __u64 el2_regs[] = {
SYS_REG(SP_EL2),
SYS_REG(VDISR_EL2),
SYS_REG(VSESR_EL2),
SYS_REG(NVHCR_EL2),
};
static __u64 el2_e2h0_regs[] = {