diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index a57870fa96db..a42683af79fb 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -968,6 +968,7 @@ struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id); extern struct arm64_ftr_override id_aa64mmfr0_override; extern struct arm64_ftr_override id_aa64mmfr1_override; extern struct arm64_ftr_override id_aa64mmfr2_override; +extern struct arm64_ftr_override id_aa64mmfr4_override; extern struct arm64_ftr_override id_aa64pfr0_override; extern struct arm64_ftr_override id_aa64pfr1_override; extern struct arm64_ftr_override id_aa64zfr0_override; diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index aba4ec09acd2..4bfbd827c5aa 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -287,21 +287,6 @@ GENMASK(19, 18) | \ GENMASK(15, 0)) -/* - * Polarity masks for HCRX_EL2, limited to the bits that we know about - * at this point in time. It doesn't mean that we actually *handle* - * them, but that at least those that are not advertised to a guest - * will be RES0 for that guest. - */ -#define __HCRX_EL2_MASK (BIT_ULL(6)) -#define __HCRX_EL2_nMASK (GENMASK_ULL(24, 14) | \ - GENMASK_ULL(11, 7) | \ - GENMASK_ULL(5, 0)) -#define __HCRX_EL2_RES0 ~(__HCRX_EL2_nMASK | __HCRX_EL2_MASK) -#define __HCRX_EL2_RES1 ~(__HCRX_EL2_nMASK | \ - __HCRX_EL2_MASK | \ - __HCRX_EL2_RES0) - /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ #define HPFAR_MASK (~UL(0xf)) /* diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index dbc77d059d7e..a3c1928bdf74 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -266,6 +266,25 @@ static inline bool vserror_state_is_nested(struct kvm_vcpu *vcpu) (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA); } +static inline bool kvm_has_nv2(struct kvm *kvm) +{ + return (cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) && + kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)); +} + +static inline bool kvm_has_nv3(struct kvm *kvm) +{ + return (cpus_have_final_cap(ARM64_HAS_NV3) && + kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV3)); +} + +static inline bool is_nested_nv3_ctxt(struct kvm_vcpu *vcpu) +{ + return (has_vhe() && kvm_has_nv3(vcpu->kvm) && is_nested_ctxt(vcpu) && + (__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_NV) && + (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_NVTGE)); +} + /* * The layout of SPSR for an AArch32 state is different when observed from an * AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32 @@ -623,7 +642,7 @@ static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu) */ static inline u64 vcpu_sanitised_cptr_el2(const struct kvm_vcpu *vcpu) { - u64 cptr = __vcpu_sys_reg(vcpu, CPTR_EL2); + u64 cptr = vcpu_read_sys_reg(vcpu, CPTR_EL2); if (!vcpu_el2_e2h_is_set(vcpu)) cptr = translate_cptr_el2_to_cpacr_el1(cptr); @@ -692,6 +711,29 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) if (kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V)) vcpu->arch.hcrx_el2 |= HCRX_EL2_EnASR; + + /* + * NV3 is a host-specific extension, and we always use + * it when present and that the guest uses NV. It may + * be hidden from the guest though. + */ + if (cpus_have_final_cap(ARM64_HAS_NV3) && + vcpu_has_nv(vcpu) && vcpu_el2_e2h_is_set(vcpu)) { + vcpu->arch.hcrx_el2 |= HCRX_EL2_NVTGE; + + /* + * If the guest is NV2-capable, then we need to see + * all the TLBIs, as configured in HCR_EL2. + * Otherwise, relax the TLBI traps to only TGE=0. + */ + if (!kvm_has_nv2(vcpu->kvm)) { + vcpu->arch.hcrx_el2 |= (HCRX_EL2_NVnTTLB | + HCRX_EL2_NVnTTLBIS); + + if (kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS)) + vcpu->arch.hcrx_el2 |= HCRX_EL2_NVnTTLBOS; + } + } } } diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 71228fa65a7c..f036025c5a5b 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -546,6 +546,7 @@ enum vcpu_sysreg { MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */ CNTHCTL_EL2, /* Counter-timer Hypervisor Control register */ ZCR_EL2, /* SVE Control Register (EL2) */ + HCR_EL2, /* Hypervisor Control Register */ /* Any VNCR-capable reg goes after this point */ MARKER(__VNCR_START__), @@ -574,7 +575,7 @@ enum vcpu_sysreg { VNCR(TFSR_EL1), /* Tag Fault Status Register (EL1) */ VNCR(VPIDR_EL2),/* Virtualization Processor ID Register */ VNCR(VMPIDR_EL2),/* Virtualization Multiprocessor ID Register */ - VNCR(HCR_EL2), /* Hypervisor Configuration Register */ + VNCR(NVHCR_EL2),/* NV Hypervisor Configuration Register */ VNCR(HSTR_EL2), /* Hypervisor System Trap Register */ VNCR(VTTBR_EL2),/* Virtualization Translation Table Base Register */ VNCR(VTCR_EL2), /* Virtualization Translation Control Register */ diff --git a/arch/arm64/include/asm/vncr_mapping.h b/arch/arm64/include/asm/vncr_mapping.h index 14366d35ce82..9e8a49fa8b63 100644 --- a/arch/arm64/include/asm/vncr_mapping.h +++ b/arch/arm64/include/asm/vncr_mapping.h @@ -11,7 +11,7 @@ #define VNCR_VTCR_EL2 0x040 #define VNCR_VMPIDR_EL2 0x050 #define VNCR_CNTVOFF_EL2 0x060 -#define VNCR_HCR_EL2 0x078 +#define VNCR_NVHCR_EL2 0x078 #define VNCR_HSTR_EL2 0x080 #define VNCR_VPIDR_EL2 0x088 #define VNCR_TPIDR_EL2 0x090 diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 9a22df0c5120..14fbfa8e6b7b 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -785,6 +785,7 @@ static const struct arm64_ftr_bits ftr_raz[] = { struct arm64_ftr_override __read_mostly id_aa64mmfr0_override; struct arm64_ftr_override __read_mostly id_aa64mmfr1_override; struct arm64_ftr_override __read_mostly id_aa64mmfr2_override; +struct arm64_ftr_override __read_mostly id_aa64mmfr4_override; struct arm64_ftr_override __read_mostly id_aa64pfr0_override; struct arm64_ftr_override __read_mostly id_aa64pfr1_override; struct arm64_ftr_override __read_mostly id_aa64zfr0_override; @@ -858,7 +859,8 @@ static const struct __ftr_reg_entry { ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2, &id_aa64mmfr2_override), ARM64_FTR_REG(SYS_ID_AA64MMFR3_EL1, ftr_id_aa64mmfr3), - ARM64_FTR_REG(SYS_ID_AA64MMFR4_EL1, ftr_id_aa64mmfr4), + ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR4_EL1, ftr_id_aa64mmfr4, + &id_aa64mmfr4_override), /* Op1 = 0, CRn = 10, CRm = 4 */ ARM64_FTR_REG(SYS_MPAMIDR_EL1, ftr_mpamidr), @@ -2620,6 +2622,20 @@ static const struct arm64_cpu_capabilities arm64_features[] = { { /* Sentinel */ } }, }, + { + .desc = "FEAT_NV2p1", + .capability = ARM64_HAS_NV2P1, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + ARM64_CPUID_FIELDS(ID_AA64MMFR4_EL1, NV_frac, NV2P1) + }, + { + .desc = "FEAT_NV3", + .capability = ARM64_HAS_NV3, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + ARM64_CPUID_FIELDS(ID_AA64MMFR4_EL1, NV_frac, NV3) + }, { .capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE, .type = ARM64_CPUCAP_SYSTEM_FEATURE, diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index d4c7d45ae6bc..d15c2cb1b0f2 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -51,6 +51,7 @@ PI_EXPORT_SYM(id_aa64isar2_override); PI_EXPORT_SYM(id_aa64mmfr0_override); PI_EXPORT_SYM(id_aa64mmfr1_override); PI_EXPORT_SYM(id_aa64mmfr2_override); +PI_EXPORT_SYM(id_aa64mmfr4_override); PI_EXPORT_SYM(id_aa64pfr0_override); PI_EXPORT_SYM(id_aa64pfr1_override); PI_EXPORT_SYM(id_aa64smfr0_override); diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c index bc57b290e5e7..4e47616bcac2 100644 --- a/arch/arm64/kernel/pi/idreg-override.c +++ b/arch/arm64/kernel/pi/idreg-override.c @@ -106,6 +106,15 @@ static const struct ftr_set_desc mmfr2 __prel64_initconst = { }, }; +static const struct ftr_set_desc mmfr4 __prel64_initconst = { + .name = "id_aa64mmfr4", + .override = &id_aa64mmfr4_override, + .fields = { + FIELD("nv_frac", ID_AA64MMFR4_EL1_NV_frac_SHIFT, NULL), + {} + }, +}; + static bool __init pfr0_sve_filter(u64 val) { /* @@ -220,6 +229,7 @@ PREL64(const struct ftr_set_desc, reg) regs[] __prel64_initconst = { { &mmfr0 }, { &mmfr1 }, { &mmfr2 }, + { &mmfr4 }, { &pfr0 }, { &pfr1 }, { &isar1 }, diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 4155fe89b58a..db60facad9f3 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -876,8 +876,14 @@ static void timer_set_traps(struct kvm_vcpu *vcpu, struct timer_map *map) assign_clear_set_bit(tvt02, CNTHCTL_EL1NVVCT, clr, set); assign_clear_set_bit(tpt02, CNTHCTL_EL1NVPCT, clr, set); - /* This only happens on VHE, so use the CNTHCTL_EL2 accessor. */ - sysreg_clear_set(cnthctl_el2, clr, set); + /* + * This only happens on VHE, so use the CNTHCTL_EL2 accessor, unless + * we are sure CNTKCTL_EL1 is completely stateful with FEAT_NV2p1. + */ + if (!cpus_have_final_cap(ARM64_HAS_NV2P1)) + sysreg_clear_set(cnthctl_el2, clr, set); + else + sysreg_clear_set(cntkctl_el1, clr, set); } void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c index 0622162b089e..1053676551af 100644 --- a/arch/arm64/kvm/config.c +++ b/arch/arm64/kvm/config.c @@ -225,6 +225,7 @@ struct reg_feat_map_desc { #define FEAT_HCX ID_AA64MMFR1_EL1, HCX, IMP #define FEAT_S2PIE ID_AA64MMFR3_EL1, S2PIE, IMP #define FEAT_GCIE ID_AA64PFR2_EL1, GCIE, IMP +#define FEAT_NV3 ID_AA64MMFR4_EL1, NV_frac, NV3 static bool not_feat_aa64el3(struct kvm *kvm) { @@ -904,6 +905,12 @@ static const DECLARE_FEAT_MAP_FGT(hdfgwtr2_desc, hdfgwtr2_masks, static const struct reg_bits_to_feat_map hcrx_feat_map[] = { + NEEDS_FEAT(HCRX_EL2_NVTGE | + HCRX_EL2_NVnTTLB | + HCRX_EL2_NVnTTLBIS | + HCRX_EL2_NVnTTLBOS, + FEAT_NV3), + NEEDS_FEAT(HCRX_EL2_SRMASKEn, FEAT_SRMASK), NEEDS_FEAT(HCRX_EL2_PACMEn, feat_pauth_lr), NEEDS_FEAT(HCRX_EL2_EnFPM, FEAT_FPMR), NEEDS_FEAT(HCRX_EL2_GCSEn, FEAT_GCS), @@ -930,10 +937,12 @@ static const struct reg_bits_to_feat_map hcrx_feat_map[] = { NEEDS_FEAT(HCRX_EL2_EnASR, FEAT_LS64_V), NEEDS_FEAT(HCRX_EL2_EnALS, FEAT_LS64), NEEDS_FEAT(HCRX_EL2_EnAS0, FEAT_LS64_ACCDATA), + FORCE_RES0(HCRX_EL2_RES0), + FORCE_RES1(HCRX_EL2_RES1), }; -static const DECLARE_FEAT_MAP(hcrx_desc, __HCRX_EL2, +static const DECLARE_FEAT_MAP(hcrx_desc, HCRX_EL2, hcrx_feat_map, FEAT_HCX); static const struct reg_bits_to_feat_map hcr_feat_map[] = { @@ -1010,6 +1019,9 @@ static const struct reg_bits_to_feat_map hcr_feat_map[] = { static const DECLARE_FEAT_MAP(hcr_desc, HCR_EL2, hcr_feat_map, FEAT_AA64EL2); +static const DECLARE_FEAT_MAP(nvhcr_desc, NVHCR_EL2, + hcr_feat_map, FEAT_NV3); + static const struct reg_bits_to_feat_map sctlr2_feat_map[] = { NEEDS_FEAT(SCTLR2_EL1_NMEA | SCTLR2_EL1_EASE, @@ -1384,6 +1396,7 @@ void __init check_feature_map(void) check_reg_desc(&hdfgwtr2_desc); check_reg_desc(&hcrx_desc); check_reg_desc(&hcr_desc); + check_reg_desc(&nvhcr_desc); check_reg_desc(&sctlr2_desc); check_reg_desc(&tcr2_el2_desc); check_reg_desc(&sctlr_el1_desc); @@ -1579,11 +1592,21 @@ struct resx get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg) break; case HCRX_EL2: resx = compute_reg_resx_bits(kvm, &hcrx_desc, 0, 0); - resx.res1 |= __HCRX_EL2_RES1; break; case HCR_EL2: resx = compute_reg_resx_bits(kvm, &hcr_desc, 0, 0); break; + case NVHCR_EL2: + /* + * Only apply sanitisation if we do have FEAT_NV3. + * Otherwise, the register aliases with HCR_EL2 in VNCR, + * and we're better off relying on data transfers between + * NVHCR_EL2 and HCR_EL2 to sanitise things. + */ + resx = (kvm_has_nv3(kvm) ? + compute_reg_resx_bits(kvm, &nvhcr_desc, 0, 0) : + (typeof(resx)){}); + break; case SCTLR2_EL1: case SCTLR2_EL2: resx = compute_reg_resx_bits(kvm, &sctlr2_desc, 0, 0); diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 3c82f392845d..625604019fb3 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -136,6 +136,8 @@ enum cgt_group_id { CGT_CPTR_TTA, CGT_MDCR_HPMN, + CGT_HCR_NV_HCRX_nNVTGE, + /* Must be last */ __NR_CGT_GROUP_IDS__ }; @@ -588,6 +590,15 @@ static enum trap_behaviour check_mdcr_hpmn(struct kvm_vcpu *vcpu) return BEHAVE_HANDLE_LOCALLY; } +static enum trap_behaviour check_hcr_nv_hcrx_nnvtge(struct kvm_vcpu *vcpu) +{ + if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_EL2_NV) && + !(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_NVTGE)) + return BEHAVE_FORWARD_RW; + + return BEHAVE_HANDLE_LOCALLY; +} + #define CCC(id, fn) \ [id - __COMPLEX_CONDITIONS__] = fn @@ -598,6 +609,7 @@ static const complex_condition_check ccc[] = { CCC(CGT_CNTHCTL_EL1NVVCT, check_cnthctl_el1nvvct), CCC(CGT_CPTR_TTA, check_cptr_tta), CCC(CGT_MDCR_HPMN, check_mdcr_hpmn), + CCC(CGT_HCR_NV_HCRX_nNVTGE, check_hcr_nv_hcrx_nnvtge), }; /* @@ -853,6 +865,7 @@ static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = { SR_TRAP(SYS_SCTLR2_EL2, CGT_HCR_NV), SR_RANGE_TRAP(SYS_HCR_EL2, SYS_HCRX_EL2, CGT_HCR_NV), + SR_TRAP(SYS_NVHCR_EL2, CGT_HCR_NV_HCRX_nNVTGE), SR_TRAP(SYS_SMPRIMAP_EL2, CGT_HCR_NV), SR_TRAP(SYS_SMCR_EL2, CGT_HCR_NV), SR_RANGE_TRAP(SYS_TTBR0_EL2, @@ -2320,7 +2333,6 @@ int __init populate_nv_trap_config(void) BUILD_BUG_ON(__NR_CGT_GROUP_IDS__ > BIT(TC_CGT_BITS)); BUILD_BUG_ON(__NR_FGT_GROUP_IDS__ > BIT(TC_FGT_BITS)); BUILD_BUG_ON(__NR_FG_FILTER_IDS__ > BIT(TC_FGF_BITS)); - BUILD_BUG_ON(__HCRX_EL2_MASK & __HCRX_EL2_nMASK); for (int i = 0; i < ARRAY_SIZE(encoding_to_cgt); i++) { const struct encoding_to_trap_config *cgt = &encoding_to_cgt[i]; @@ -2346,10 +2358,6 @@ int __init populate_nv_trap_config(void) } } - if (__HCRX_EL2_RES0 != HCRX_EL2_RES0) - kvm_info("Sanitised HCR_EL2_RES0 = %016llx, expecting %016llx\n", - __HCRX_EL2_RES0, HCRX_EL2_RES0); - kvm_info("nv: %ld coarse grained trap handlers\n", ARRAY_SIZE(encoding_to_cgt)); diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 2aceda749641..1ce7130e2549 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -108,9 +108,10 @@ static inline void __activate_cptr_traps_vhe(struct kvm_vcpu *vcpu) * The architecture is a bit crap (what a surprise): an EL2 guest * writing to CPTR_EL2 via CPACR_EL1 can't set any of TCPAC or TTA, * as they are RES0 in the guest's view. To work around it, trap the - * sucker using the very same bit it can't set... + * sucker using the very same bit it can't set. FEAT_NV2p1 fixes it. */ - if (vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu)) + if (!cpus_have_final_cap(ARM64_HAS_NV2P1) && + vcpu_el2_e2h_is_set(vcpu) && is_hyp_ctxt(vcpu)) val |= CPTR_EL2_TCPAC; /* @@ -325,6 +326,24 @@ static inline void __deactivate_traps_mpam(void) write_sysreg_s(MPAMHCR_HOST_FLAGS, SYS_MPAMHCR_EL2); } +/* + * Just like for HCR_EL2, we can't let the guest mess with some of the + * basics we rely on in HCRX_EL2. However, the major difference is that + * HCRX_EL2 only affects EL1, and never EL2 (sudden outburst of sanity, I + * guess). So it is always the guest inflicting it on its own guestx. + * + * Things we don't want to let the guest control are: + * + * - TMEA: That's for us to decide how an SEA is routed, not the guest. + * + * - PTTWI: Similarly, it is for us to decide whether Reduced Coherency for + * the PTW is a thing. It really isn't. + * + * - EnIDCP128: We don't allow IMPDEF sysregs -- full stop. + */ +#define NV_HCRX_GUEST_EXCLUDE (HCRX_EL2_TMEA | HCRX_EL2_PTTWI | \ + HCRX_EL2_EnIDCP128) + static inline void __activate_traps_common(struct kvm_vcpu *vcpu) { struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt); @@ -350,8 +369,8 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu) u64 hcrx = vcpu->arch.hcrx_el2; if (is_nested_ctxt(vcpu)) { u64 val = __vcpu_sys_reg(vcpu, HCRX_EL2); - hcrx |= val & __HCRX_EL2_MASK; - hcrx &= ~(~val & __HCRX_EL2_nMASK); + hcrx |= (val & ~NV_HCRX_GUEST_EXCLUDE); + hcrx &= ~(~val & ~NV_HCRX_GUEST_EXCLUDE); } ctxt_sys_reg(hctxt, HCRX_EL2) = read_sysreg_s(SYS_HCRX_EL2); diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h index a17cbe7582de..c382848d3194 100644 --- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h +++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h @@ -172,6 +172,10 @@ static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt) if (ctxt_has_sctlr2(ctxt)) ctxt_sys_reg(ctxt, SCTLR2_EL1) = read_sysreg_el1(SYS_SCTLR2); + + /* Retrieve L2's HCR_EL2, and save it for future use */ + if (is_nested_nv3_ctxt(ctxt_to_vcpu(ctxt))) + ctxt_sys_reg(ctxt, NVHCR_EL2) = read_sysreg_s(SYS_NVHCR_EL2); } static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt) @@ -285,6 +289,13 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt, if (ctxt_has_sctlr2(ctxt)) write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR2_EL1), SYS_SCTLR2); + + /* + * Publish the L2 view of HCR_EL2 to the HW if L1 is using NV3. + * Otherwise, the data is already in place in the L1's own VNCR. + */ + if (is_nested_nv3_ctxt(ctxt_to_vcpu(ctxt))) + write_sysreg_s(ctxt_sys_reg(ctxt, NVHCR_EL2), SYS_NVHCR_EL2); } /* Read the VCPU state's PSTATE, but translate (v)EL2 to EL1. */ diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index bbe9cebd3d9d..4ed745bdde41 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -70,6 +70,12 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu) if (!vcpu_el2_e2h_is_set(vcpu)) hcr |= HCR_NV1; + /* Publish the guest's view of HCR_EL2 to the HW */ + if (cpus_have_final_cap(ARM64_HAS_NV3) && vcpu_el2_e2h_is_set(vcpu)) + write_sysreg_s(__vcpu_sys_reg(vcpu, HCR_EL2), SYS_NVHCR_EL2); + else + __vcpu_assign_sys_reg(vcpu, NVHCR_EL2, __vcpu_sys_reg(vcpu, HCR_EL2)); + /* * Nothing in HCR_EL2 should impact running in hypervisor * context, apart from bits we have defined as RESx (E2H, @@ -339,18 +345,24 @@ static bool kvm_hyp_handle_eret(struct kvm_vcpu *vcpu, u64 *exit_code) u64 esr = kvm_vcpu_get_esr(vcpu); u64 spsr, elr, mode; + /* With NV3, the fast path is handled in HW */ + if (cpus_have_final_cap(ARM64_HAS_NV3) && vcpu_el2_e2h_is_set(vcpu)) + return false; + /* * Going through the whole put/load motions is a waste of time * if this is a VHE guest hypervisor returning to its own * userspace, or the hypervisor performing a local exception * return. No need to save/restore registers, no need to - * switch S2 MMU. Just do the canonical ERET. + * switch S2 MMU. Just do the canonical ERET unless we are in + * nested context. * - * Unless the trap has to be forwarded further down the line, - * of course... + * Note that this is made possible because KVM itself never traps + * ERET when running an L2. The consequence is that any ERET trap is + * the result of HCR_EL2 or HFGITR_EL2 programming by L1 for its own + * guest, and the exception must be forwarded to L1. */ - if ((__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_NV) || - (__vcpu_sys_reg(vcpu, HFGITR_EL2) & HFGITR_EL2_ERET)) + if (is_nested_ctxt(vcpu)) return false; spsr = read_sysreg_el1(SYS_SPSR); @@ -441,6 +453,9 @@ static bool kvm_hyp_handle_cpacr_el1(struct kvm_vcpu *vcpu, u64 *exit_code) u64 esr = kvm_vcpu_get_esr(vcpu); int rt; + if (cpus_have_final_cap(ARM64_HAS_NV2P1)) + return false; + if (!is_hyp_ctxt(vcpu) || esr_sys64_to_sysreg(esr) != SYS_CPACR_EL1) return false; @@ -534,19 +549,17 @@ static const exit_handler_fn hyp_exit_handlers[] = { [0x3F] = kvm_hyp_handle_impdef, }; -static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) +static void fixup_nv_guest_exit(struct kvm_vcpu *vcpu) { - synchronize_vcpu_pstate(vcpu); - /* * If we were in HYP context on entry, adjust the PSTATE view * so that the usual helpers work correctly. This enforces our * invariant that the guest's HYP context status is preserved * across a run. */ - if (vcpu_has_nv(vcpu) && - unlikely(host_data_test_flag(VCPU_IN_HYP_CONTEXT))) { + if (unlikely(host_data_test_flag(VCPU_IN_HYP_CONTEXT))) { u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); + u64 hcr; switch (mode) { case PSR_MODE_EL1t: @@ -559,11 +572,26 @@ static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) *vcpu_cpsr(vcpu) &= ~(PSR_MODE_MASK | PSR_MODE32_BIT); *vcpu_cpsr(vcpu) |= mode; + + /* Publish the latest HCR_EL2 to the emulation */ + hcr = (cpus_have_final_cap(ARM64_HAS_NV3) && + vcpu_el2_e2h_is_set(vcpu)) ? + read_sysreg_s(SYS_NVHCR_EL2) : + __vcpu_sys_reg(vcpu, NVHCR_EL2); + + __vcpu_assign_sys_reg(vcpu, HCR_EL2, hcr); } /* Apply extreme paranoia! */ - BUG_ON(vcpu_has_nv(vcpu) && - !!host_data_test_flag(VCPU_IN_HYP_CONTEXT) != is_hyp_ctxt(vcpu)); + BUG_ON(!!host_data_test_flag(VCPU_IN_HYP_CONTEXT) != is_hyp_ctxt(vcpu)); +} + +static bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + synchronize_vcpu_pstate(vcpu); + + if (vcpu_has_nv(vcpu)) + fixup_nv_guest_exit(vcpu); return __fixup_guest_exit(vcpu, exit_code, hyp_exit_handlers); } diff --git a/arch/arm64/kvm/hyp/vhe/sysreg-sr.c b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c index be685b63e8cf..0c4ef1ce32ae 100644 --- a/arch/arm64/kvm/hyp/vhe/sysreg-sr.c +++ b/arch/arm64/kvm/hyp/vhe/sysreg-sr.c @@ -42,10 +42,12 @@ static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu) u64 val; /* - * We don't save CPTR_EL2, as accesses to CPACR_EL1 - * are always trapped, ensuring that the in-memory - * copy is always up-to-date. A small blessing... + * Without FEAT_NV2p1, we don't save CPTR_EL2, as accesses + * to CPACR_EL1 are always trapped, ensuring that the + * in-memory copy is always up-to-date. A small blessing... */ + if (cpus_have_final_cap(ARM64_HAS_NV2P1)) + __vcpu_assign_sys_reg(vcpu, CPTR_EL2, read_sysreg_el1(SYS_CPACR)); __vcpu_assign_sys_reg(vcpu, SCTLR_EL2, read_sysreg_el1(SYS_SCTLR)); __vcpu_assign_sys_reg(vcpu, TTBR0_EL2, read_sysreg_el1(SYS_TTBR0)); __vcpu_assign_sys_reg(vcpu, TTBR1_EL2, read_sysreg_el1(SYS_TTBR1)); @@ -67,11 +69,18 @@ static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu) * The EL1 view of CNTKCTL_EL1 has a bunch of RES0 bits where * the interesting CNTHCTL_EL2 bits live. So preserve these * bits when reading back the guest-visible value. + * + * While NV2p1 fixes some of that, it makes CNTHCTL_EL2.ECV + * even more broken than it already was with NV2. */ val = read_sysreg_el1(SYS_CNTKCTL); - val &= CNTKCTL_VALID_BITS; - __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, &=, ~CNTKCTL_VALID_BITS); - __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, |=, val); + if (!cpus_have_final_cap(ARM64_HAS_NV2P1)) { + val &= CNTKCTL_VALID_BITS; + __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, &=, ~CNTKCTL_VALID_BITS); + __vcpu_rmw_sys_reg(vcpu, CNTHCTL_EL2, |=, val); + } else { + __vcpu_assign_sys_reg(vcpu, CNTHCTL_EL2, val); + } } __vcpu_assign_sys_reg(vcpu, SP_EL2, read_sysreg(sp_el1)); diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index dfb96edbdc43..70d605e60cfb 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1728,7 +1728,7 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) * You get EITHER * * - FEAT_VHE without FEAT_E2H0 - * - FEAT_NV limited to FEAT_NV2 + * - FEAT_NV limited to FEAT_NV2(p1)/NV3 * - HCR_EL2.NV1 being RES0 * * OR @@ -1740,7 +1740,13 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val) if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features)) { val = 0; } else { - val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY); + val &= ID_AA64MMFR4_EL1_NV_frac; + if (cpus_have_final_cap(ARM64_HAS_NV3)) + val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV3); + else if (cpus_have_final_cap(ARM64_HAS_NV2P1)) + val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64MMFR4_EL1, NV_frac, NV2P1); + else + val = SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY); val |= SYS_FIELD_PREP_ENUM(ID_AA64MMFR4_EL1, E2H0, NI_NV1); } break; @@ -1826,6 +1832,10 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) resx = get_reg_fixed_bits(kvm, HCR_EL2); set_sysreg_masks(kvm, HCR_EL2, resx); + /* NVHCR_EL2 */ + resx = get_reg_fixed_bits(kvm, NVHCR_EL2); + set_sysreg_masks(kvm, NVHCR_EL2, resx); + /* HCRX_EL2 */ resx = get_reg_fixed_bits(kvm, HCRX_EL2); set_sysreg_masks(kvm, HCRX_EL2, resx); diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index b844fde90fbc..400c9894f348 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -183,8 +183,6 @@ static void locate_register(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg, switch (reg) { MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1, translate_sctlr_el2_to_sctlr_el1 ); - MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1, - translate_cptr_el2_to_cpacr_el1 ); MAPPED_EL2_SYSREG(TTBR0_EL2, TTBR0_EL1, translate_ttbr0_el2_to_ttbr0_el1 ); MAPPED_EL2_SYSREG(TTBR1_EL2, TTBR1_EL1, NULL ); @@ -210,6 +208,33 @@ static void locate_register(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg, loc->loc = ((is_hyp_ctxt(vcpu) && vcpu_el2_e2h_is_set(vcpu)) ? SR_LOC_SPECIAL : SR_LOC_MEMORY); break; + case CPTR_EL2: + /* + * CPTR_EL2 is just as special, and needs a certain amount + * of handholding. It always lives in memory, due to being + * heavily trapped thanks to CPACR_EL1.TCPAC being RES0. + * FEAT_NV2p1 fixes this. + */ + locate_mapped_el2_register(vcpu, CPTR_EL2, CPACR_EL1, + translate_cptr_el2_to_cpacr_el1, + loc); + if (is_hyp_ctxt(vcpu) && vcpu_el2_e2h_is_set(vcpu)) + loc->loc = SR_LOC_SPECIAL; + break; + case NVHCR_EL2: + /* + * Yes, NVHCR_EL2 maps to itself when loaded in nested + * context. If you feel like the architecture is double + * backing on itself upside down, you're not alone. + */ + WARN_ON_ONCE(!kvm_has_nv3(vcpu->kvm)); + if (is_hyp_ctxt(vcpu)) { + loc->loc = SR_LOC_MEMORY; + } else { + loc->loc = SR_LOC_LOADED | SR_LOC_MAPPED; + loc->map_reg = NVHCR_EL2; + } + break; default: loc->loc = locate_direct_register(vcpu, reg); } @@ -249,6 +274,7 @@ static u64 read_sr_from_cpu(enum vcpu_sysreg reg) case DACR32_EL2: val = read_sysreg_s(SYS_DACR32_EL2); break; case IFSR32_EL2: val = read_sysreg_s(SYS_IFSR32_EL2); break; case DBGVCR32_EL2: val = read_sysreg_s(SYS_DBGVCR32_EL2); break; + case NVHCR_EL2: val = read_sysreg_s(SYS_NVHCR_EL2); break; default: WARN_ON_ONCE(1); } @@ -287,6 +313,7 @@ static void write_sr_to_cpu(enum vcpu_sysreg reg, u64 val) case DACR32_EL2: write_sysreg_s(val, SYS_DACR32_EL2); break; case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break; case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); break; + case NVHCR_EL2: write_sysreg_s(val, SYS_NVHCR_EL2); break; default: WARN_ON_ONCE(1); } } @@ -311,9 +338,16 @@ u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg) switch (reg) { case CNTHCTL_EL2: val = read_sysreg_el1(SYS_CNTKCTL); - val &= CNTKCTL_VALID_BITS; - val |= __vcpu_sys_reg(vcpu, reg) & ~CNTKCTL_VALID_BITS; + if (!cpus_have_final_cap(ARM64_HAS_NV2P1)) { + val &= CNTKCTL_VALID_BITS; + val |= __vcpu_sys_reg(vcpu, reg) & ~CNTKCTL_VALID_BITS; + } return val; + case CPTR_EL2: + if (cpus_have_final_cap(ARM64_HAS_NV2P1)) + return read_sysreg_el1(SYS_CPACR); + else + return __vcpu_sys_reg(vcpu, reg); default: WARN_ON_ONCE(1); } @@ -359,6 +393,9 @@ void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, enum vcpu_sysreg reg) */ write_sysreg_el1(val, SYS_CNTKCTL); break; + case CPTR_EL2: + write_sysreg_el1(val, SYS_CPACR); + break; default: WARN_ON_ONCE(1); } @@ -2887,6 +2924,16 @@ static unsigned int vncr_el2_visibility(const struct kvm_vcpu *vcpu, return REG_HIDDEN; } +static unsigned int nvhcr_el2_visibility(const struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd) +{ + if (el2_visibility(vcpu, rd) == 0 && + kvm_has_feat(vcpu->kvm, ID_AA64MMFR4_EL1, NV_frac, NV3)) + return 0; + + return REG_HIDDEN; +} + static unsigned int sctlr2_visibility(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd) { @@ -3801,6 +3848,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { sve_el2_visibility), EL2_REG_VNCR(HCRX_EL2, reset_val, 0), + EL2_REG_FILTERED(NVHCR_EL2, undef_access, reset_val, 0, + nvhcr_el2_visibility), EL2_REG(TTBR0_EL2, access_rw, reset_val, 0), EL2_REG(TTBR1_EL2, access_rw, reset_val, 0), diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 9b85a84f6fd4..7e0414509e89 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -51,6 +51,8 @@ HAS_LS64_V HAS_LSUI HAS_MOPS HAS_NESTED_VIRT +HAS_NV2P1 +HAS_NV3 HAS_BBML2_NOABORT HAS_PAN HAS_PMUV3 diff --git a/arch/arm64/tools/gen-sysreg.awk b/arch/arm64/tools/gen-sysreg.awk index 86860ab672dc..53844f5c5e52 100755 --- a/arch/arm64/tools/gen-sysreg.awk +++ b/arch/arm64/tools/gen-sysreg.awk @@ -228,7 +228,7 @@ $1 == "EndSysreg" && block_current() == "Sysreg" { } # Currently this is effectivey a comment, in future we may want to emit -# defines for the fields. +# defines for the fields. We do emit RESx and UNKN values in any case. ($1 == "Fields" || $1 == "Mapping") && block_current() == "Sysreg" { expect_fields(2) @@ -239,9 +239,9 @@ $1 == "EndSysreg" && block_current() == "Sysreg" { print "" next_bit = -1 - res0 = null - res1 = null - unkn = null + res0 = $2 "_RES0" + res1 = $2 "_RES1" + unkn = $2 "_UNKN" next } diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 7cb61aca3797..67d013b376a3 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2386,17 +2386,40 @@ EndEnum EndSysreg Sysreg ID_AA64MMFR4_EL1 3 0 0 7 4 -Res0 63:48 -UnsignedEnum 47:44 SRMASK +UnsignedEnum 63:60 MTEFGT + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 59:56 SCRX + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 55:52 TEV + 0b0000 NI + 0b0001 IMP +EndEnum +UnsignedEnum 51:48 TPS + 0b0000 VAL_0000 + 0b0001 VAL_0001 + 0b0010 VAL_0010 +EndEnum +UnsignedEnum 47:44 SRMASK + 0b0000 NI + 0b0001 IMP + 0b0010 SRMASK2 +EndEnum +UnsignedEnum 43:40 TLBID 0b0000 NI 0b0001 IMP EndEnum -Res0 43:40 UnsignedEnum 39:36 E3DSE 0b0000 NI 0b0001 IMP EndEnum -Res0 35:32 +UnsignedEnum 35:32 EAESR + 0b0000 NI + 0b0001 IMP +EndEnum UnsignedEnum 31:28 RMEGDI 0b0000 NI 0b0001 IMP @@ -2410,6 +2433,7 @@ UnsignedEnum 23:20 NV_frac 0b0000 NV_NV2 0b0001 NV2_ONLY 0b0010 NV2P1 + 0b0011 NV3 EndEnum UnsignedEnum 19:16 FGWTE3 0b0000 NI @@ -4242,6 +4266,9 @@ Field 1 E2TRE Field 0 E0HTRE EndSysreg +Sysreg NVHCR_EL2 3 4 1 5 0 +Mapping HCR_EL2 +EndSysreg Sysreg HDFGRTR2_EL2 3 4 3 1 0 Res0 63:25 @@ -4521,7 +4548,14 @@ Fields ZCR_ELx EndSysreg Sysreg HCRX_EL2 3 4 1 2 2 -Res0 63:25 +Res0 63:35 +Field 34 NVnTTLBOS +Field 33 NVnTTLBIS +Field 32 NVnTTLB +Res0 31:28 +Field 27 NVTGE +Field 26 SRMASKEn +Res0 25 Field 24 PACMEn Field 23 EnFPM Field 22 GCSEn diff --git a/tools/testing/selftests/kvm/arm64/get-reg-list.c b/tools/testing/selftests/kvm/arm64/get-reg-list.c index cfa99979d57c..533994687b5d 100644 --- a/tools/testing/selftests/kvm/arm64/get-reg-list.c +++ b/tools/testing/selftests/kvm/arm64/get-reg-list.c @@ -67,6 +67,7 @@ static struct feature_id_reg feat_id_regs[] = { REG_FEAT(VDISR_EL2, ID_AA64PFR0_EL1, RAS, IMP), REG_FEAT(VSESR_EL2, ID_AA64PFR0_EL1, RAS, IMP), REG_FEAT(VNCR_EL2, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY), + REG_FEAT(NVHCR_EL2, ID_AA64MMFR4_EL1, NV_frac, NV3), REG_FEAT(CNTHV_CTL_EL2, ID_AA64MMFR1_EL1, VH, IMP), REG_FEAT(CNTHV_CVAL_EL2,ID_AA64MMFR1_EL1, VH, IMP), REG_FEAT(ZCR_EL2, ID_AA64PFR0_EL1, SVE, IMP), @@ -771,6 +772,7 @@ static __u64 el2_regs[] = { SYS_REG(SP_EL2), SYS_REG(VDISR_EL2), SYS_REG(VSESR_EL2), + SYS_REG(NVHCR_EL2), }; static __u64 el2_e2h0_regs[] = {