diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 90631e45d6d3..3ec414442578 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -386,6 +386,8 @@ struct kvm_arch { #define KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS 10 /* Unhandled SEAs are taken to userspace */ #define KVM_ARCH_FLAG_EXIT_SEA 11 + /* No further ID register changes possible */ +#define KVM_ARCH_FLAG_ID_REGS_FINAL 12 unsigned long flags; /* VM-wide vCPU feature set */ @@ -1168,6 +1170,9 @@ struct kvm_vcpu_arch { #define vcpu_has_ptrauth(vcpu) false #endif +#define kvm_id_regs_final(kvm) \ + test_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &(kvm)->arch.flags) + #define vcpu_on_unsupported_cpu(vcpu) \ vcpu_get_flag(vcpu, ON_UNSUPPORTED_CPU) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 03959ced2b1d..d319abbb9be3 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -954,7 +954,7 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) return ret; } - ret = kvm_finalize_sys_regs(vcpu); + ret = kvm_vcpu_finalize_sys_regs(vcpu); if (ret) return ret; diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index bb76aad0660b..e9cf6cc34034 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2508,9 +2508,10 @@ static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, /* * Once the VM has started the ID registers are immutable. Reject any - * write that does not match the final register value. + * write that does not match the final register value once we have + * got far enough into first running the VM to use the values. */ - if (kvm_vm_has_ran_once(vcpu->kvm)) { + if (kvm_id_regs_final(vcpu->kvm)) { if (val != read_id_reg(vcpu, rd)) ret = -EBUSY; else @@ -2544,7 +2545,7 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val) lockdep_assert_held(&kvm->arch.config_lock); - if (KVM_BUG_ON(kvm_vm_has_ran_once(kvm) || !p, kvm)) + if (KVM_BUG_ON(kvm_id_regs_final(kvm) || !p, kvm)) return; *p = val; @@ -3240,10 +3241,10 @@ static int set_imp_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, return -EINVAL; /* - * Once the VM has started the ID registers are immutable. Reject the - * write if userspace tries to change it. + * Once we have been far enough into starting the VM the ID registers + * are immutable. Reject the write if userspace tries to change it. */ - if (kvm_vm_has_ran_once(kvm)) + if (kvm_id_regs_final(kvm)) return -EBUSY; /* @@ -5858,26 +5859,17 @@ void kvm_calculate_traps(struct kvm_vcpu *vcpu) } /* - * Perform last adjustments to the ID registers that are implied by the + * Do system register finalization that is shared by the whole guest. This + * includes last adjustments to the ID registers that are implied by the * configuration outside of the ID regs themselves, as well as any * initialisation that directly depend on these ID registers (such as * RES0/RES1 behaviours). This is not the place to configure traps though. - * - * Because this can be called once per CPU, changes must be idempotent. */ -int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu) +static int kvm_vm_finalize_sys_regs(struct kvm *kvm) { - struct kvm *kvm = vcpu->kvm; + lockdep_assert_held(&kvm->arch.config_lock); - guard(mutex)(&kvm->arch.config_lock); - - if (vcpu_has_nv(vcpu)) { - int ret = kvm_init_nv_sysregs(vcpu); - if (ret) - return ret; - } - - if (kvm_vm_has_ran_once(kvm)) + if (kvm_id_regs_final(kvm)) return 0; /* @@ -5925,6 +5917,31 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu) kvm_vgic_finalize_idregs(kvm); } + set_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &kvm->arch.flags); + + return 0; +} + +/* + * Because this can be called once per CPU, changes must be idempotent. + */ +int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + int ret; + + guard(mutex)(&kvm->arch.config_lock); + + ret = kvm_vm_finalize_sys_regs(kvm); + if (ret) + return ret; + + if (vcpu_has_nv(vcpu)) { + ret = kvm_init_nv_sysregs(vcpu); + if (ret) + return ret; + } + return 0; } diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h index bd56a45abbf9..a3cccad2766f 100644 --- a/arch/arm64/kvm/sys_regs.h +++ b/arch/arm64/kvm/sys_regs.h @@ -254,7 +254,7 @@ int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg, bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index); -int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu); +int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu); #define AA32(_x) .aarch32_map = AA32_##_x #define Op0(_x) .Op0 = _x diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 1537f4d20318..8271729130c6 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -123,10 +123,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) goto out_unlock; } - kvm_for_each_vcpu(i, vcpu, kvm) { - if (vcpu_has_run_once(vcpu)) - goto out_unlock; - } + if (kvm_id_regs_final(kvm)) + goto out_unlock; ret = 0; switch (type) { diff --git a/tools/testing/selftests/kvm/arm64/set_id_regs.c b/tools/testing/selftests/kvm/arm64/set_id_regs.c index 7429a1055df5..10849d21c0dd 100644 --- a/tools/testing/selftests/kvm/arm64/set_id_regs.c +++ b/tools/testing/selftests/kvm/arm64/set_id_regs.c @@ -13,6 +13,7 @@ #include "kvm_util.h" #include "processor.h" #include "test_util.h" +#include "vgic.h" #include enum ftr_type { @@ -803,6 +804,107 @@ static void test_reset_preserves_id_regs(struct kvm_vcpu *vcpu) ksft_test_result_pass("%s\n", __func__); } +/* + * ID registers must stay immutable even when a vCPU's first KVM_RUN fails + * after finalization but before KVM_ARCH_FLAG_HAS_RAN_ONCE is set. + */ +static void test_idreg_frozen_after_failed_run(void) +{ + static const u32 imp_id_regs[] = { + SYS_MIDR_EL1, + SYS_REVIDR_EL1, + SYS_AIDR_EL1, + }; + struct kvm_vcpu_init init; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + int r; + + if (!kvm_has_cap(KVM_CAP_ARM_PMU_V3)) { + ksft_print_msg("PMUv3 unsupported, cannot fail the first run\n"); + ksft_test_result_skip("%s\n", __func__); + return; + } + + /* Skip the default vGIC so the KVM_CREATE_DEVICE gate is reachable. */ + test_disable_default_vgic(); + + vm = vm_create(1); + vm_enable_cap(vm, KVM_CAP_ARM_WRITABLE_IMP_ID_REGS, 0); + kvm_get_default_vcpu_target(vm, &init); + init.features[0] |= (1 << KVM_ARM_VCPU_PMU_V3); + vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); + kvm_arch_vm_finalize_vcpus(vm); + + /* + * A PMUv3 vCPU left without PMU init is rejected by + * kvm_arm_pmu_v3_enable(), which runs after sysreg finalization. + */ + r = _vcpu_run(vcpu); + TEST_ASSERT(r < 0 && errno == EINVAL, + "first KVM_RUN should fail post-finalization: r=%d errno=%d", + r, errno); + + /* + * Feature ID registers: use values that would have been accepted before + * finalization, so that a rejection means the registers are final + * rather than the value being invalid. + */ + for (int i = 0; i < ARRAY_SIZE(test_regs); i++) { + const struct reg_ftr_bits *ftr_bits = test_regs[i].ftr_bits; + u64 reg = KVM_ARM64_SYS_REG(test_regs[i].reg); + u64 val = vcpu_get_reg(vcpu, reg); + + for (int j = 0; ftr_bits[j].type != FTR_END; j++) { + u64 ftr = (val & ftr_bits[j].mask) >> ftr_bits[j].shift; + u64 safe = get_safe_value(&ftr_bits[j], ftr); + u64 new_val; + + if (safe == ftr) + continue; + + new_val = (val & ~ftr_bits[j].mask) | + (safe << ftr_bits[j].shift); + + r = __vcpu_set_reg(vcpu, reg, new_val); + TEST_ASSERT(r < 0 && errno == EBUSY, + "%s write after failed first run: r=%d errno=%d", + ftr_bits[j].name, r, errno); + TEST_ASSERT_EQ(vcpu_get_reg(vcpu, reg), val); + } + + /* A write matching the finalized value is still accepted. */ + vcpu_set_reg(vcpu, reg, val); + } + + /* + * The VM-wide implementation ID registers are gated separately. Bit 0 + * is within the writable mask of all three, so flipping it is a change + * KVM would otherwise accept. + */ + for (int i = 0; i < ARRAY_SIZE(imp_id_regs); i++) { + u64 reg = KVM_ARM64_SYS_REG(imp_id_regs[i]); + u64 val = vcpu_get_reg(vcpu, reg); + + r = __vcpu_set_reg(vcpu, reg, val ^ 1); + TEST_ASSERT(r < 0 && errno == EBUSY, + "implementation ID reg write after failed first run: r=%d errno=%d", + r, errno); + TEST_ASSERT_EQ(vcpu_get_reg(vcpu, reg), val); + } + + /* Creating an in-kernel irqchip would change the ID registers too. */ + if (kvm_supports_vgic_v3()) { + r = __kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V3); + TEST_ASSERT(r < 0 && errno == EBUSY, + "vGIC creation after failed first run: r=%d errno=%d", + r, errno); + } + + kvm_vm_free(vm); + ksft_test_result_pass("%s\n", __func__); +} + int main(void) { struct kvm_vcpu *vcpu; @@ -828,7 +930,7 @@ int main(void) ksft_print_header(); - test_cnt = 3 + MPAM_IDREG_TEST + MTE_IDREG_TEST; + test_cnt = 4 + MPAM_IDREG_TEST + MTE_IDREG_TEST; for (i = 0; i < ARRAY_SIZE(test_regs); i++) for (j = 0; test_regs[i].ftr_bits[j].type != FTR_END; j++) test_cnt++; @@ -847,5 +949,7 @@ int main(void) kvm_vm_free(vm); + test_idreg_frozen_after_failed_run(); + ksft_finished(); }