diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index c686fee9767a..e5b92ac09e69 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -215,7 +215,6 @@ struct kvm_nvhe_init_params { unsigned long hcr_el2; unsigned long vttbr; unsigned long vtcr; - unsigned long tmp; }; /* diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index b6367ff3a49c..9c853ed3ceab 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -124,7 +124,6 @@ int main(void) DEFINE(NVHE_INIT_HCR_EL2, offsetof(struct kvm_nvhe_init_params, hcr_el2)); DEFINE(NVHE_INIT_VTTBR, offsetof(struct kvm_nvhe_init_params, vttbr)); DEFINE(NVHE_INIT_VTCR, offsetof(struct kvm_nvhe_init_params, vtcr)); - DEFINE(NVHE_INIT_TMP, offsetof(struct kvm_nvhe_init_params, tmp)); #endif #ifdef CONFIG_CPU_PM DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index db60facad9f3..6ac3321f4c57 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -1535,13 +1535,14 @@ static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu) ctx = vcpu_get_timer(vcpu, i); irq = timer_irq(ctx); - if (kvm_vgic_set_owner(vcpu, irq, ctx)) - break; /* With GICv5, the default PPI is what you get -- nothing else */ if (vgic_is_v5(vcpu->kvm) && irq != get_vgic_ppi(vcpu->kvm, default_ppi[i])) break; + if (kvm_vgic_set_owner(vcpu, irq, ctx)) + break; + /* * We know by construction that we only have PPIs, so all values * are less than 32 for non-GICv5 VGICs. On GICv5, they are diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 29108e5c0206..db37678dcb05 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -530,10 +530,20 @@ void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index) kvm_handle_guest_serror(vcpu, kvm_vcpu_get_esr(vcpu)); } +static bool nvhe_hyp_panic_host_s2_disabled(void) +{ + return !is_protected_kvm_enabled() || + IS_ENABLED(CONFIG_PKVM_DISABLE_STAGE2_ON_PANIC); +} + static void print_nvhe_hyp_panic(const char *name, u64 panic_addr) { - kvm_err("nVHE hyp %s at: [<%016llx>] %pB!\n", name, panic_addr, - (void *)(panic_addr + kaslr_offset())); + /* Kallsyms might not be mapped in the host stage-2 */ + if (nvhe_hyp_panic_host_s2_disabled()) + kvm_err("nVHE hyp %s at: [<%016llx>] %pB!\n", name, panic_addr, + (void *)(panic_addr + kaslr_offset())); + else + kvm_err("nVHE hyp %s at: %016llx!\n", name, panic_addr); } static void kvm_nvhe_report_cfi_failure(u64 panic_addr) @@ -561,8 +571,7 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr, unsigned int line = 0; /* All hyp bugs, including warnings, are treated as fatal. */ - if (!is_protected_kvm_enabled() || - IS_ENABLED(CONFIG_PKVM_DISABLE_STAGE2_ON_PANIC)) { + if (nvhe_hyp_panic_host_s2_disabled()) { struct bug_entry *bug = find_bug(elr_in_kimg); if (bug) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 89cb553be1e5..0b3e0b28dfc7 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -106,16 +106,19 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init) and x2, x1, x2 cbz x2, 1f - // hVHE: Replay the EL2 setup to account for the E2H bit - // TPIDR_EL2 is used to preserve x0 across the macro maze... + /* + * hVHE: Replay the EL2 setup to account for the E2H bit + * TPIDR_EL2 and FAR_EL2 are used to preserve x0 and LR across + * the macro maze... + */ isb msr tpidr_el2, x0 - str lr, [x0, #NVHE_INIT_TMP] + msr far_el2, lr bl __kvm_init_el2_state + mrs lr, far_el2 mrs x0, tpidr_el2 - ldr lr, [x0, #NVHE_INIT_TMP] 1: ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 75a84761e69c..44aae52c473d 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -4969,10 +4969,8 @@ static int kvm_handle_cp_64(struct kvm_vcpu *vcpu, * Make a 64-bit value out of Rt and Rt2. As we use the same trap * backends between AArch32 and AArch64, we get away with it. */ - if (params.is_write) { - params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff; - params.regval |= vcpu_get_reg(vcpu, Rt2) << 32; - } + params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff; + params.regval |= vcpu_get_reg(vcpu, Rt2) << 32; /* * If the table contains a handler, handle the diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 907057881b26..4012df6002ea 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -176,6 +176,7 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) } kvm->arch.vgic.vgic_model = 0; + kvm->arch.vgic.in_kernel = false; goto out_unlock; } @@ -210,6 +211,9 @@ static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis) struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); int i; + if (dist->spis) + return 0; + dist->active_spis = (atomic_t)ATOMIC_INIT(0); dist->spis = kzalloc_objs(struct vgic_irq, nr_spis, GFP_KERNEL_ACCOUNT); if (!dist->spis) @@ -787,7 +791,8 @@ int kvm_vgic_hyp_init(void) if (has_mask && !gic_kvm_info->maint_irq) { kvm_err("No vgic maintenance irq\n"); - return -ENXIO; + ret = -ENXIO; + goto out_free; } /* @@ -820,6 +825,7 @@ int kvm_vgic_hyp_init(void) kvm_vgic_global_state.maint_irq = gic_kvm_info->maint_irq; +out_free: kfree(gic_kvm_info); gic_kvm_info = NULL; diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 740b39875728..f6538b1976f9 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -2026,15 +2026,16 @@ static u32 compute_next_devid_offset(struct list_head *h, static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite) { - struct its_ite *next; - u32 next_offset; + struct its_ite *next = ite; - if (list_is_last(&ite->ite_list, h)) - return 0; - next = list_next_entry(ite, ite_list); - next_offset = next->event_id - ite->event_id; + /* Point at the next ITE that vgic_its_save_ite() stores as valid. */ + list_for_each_entry_continue(next, h, ite_list) { + if (next->collection) + return min_t(u32, next->event_id - ite->event_id, + VITS_ITE_MAX_EVENTID_OFFSET); + } - return min_t(u32, next_offset, VITS_ITE_MAX_EVENTID_OFFSET); + return 0; } /** @@ -2110,6 +2111,14 @@ static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev, u32 next_offset; u64 val; + /* + * MAPC with V=0 keeps the ITEs mapped but drops their collection, + * and with it the ICID. Save a zeroed entry, which the restore path + * reads back as invalid. + */ + if (!ite->collection) + return vgic_its_write_entry_lock(its, gpa, 0ULL, ite); + next_offset = compute_next_eventid_offset(&dev->itt_head, ite); val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) | ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) | @@ -2523,6 +2532,9 @@ static int vgic_its_save_collection_table(struct vgic_its *its) max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K; list_for_each_entry(collection, &its->collection_list, coll_list) { + if (!vgic_its_check_id(its, baser, collection->collection_id, NULL)) + return -EINVAL; + ret = vgic_its_save_cte(its, collection, gpa); if (ret) return ret; diff --git a/arch/arm64/kvm/vgic/vgic-v2.c b/arch/arm64/kvm/vgic/vgic-v2.c index cafa3cb32bda..7182f63fc938 100644 --- a/arch/arm64/kvm/vgic/vgic-v2.c +++ b/arch/arm64/kvm/vgic/vgic-v2.c @@ -170,8 +170,9 @@ void vgic_v2_deactivate(struct kvm_vcpu *vcpu, u32 val) /* Make sure we're in the same context as LR handling */ local_irq_save(flags); + /* Guest-supplied INTID: out of range yields no irq, so ignore it */ irq = vgic_get_vcpu_irq(vcpu, val); - if (WARN_ON_ONCE(!irq)) + if (!irq) goto out; /* See the corresponding v3 code for the rationale */ diff --git a/arch/arm64/kvm/vgic/vgic-v3-nested.c b/arch/arm64/kvm/vgic/vgic-v3-nested.c index 5c69fa615823..e3e84d7f5ad5 100644 --- a/arch/arm64/kvm/vgic/vgic-v3-nested.c +++ b/arch/arm64/kvm/vgic/vgic-v3-nested.c @@ -152,7 +152,7 @@ static void vgic_compute_mi_state(struct kvm_vcpu *vcpu, struct mi_state *mi_sta eisr |= BIT(i); if (!(lr & ICH_LR_STATE)) elrsr |= BIT(i); - pend |= (lr & ICH_LR_PENDING_BIT); + pend |= (lr & ICH_LR_STATE) == ICH_LR_PENDING_BIT; } mi_state->eisr = eisr; diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 91514d6faf74..963b5044490b 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -611,9 +611,13 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) bool is_pending; bool stored; + irq = vgic_get_irq(kvm, index); + if (!irq) + continue; + vcpu = irq->target_vcpu; if (!vcpu) - continue; + goto put_irq; pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser); @@ -624,7 +628,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) if (ptr != last_ptr) { ret = kvm_read_guest_lock(kvm, ptr, &val, 1); if (ret) - goto out; + goto put_irq; last_ptr = ptr; } @@ -636,7 +640,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) vgic_v4_get_vlpi_state(irq, &is_pending); if (stored == is_pending) - continue; + goto put_irq; if (is_pending) val |= 1 << bit_nr; @@ -644,6 +648,8 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) val &= ~(1 << bit_nr); ret = vgic_write_guest_lock(kvm, ptr, &val, 1); +put_irq: + vgic_put_irq(kvm, irq); if (ret) goto out; } diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index ccb7e3a90cd0..faae11e713e3 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -93,8 +93,9 @@ struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid) /* SPIs */ if (intid >= VGIC_NR_PRIVATE_IRQS && intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) { - intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS); - return &kvm->arch.vgic.spis[intid - VGIC_NR_PRIVATE_IRQS]; + intid -= VGIC_NR_PRIVATE_IRQS; + intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis); + return &kvm->arch.vgic.spis[intid]; } /* LPIs */ @@ -117,6 +118,8 @@ struct vgic_irq *vgic_get_vcpu_irq(struct kvm_vcpu *vcpu, u32 intid) switch (type) { case KVM_DEV_TYPE_ARM_VGIC_V5: intid = vgic_v5_get_hwirq_id(intid); + if (intid >= VGIC_V5_NR_PRIVATE_IRQS) + return NULL; intid = array_index_nospec(intid, VGIC_V5_NR_PRIVATE_IRQS); break; default: diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index bd1bb03500b3..40a12bfc98c9 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -65,6 +65,8 @@ switch (t) { \ case KVM_DEV_TYPE_ARM_VGIC_V5: \ __ret = is_v5_type(GICV5_HWIRQ_TYPE_PPI, (i)); \ + __ret &= FIELD_GET(GICV5_HWIRQ_ID, (i)) < \ + VGIC_V5_NR_PRIVATE_IRQS; \ break; \ default: \ __ret = (i) >= VGIC_NR_SGIS; \