diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c index f088006f3967..c028c4fb95e0 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.c +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -73,6 +73,21 @@ static DEFINE_XARRAY(vm_info); #define GICV5_VMTEL2_LPI_SECTION 2 #define GICV5_VMTEL2_SPI_SECTION 3 +static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, + unsigned int id_bits, + unsigned int istsz); +static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, + unsigned int id_bits, + unsigned int istsz, + unsigned int l2_split); +static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_spi_ist_free(struct kvm *kvm); + /* * Our IRS might be coherent or non-coherent. If coherent, we can just emit a * DSB to ensure that we're in sync. However, when non-coherent, we need to @@ -561,25 +576,6 @@ int vgic_v5_vmte_init(struct kvm *kvm) return ret; } -/* - * The following set of forward declarations makes the code layout a *little* - * clearer as it lets us keep the IST-related code together. - */ -static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, - unsigned int id_bits, - unsigned int istsz); -static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, - unsigned int istsz, unsigned int l2_split); -static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, - unsigned int istsz, unsigned int l2_split); -static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, - unsigned int id_bits, - unsigned int istsz, - unsigned int l2_split); -static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi); -static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi); -static int vgic_v5_spi_ist_free(struct kvm *kvm); - /* * Release the VMT Entry, freeing up any allocated data structures before * zeroing the VMTE. @@ -723,6 +719,18 @@ int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu) return 0; } +phys_addr_t vgic_v5_get_vmt_base(void) +{ + phys_addr_t vmt_base; + + if (!vmt_info->two_level) + vmt_base = virt_to_phys(vmt_info->linear.vmt_base); + else + vmt_base = virt_to_phys(vmt_info->l2.vmt_base); + + return vmt_base; +} + /* * Assign an already allocated IST to the VM by populating the fields in the * corresponding VMTE. We re-use this code for both an SPI IST and LPI IST, even @@ -785,8 +793,30 @@ static int vgic_v5_vmte_assign_ist(struct kvm *kvm, phys_addr_t ist_base, /* Finally, mark the entry as valid */ cmd = spi_ist ? SPI_VIST_MAKE_VALID : LPI_VIST_MAKE_VALID; ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd); + if (ret) { + /* + * A timeout does not tell us whether the IRS consumed the + * command, so we're in some indeterminate and potentially + * transient state. It may still make the VIST valid and access + * the backing memory, so leave both the VMTE and allocation + * intact. Killing the VM ensures that teardown is the only path + * that can reclaim them, after it has successfully made the + * complete VMTE invalid. + */ + if (ret == -ETIMEDOUT) { + kvm_vm_dead(kvm); + return ret; + } - return ret; + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], 0ULL); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return ret; + } + + return 0; } /* @@ -1137,7 +1167,8 @@ int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits) ret = vgic_v5_vmte_assign_ist(kvm, base_addr, false, id_bits, 0, istsz, true); if (ret) { - vgic_v5_free_allocated_spi_ist(kvm); + if (ret != -ETIMEDOUT) + vgic_v5_free_allocated_spi_ist(kvm); return ret; } @@ -1207,7 +1238,7 @@ int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits) phys_addr = virt_to_phys(vmi->h_lpi_ist); ret = vgic_v5_vmte_assign_ist(kvm, phys_addr, two_level, id_bits, l2sz, istsz, false); - if (ret) + if (ret && ret != -ETIMEDOUT) vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz); return ret; diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h index a711a247cc6b..38b6f5269a2a 100644 --- a/arch/arm64/kvm/vgic/vgic-v5-tables.h +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h @@ -106,6 +106,7 @@ extern raw_spinlock_t vgic_v5_irs_lock; int vgic_v5_vmt_allocate(unsigned int max_vpes); int vgic_v5_vmt_free(void); +phys_addr_t vgic_v5_get_vmt_base(void); int vgic_v5_allocate_vm_id(struct kvm *kvm); void vgic_v5_release_vm_id(struct kvm *kvm); diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index c65355483f64..3d8f17301567 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -16,6 +16,9 @@ #define ppi_caps kvm_vgic_global_state.vgic_v5_ppi_caps #define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps +static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits, phys_addr_t vmt_base); +static int vgic_v5_irs_clear_vmt(void); + /* * Not all PPIs are guaranteed to be implemented for GICv5. Deterermine which * ones are, and generate a mask. @@ -43,6 +46,21 @@ static u32 irs_readl_relaxed(const u32 reg_offset) return readl_relaxed(irs_caps.irs_base + reg_offset); } +static void irs_writel_relaxed(const u32 val, const u32 reg_offset) +{ + writel_relaxed(val, irs_caps.irs_base + reg_offset); +} + +static u64 irs_readq_relaxed(const u32 reg_offset) +{ + return readq_relaxed(irs_caps.irs_base + reg_offset); +} + +static void irs_writeq_relaxed(const u64 val, const u32 reg_offset) +{ + writeq_relaxed(val, irs_caps.irs_base + reg_offset); +} + static void vgic_v5_irs_cache_id_regs(const struct gic_kvm_info *info) { irs_caps.irs_base = info->gicv5_irs.base; @@ -62,6 +80,7 @@ int vgic_v5_probe(const struct gic_kvm_info *info) int ret; kvm_vgic_global_state.type = VGIC_V5; + kvm_vgic_global_state.max_gic_vcpus = VGIC_V5_MAX_CPUS; kvm_vgic_global_state.vcpu_base = 0; kvm_vgic_global_state.vctrl_base = NULL; @@ -82,13 +101,52 @@ int vgic_v5_probe(const struct gic_kvm_info *info) vgic_v5_irs_cache_id_regs(info); vgic_v5_get_implemented_ppis(); + /* + * Even if the HW supports more per-VM vCPUs, artificially cap as we + * can't use them all. + */ kvm_vgic_global_state.max_gic_vcpus = min(vgic_v5_irs_max_vpes(&irs_caps), VGIC_V5_MAX_CPUS); + /* + * GICv5 requires a set of tables to be allocated in order to manage + * VMs. We allocate them in advance here, which alas means that we + * already have to make a decisions regarding the maximum number of VMs + * we want to run. For now, we match the maximum number offered by the + * hardware, but this might not be a wise choice in the long term. + */ + ret = vgic_v5_vmt_allocate(kvm_vgic_global_state.max_gic_vcpus); + if (ret) { + kvm_err("Failed to allocate the GICv5 VM tables; no GICv5 support\n"); + return -ENODEV; + } + + /* + * We've now allocated the VM table, but the host's IRS doesn't know + * about it yet. Provide the base address of the VMT to the IRS, as well + * as the number of ID bits that it covers and the structure used + * (linear/two-level). + */ + ret = vgic_v5_irs_assign_vmt(vgic_v5_irs_two_level_vmt_support(&irs_caps), + ilog2(vgic_v5_irs_max_vms(&irs_caps)), + vgic_v5_get_vmt_base()); + if (ret) { + kvm_err("Failed to assign the GICv5 VM tables to the IRS; no GICv5 support\n"); + if (!vgic_v5_irs_clear_vmt()) + vgic_v5_vmt_free(); + return -ENODEV; + } + ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V5); if (ret) { kvm_err("Cannot register GICv5 KVM device.\n"); - goto skip_v5; + /* + * Don't free the VMT itself if the hardware still has a valid + * pointer to it. + */ + if (!vgic_v5_irs_clear_vmt()) + vgic_v5_vmt_free(); + return -ENODEV; } v5_registered = true; @@ -113,12 +171,13 @@ int vgic_v5_probe(const struct gic_kvm_info *info) ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3); if (ret) { kvm_err("Cannot register GICv3-legacy KVM device.\n"); - return ret; + /* vGICv5 should still work */ + return v5_registered ? 0 : ret; } /* We potentially limit the max VCPUs further than we need to here */ kvm_vgic_global_state.max_gic_vcpus = min(VGIC_V3_MAX_CPUS, - VGIC_V5_MAX_CPUS); + kvm_vgic_global_state.max_gic_vcpus); static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif); kvm_info("GCIE legacy system register CPU interface\n"); @@ -128,20 +187,145 @@ int vgic_v5_probe(const struct gic_kvm_info *info) return 0; } +/* + * Wait for completion of a change in any of IRS_VMT_BASER, IRS_VMAP_L2_VMTR, + * IRS_VMAP_VMR, IRS_VMAP_VPER, IRS_VMAP_VISTR, IRS_VMAP_L2_VISTR. + */ +static int vgic_v5_irs_wait_for_vm_op(void) +{ + return gicv5_wait_for_op_atomic(irs_caps.irs_base, + GICV5_IRS_VMT_STATUSR, + GICV5_IRS_VMT_STATUSR_IDLE, + NULL); +} + +static int vgic_v5_irs_write_vm_mmio_reg(u64 val, u32 offset) +{ + int ret; + + lockdep_assert_held(&vgic_v5_irs_lock); + + /* Make sure that we are idle to begin with */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + irs_writeq_relaxed(val, offset); + + return vgic_v5_irs_wait_for_vm_op(); +} + +static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits, + phys_addr_t vmt_base) +{ + u64 vmt_baser; + u32 vmt_cfgr; + int ret; + + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + vmt_baser = irs_readq_relaxed(GICV5_IRS_VMT_BASER); + if (!!FIELD_GET(GICV5_IRS_VMT_BASER_VALID, vmt_baser)) + return -EBUSY; + + vmt_cfgr = FIELD_PREP(GICV5_IRS_VMT_CFGR_VM_ID_BITS, vm_id_bits); + if (two_level) + vmt_cfgr |= FIELD_PREP(GICV5_IRS_VMT_CFGR_STRUCTURE, + GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL); + + irs_writel_relaxed(vmt_cfgr, GICV5_IRS_VMT_CFGR); + + /* The base address is intentionally only masked and not shifted */ + vmt_baser = FIELD_PREP(GICV5_IRS_VMT_BASER_VALID, true) | + (vmt_base & GICV5_IRS_VMT_BASER_ADDR); + irs_writeq_relaxed(vmt_baser, GICV5_IRS_VMT_BASER); + + return vgic_v5_irs_wait_for_vm_op(); +} + +static int vgic_v5_irs_clear_vmt(void) +{ + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + return vgic_v5_irs_write_vm_mmio_reg(0, GICV5_IRS_VMT_BASER); +} + +static int vgic_v5_irs_vmap_l2_vmt(u16 vm_id) +{ + u64 val = FIELD_PREP(GICV5_IRS_VMAP_L2_VMTR_VM_ID, vm_id) | + GICV5_IRS_VMAP_L2_VMTR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_L2_VMTR); +} + +static int __vgic_v5_irs_vmap_vm(u16 vm_id, bool unmap) +{ + u64 val = FIELD_PREP(GICV5_IRS_VMAP_VMR_VM_ID, vm_id) | + FIELD_PREP(GICV5_IRS_VMAP_VMR_U, unmap) | + GICV5_IRS_VMAP_VMR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VMR); +} + +static int vgic_v5_irs_set_vm_valid(u16 vm_id) +{ + return __vgic_v5_irs_vmap_vm(vm_id, false); +} + +static int vgic_v5_irs_set_vm_invalid(u16 vm_id) +{ + return __vgic_v5_irs_vmap_vm(vm_id, true); +} + +static int __vgic_v5_irs_update_vist_validity(u16 vm_id, bool spi_ist, bool unmap) +{ + u8 type = spi_ist ? 0b011 : 0b010; + u64 val = FIELD_PREP(GICV5_IRS_VMAP_VISTR_TYPE, type) | + FIELD_PREP(GICV5_IRS_VMAP_VISTR_VM_ID, vm_id) | + FIELD_PREP(GICV5_IRS_VMAP_VISTR_U, unmap) | + GICV5_IRS_VMAP_VISTR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VISTR); +} + +static int vgic_v5_irs_set_vist_valid(u16 vm_id, bool spi_ist) +{ + return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, false); +} + +/* + * LPI ISTs can be invalidated explicitly. SPI ISTs are invalidated by making + * the VMTE invalid during teardown. + */ +static int vgic_v5_irs_set_vist_invalid(u16 vm_id, bool spi_ist) +{ + return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, true); +} + static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) { + struct vgic_v5_vm *vm = data->domain->host_data; enum gicv5_vcpu_cmd *cmd = vcpu_info; guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); switch (*cmd) { case VMT_L2_MAP: + return vgic_v5_irs_vmap_l2_vmt(vm->vm_id); case VMTE_MAKE_VALID: + return vgic_v5_irs_set_vm_valid(vm->vm_id); case VMTE_MAKE_INVALID: + return vgic_v5_irs_set_vm_invalid(vm->vm_id); case SPI_VIST_MAKE_VALID: + return vgic_v5_irs_set_vist_valid(vm->vm_id, true); case LPI_VIST_MAKE_VALID: + return vgic_v5_irs_set_vist_valid(vm->vm_id, false); case LPI_VIST_MAKE_INVALID: - /* Not yet implemented */ + return vgic_v5_irs_set_vist_invalid(vm->vm_id, false); default: return -EINVAL; }