diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 6c691fb2af5f..a833d90845b9 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6483,7 +6483,8 @@ Errors: ========== =============================================================== EINVAL The specified `gpa` and `size` were invalid (e.g. not - page aligned, causes an overflow, or size is zero). + page aligned, causes an overflow, or size is zero), or the VM + is UCONTROL (s390). ENOENT The specified `gpa` is outside defined memslots. EINTR An unmasked signal is pending and no page was processed. EFAULT The parameter address was invalid. @@ -6506,7 +6507,7 @@ Errors: KVM_PRE_FAULT_MEMORY populates KVM's stage-2 page tables used to map memory for the current vCPU state. KVM maps memory as if the vCPU generated a stage-2 read page fault, e.g. faults in memory as needed, but doesn't break -CoW. However, KVM does not mark any newly created stage-2 PTE as Accessed. +CoW. On x86, KVM does not mark any newly created stage-2 PTE as Accessed. In the case of confidential VM types where there is an initial set up of private guest memory before the guest is 'finalized'/measured, this ioctl @@ -8933,6 +8934,21 @@ helpful if user space wants to emulate instructions which are not This capability can be enabled dynamically even if VCPUs were already created and are running. +7.47 KVM_CAP_S390_HPAGE_2G +-------------------------- + +:Architectures: s390 +:Parameters: none +:Returns: 0 on success; -EINVAL if hpage_2g module parameter was not set, + cmma is enabled, or the VM has the KVM_VM_S390_UCONTROL + flag set; -EBUSY if vCPUs were already created for the VM. + +With this capability the KVM support for memory backing with 2g pages +through hugetlbfs can be enabled for a VM. After the capability is +enabled, cmma can't be enabled anymore and pfmfi and the storage key +interpretation are disabled. If cmma has already been enabled or the +hpage_2g module parameter is not set to 1, -EINVAL is returned. + 8. Other capabilities. ====================== diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 8a4f4a39f7a2..eaa34c5bd3c1 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -359,7 +359,7 @@ struct kvm_s390_float_interrupt { struct kvm_s390_mchk_info mchk; struct kvm_s390_ext_info srv_signal; int last_sleep_cpu; - struct mutex ais_lock; + spinlock_t ais_lock; u8 simm; u8 nimm; }; @@ -448,6 +448,12 @@ struct kvm_vcpu_arch { struct kvm_vm_stat { struct kvm_vm_stat_generic generic; u64 inject_io; + u64 io_390_adapter_map; + u64 io_390_adapter_unmap; + u64 io_390_inatomic; + u64 io_flic_inject_airq; + u64 io_set_adapter_int; + u64 io_390_inatomic_no_inject; u64 inject_float_mchk; u64 inject_pfault_done; u64 inject_service_signal; @@ -479,6 +485,9 @@ struct s390_io_adapter { bool masked; bool swap; bool suppressible; + spinlock_t maps_lock; + struct list_head maps; + unsigned int nr_maps; }; #define MAX_S390_IO_ADAPTERS ((MAX_ISC + 1) * 8) @@ -504,6 +513,18 @@ struct kvm_s390_cpu_model { struct kvm_s390_vm_cpu_uv_feat uv_feat_guest; }; +#define S390_ARCH_FAC_FORMAT_2 2 +struct kvm_s390_flcb2 { + union { + struct { + u8 reserved0[7]; + u8 length; + }; + u64 header_val; + }; + u64 facilities[S390_ARCH_FAC_LIST_SIZE_U64]; +}; + typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); struct kvm_s390_crypto { diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 0f184dbdbe5e..0f21501d3e86 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -104,6 +104,7 @@ struct sclp_info { unsigned char has_aisii : 1; unsigned char has_aeni : 1; unsigned char has_aisi : 1; + unsigned char has_astfleie2 : 1; unsigned int ibc; unsigned int mtid; unsigned int mtid_cp; diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h index 60345dd2cba2..4192769b5ce0 100644 --- a/arch/s390/include/uapi/asm/kvm.h +++ b/arch/s390/include/uapi/asm/kvm.h @@ -444,6 +444,7 @@ struct kvm_s390_vm_cpu_machine { #define KVM_S390_VM_CPU_FEAT_PFMFI 11 #define KVM_S390_VM_CPU_FEAT_SIGPIF 12 #define KVM_S390_VM_CPU_FEAT_KSS 13 +#define KVM_S390_VM_CPU_FEAT_ASTFLEIE2 14 struct kvm_s390_vm_cpu_feat { __u64 feat[16]; }; diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index 5b835bc6a194..8d3ee17a1bcb 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -30,6 +30,7 @@ config KVM select KVM_VFIO select VIRT_XFER_TO_GUEST_WORK select KVM_MMU_LOCKLESS_AGING + select KVM_GENERIC_PRE_FAULT_MEMORY help Support hosting paravirtualized guest machines using the SIE virtualization capability on the mainframe. This should work diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h index 873e13ac5a27..fad605305e05 100644 --- a/arch/s390/kvm/dat.h +++ b/arch/s390/kvm/dat.h @@ -501,6 +501,7 @@ struct guest_fault { bool write_attempt; /* Write access attempted */ bool attempt_pfault; /* Attempt a pfault first */ bool valid; /* This entry contains valid data */ + bool crste_region3; /* Whether crstep refers to a region3 entry */ void (*callback)(struct guest_fault *f); void *priv; }; diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index 1d289f8fa3b2..77829f787dc7 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -105,6 +105,11 @@ static void gmap_add_child(struct gmap *parent, struct gmap *child) else clear_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &child->flags); + if (test_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &parent->flags)) + set_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &child->flags); + else + clear_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &child->flags); + if (kvm_is_ucontrol(parent->kvm)) clear_bit(GMAP_FLAG_OWNS_PAGETABLES, &child->flags); list_add(&child->list, &parent->children); @@ -543,6 +548,7 @@ static int gmap_handle_minor_crste_fault(struct gmap *gmap, struct guest_fault * f->pfn = PHYS_PFN(large_crste_to_phys(oldcrste, f->gfn)); f->writable = oldcrste.s.fc1.w; + f->crste_region3 = is_pud(oldcrste); /* Appropriate permissions already (race with another handler), nothing to do. */ if (!oldcrste.h.i && !(f->write_attempt && oldcrste.h.p)) return 0; @@ -630,10 +636,27 @@ int gmap_try_fixup_minor(struct gmap *gmap, struct guest_fault *fault) return rc; } +/** + * gmap_2g_allowed() - Check whether a 2G hugepage is allowed. + * @gmap: The gmap of the guest. + * @f: Describes the fault that is being resolved. + * @slot: The memslot the faulting address belongs to. + * + * The function checks whether the GMAP_FLAG_ALLOW_HPAGE_2G flag is set for + * @gmap, whether the offset of the address in the 2G virtual frame is the + * same as the offset in the physical 2G frame, and finally whether the whole + * 2G page would fit in the given memslot. + * + * Return: true if a 2G hugepage is allowed to back the faulting address, false + * otherwise. + */ static inline bool gmap_2g_allowed(struct gmap *gmap, struct guest_fault *f, struct kvm_memory_slot *slot) { - return false; + return test_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &gmap->flags) && + !((f->gfn ^ f->pfn) & ~_REGION3_FR_MASK) && + slot->base_gfn <= ALIGN_DOWN(f->gfn, _PAGES_PER_REGION3) && + slot->base_gfn + slot->npages >= ALIGN(f->gfn + 1, _PAGES_PER_REGION3); } /** @@ -702,6 +725,7 @@ static int _gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, int leve if (oldval.val != _CRSTE_EMPTY(oldval.h.tt).val && crste_origin_large(oldval) != crste_origin_large(newval)) return -EAGAIN; + f->crste_region3 = is_pud(newval); } while (!gmap_crstep_xchg_atomic(gmap, f->crstep, oldval, newval, f->gfn)); if (f->callback) f->callback(f); diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c index 39aff324203e..1980df61ef30 100644 --- a/arch/s390/kvm/intercept.c +++ b/arch/s390/kvm/intercept.c @@ -517,8 +517,9 @@ static int handle_pv_spx(struct kvm_vcpu *vcpu) static int handle_pv_sclp(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); /* * 2 cases: * a: an sccb answering interrupt was already pending or in flight. @@ -534,7 +535,7 @@ static int handle_pv_sclp(struct kvm_vcpu *vcpu) fi->srv_signal.ext_params |= 0x43000; set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); clear_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 3bcdbbbb6891..9e3e6b0d72ad 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -624,8 +624,9 @@ static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) struct kvm_s390_mchk_info mchk = {}; int deliver = 0; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); spin_lock(&li->lock); if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) || test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) { @@ -654,7 +655,7 @@ static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) deliver = 1; } spin_unlock(&li->lock); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (deliver) { VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx", @@ -941,11 +942,12 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_ext_info ext; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) || !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } ext = fi->srv_signal; @@ -954,7 +956,7 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu) clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); if (kvm_s390_pv_cpu_is_protected(vcpu)) set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (!ext.ext_params) return 0; @@ -972,17 +974,18 @@ static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_ext_info ext; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } ext = fi->srv_signal; /* only clear the event bits */ fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING; clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event"); vcpu->stat.deliver_service_signal++; @@ -997,8 +1000,9 @@ static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_interrupt_info *inti; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT], struct kvm_s390_interrupt_info, list); @@ -1008,7 +1012,7 @@ static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) } if (list_empty(&fi->lists[FIRQ_LIST_PFAULT])) clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, @@ -1039,8 +1043,9 @@ static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_interrupt_info *inti; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO], struct kvm_s390_interrupt_info, list); @@ -1058,7 +1063,7 @@ static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) } if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO])) clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, @@ -1116,10 +1121,11 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info io; u32 isc; int rc = 0; + unsigned long flags; fi = &vcpu->kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); isc = irq_type_to_isc(irq_type); isc_list = &fi->lists[isc]; inti = list_first_entry_or_null(isc_list, @@ -1146,7 +1152,7 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu, } if (list_empty(isc_list)) clear_bit(irq_type, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { rc = __do_deliver_io(vcpu, &(inti->io)); @@ -1662,8 +1668,9 @@ static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, struct kvm_s390_interrupt_info *iter; u16 id = (schid & 0xffff0000U) >> 16; u16 nr = schid & 0x0000ffffU; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); list_for_each_entry(iter, isc_list, list) { if (schid && (id != iter->io.subchannel_id || nr != iter->io.subchannel_nr)) @@ -1673,10 +1680,10 @@ static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, fi->counters[FIRQ_CNTR_IO] -= 1; if (list_empty(isc_list)) clear_bit(isc_to_irq_type(isc), &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return iter; } - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return NULL; } @@ -1769,9 +1776,10 @@ static int __inject_service(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_service_signal++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING; /* We always allow events, track them separately from the sccb ints */ @@ -1791,7 +1799,7 @@ static int __inject_service(struct kvm *kvm, fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK; set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); out: - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kfree(inti); return 0; } @@ -1800,17 +1808,18 @@ static int __inject_virtio(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_virtio++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_VIRTIO] += 1; list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]); set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1818,18 +1827,19 @@ static int __inject_pfault_done(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_pfault_done++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_PFAULT] >= (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_PFAULT] += 1; list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]); set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1838,13 +1848,14 @@ static int __inject_float_mchk(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_float_mchk++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS); fi->mchk.mcic |= inti->mchk.mcic; set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kfree(inti); return 0; } @@ -1855,6 +1866,7 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) struct kvm_s390_float_interrupt *fi; struct list_head *list; int isc; + unsigned long flags; kvm->stat.inject_io++; isc = int_word_to_isc(inti->io.io_int_word); @@ -1873,9 +1885,9 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) } fi = &kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_IO] += 1; @@ -1890,7 +1902,7 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; list_add_tail(&inti->list, list); set_bit(isc_to_irq_type(isc), &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1966,15 +1978,10 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) } int kvm_s390_inject_vm(struct kvm *kvm, - struct kvm_s390_interrupt *s390int) + struct kvm_s390_interrupt *s390int, struct kvm_s390_interrupt_info *inti) { - struct kvm_s390_interrupt_info *inti; int rc; - inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); - if (!inti) - return -ENOMEM; - inti->type = s390int->type; switch (inti->type) { case KVM_S390_INT_VIRTIO: @@ -2003,15 +2010,13 @@ int kvm_s390_inject_vm(struct kvm *kvm, inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull; break; default: - kfree(inti); return -EINVAL; } trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64, 2); rc = __inject_vm(kvm, inti); - if (rc) - kfree(inti); + return rc; } @@ -2176,12 +2181,13 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; int i; + unsigned long flags; mutex_lock(&kvm->lock); if (!kvm_s390_pv_is_protected(kvm)) fi->masked_irqs = 0; mutex_unlock(&kvm->lock); - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->pending_irqs = 0; memset(&fi->srv_signal, 0, sizeof(fi->srv_signal)); memset(&fi->mchk, 0, sizeof(fi->mchk)); @@ -2189,7 +2195,7 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm) clear_irq_list(&fi->lists[i]); for (i = 0; i < FIRQ_MAX_COUNT; i++) fi->counters[i] = 0; - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kvm_s390_gisa_clear(kvm); }; @@ -2204,6 +2210,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) int ret = 0; int n = 0; int i; + unsigned long flags; if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0) return -EINVAL; @@ -2235,7 +2242,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) } } fi = &kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); for (i = 0; i < FIRQ_LIST_COUNT; i++) { list_for_each_entry(inti, &fi->lists[i], list) { if (n == max_irqs) { @@ -2272,7 +2279,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) } out: - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); out_nolock: if (!ret && n > 0) { if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n)) @@ -2287,6 +2294,7 @@ static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_all ais; + unsigned long flags; if (attr->attr < sizeof(ais)) return -EINVAL; @@ -2294,10 +2302,10 @@ static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); ais.simm = fi->simm; ais.nimm = fi->nimm; - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais))) return -EFAULT; @@ -2411,34 +2419,46 @@ static int register_io_adapter(struct kvm_device *dev, { struct s390_io_adapter *adapter; struct kvm_s390_io_adapter adapter_info; + int rc = 0; + mutex_lock(&dev->kvm->lock); if (copy_from_user(&adapter_info, - (void __user *)attr->addr, sizeof(adapter_info))) - return -EFAULT; - - if (adapter_info.id >= MAX_S390_IO_ADAPTERS) - return -EINVAL; - + (void __user *)attr->addr, sizeof(adapter_info))) { + rc = -EFAULT; + goto out; + } + if (adapter_info.id >= MAX_S390_IO_ADAPTERS) { + rc = -EINVAL; + goto out; + } adapter_info.id = array_index_nospec(adapter_info.id, MAX_S390_IO_ADAPTERS); - if (dev->kvm->arch.adapters[adapter_info.id] != NULL) - return -EINVAL; - + if (dev->kvm->arch.adapters[adapter_info.id] != NULL) { + rc = -EINVAL; + goto out; + } adapter = kzalloc_obj(*adapter, GFP_KERNEL_ACCOUNT); - if (!adapter) - return -ENOMEM; + if (!adapter) { + rc = -ENOMEM; + goto out; + } + INIT_LIST_HEAD(&adapter->maps); + spin_lock_init(&adapter->maps_lock); + adapter->nr_maps = 0; adapter->id = adapter_info.id; adapter->isc = adapter_info.isc; adapter->maskable = adapter_info.maskable; adapter->masked = false; adapter->swap = adapter_info.swap; - adapter->suppressible = (adapter_info.flags) & + adapter->suppressible = adapter_info.flags & KVM_S390_ADAPTER_SUPPRESSIBLE; dev->kvm->arch.adapters[adapter->id] = adapter; - return 0; +out: + mutex_unlock(&dev->kvm->lock); + return rc; } int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) @@ -2453,12 +2473,151 @@ int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) return ret; } +static struct page *pin_map_page(struct kvm *kvm, u64 uaddr, + unsigned int gup_flags) +{ + struct mm_struct *mm = kvm->mm; + struct page *page = NULL; + int locked = 1; + + if (mmget_not_zero(mm)) { + mmap_read_lock(mm); + pin_user_pages_remote(mm, uaddr, 1, FOLL_WRITE | gup_flags, + &page, &locked); + if (locked) + mmap_read_unlock(mm); + mmput(mm); + } + + return page; +} + +static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) +{ + struct s390_io_adapter *adapter = get_io_adapter(kvm, id); + struct s390_map_info *map; + unsigned long flags; + __u64 host_addr; + int ret, idx; + + if (!adapter || !addr) + return -EINVAL; + + map = kzalloc_obj(*map, GFP_KERNEL_ACCOUNT); + if (!map) + return -ENOMEM; + + INIT_LIST_HEAD(&map->list); + idx = srcu_read_lock(&kvm->srcu); + host_addr = gpa_to_hva(kvm, addr); + if (kvm_is_error_hva(host_addr)) { + srcu_read_unlock(&kvm->srcu, idx); + ret = -EFAULT; + goto out; + } + srcu_read_unlock(&kvm->srcu, idx); + map->guest_addr = addr; + map->addr = host_addr; + map->page = pin_map_page(kvm, host_addr, FOLL_LONGTERM); + if (!map->page) { + ret = -EINVAL; + goto out; + } + spin_lock_irqsave(&adapter->maps_lock, flags); + if (adapter->nr_maps < MAX_S390_ADAPTER_MAPS) { + list_add_tail(&map->list, &adapter->maps); + adapter->nr_maps++; + ret = 0; + } else { + ret = -EINVAL; + } + spin_unlock_irqrestore(&adapter->maps_lock, flags); + if (ret) + unpin_user_page(map->page); +out: + if (ret) + kfree(map); + return ret; +} + +static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) +{ + struct s390_io_adapter *adapter = get_io_adapter(kvm, id); + struct s390_map_info *map, *tmp, *map_to_free; + struct page *map_page_to_put = NULL; + u64 map_addr_to_mark = 0; + unsigned long flags; + int found = 0, idx; + + if (!adapter || !addr) + return -EINVAL; + + spin_lock_irqsave(&adapter->maps_lock, flags); + list_for_each_entry_safe(map, tmp, &adapter->maps, list) { + if (map->guest_addr == addr) { + found = 1; + adapter->nr_maps--; + list_del(&map->list); + map_page_to_put = map->page; + map_addr_to_mark = map->guest_addr; + map_to_free = map; + break; + } + } + spin_unlock_irqrestore(&adapter->maps_lock, flags); + + if (found) { + kfree(map_to_free); + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map_addr_to_mark >> PAGE_SHIFT); + set_page_dirty_lock(map_page_to_put); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map_page_to_put); + } + + return found ? 0 : -ENOENT; +} + +void kvm_s390_unmap_all_adapters(struct kvm *kvm) +{ + struct s390_map_info *map, *tmp; + unsigned long flags; + int i, idx; + + for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { + struct s390_io_adapter *adapter = kvm->arch.adapters[i]; + LIST_HEAD(local_list); + + if (!adapter) + continue; + + spin_lock_irqsave(&adapter->maps_lock, flags); + list_splice_init(&adapter->maps, &local_list); + adapter->nr_maps = 0; + spin_unlock_irqrestore(&adapter->maps_lock, flags); + + list_for_each_entry_safe(map, tmp, &local_list, list) { + list_del(&map->list); + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT); + set_page_dirty_lock(map->page); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map->page); + kfree(map); + } + } +} + void kvm_s390_destroy_adapters(struct kvm *kvm) { int i; - for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) + kvm_s390_unmap_all_adapters(kvm); + + for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { kfree(kvm->arch.adapters[i]); + kvm->arch.adapters[i] = NULL; + } } static int modify_io_adapter(struct kvm_device *dev, @@ -2480,14 +2639,22 @@ static int modify_io_adapter(struct kvm_device *dev, if (ret > 0) ret = 0; break; - /* - * The following operations are no longer needed and therefore no-ops. - * The gpa to hva translation is done when an IRQ route is set up. The - * set_irq code uses get_user_pages_remote() to do the actual write. - */ case KVM_S390_IO_ADAPTER_MAP: case KVM_S390_IO_ADAPTER_UNMAP: - ret = 0; + /* If in Secure Execution mode do not long term pin. */ + mutex_lock(&dev->kvm->lock); + if (kvm_s390_pv_is_protected(dev->kvm)) { + mutex_unlock(&dev->kvm->lock); + return 0; + } + if (req.type == KVM_S390_IO_ADAPTER_MAP) { + dev->kvm->stat.io_390_adapter_map++; + ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr); + } else { + dev->kvm->stat.io_390_adapter_unmap++; + ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr); + } + mutex_unlock(&dev->kvm->lock); break; default: ret = -EINVAL; @@ -2524,6 +2691,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_req req; int ret = 0; + unsigned long flags; if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; @@ -2540,7 +2708,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) 2 : KVM_S390_AIS_MODE_SINGLE : KVM_S390_AIS_MODE_ALL, req.mode); - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); switch (req.mode) { case KVM_S390_AIS_MODE_ALL: fi->simm &= ~AIS_MODE_MASK(req.isc); @@ -2553,7 +2721,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) default: ret = -EINVAL; } - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); return ret; } @@ -2567,25 +2735,41 @@ static int kvm_s390_inject_airq(struct kvm *kvm, .parm = 0, .parm64 = isc_to_int_word(adapter->isc), }; + struct kvm_s390_interrupt_info *inti; + unsigned long flags; + int ret = 0; - if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) - return kvm_s390_inject_vm(kvm, &s390int); + inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); + if (!inti) + return -ENOMEM; - mutex_lock(&fi->ais_lock); - if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { - trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); - goto out; + if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) { + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (ret) + kfree(inti); + return ret; } - ret = kvm_s390_inject_vm(kvm, &s390int); + spin_lock_irqsave(&fi->ais_lock, flags); + if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { + trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); + spin_unlock_irqrestore(&fi->ais_lock, flags); + kfree(inti); + return ret; + } + + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { fi->nimm |= AIS_MODE_MASK(adapter->isc); trace_kvm_s390_modify_ais_mode(adapter->isc, KVM_S390_AIS_MODE_SINGLE, 2); } -out: - mutex_unlock(&fi->ais_lock); + + spin_unlock_irqrestore(&fi->ais_lock, flags); + if (ret) + kfree(inti); return ret; } @@ -2594,6 +2778,8 @@ static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr) unsigned int id = attr->attr; struct s390_io_adapter *adapter = get_io_adapter(kvm, id); + kvm->stat.io_flic_inject_airq++; + if (!adapter) return -EINVAL; @@ -2604,6 +2790,7 @@ static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_all ais; + unsigned long flags; if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; @@ -2611,10 +2798,10 @@ static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais))) return -EFAULT; - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); fi->simm = ais.simm; fi->nimm = ais.nimm; - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); return 0; } @@ -2733,22 +2920,19 @@ static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; } -static struct page *get_map_page(struct kvm *kvm, u64 uaddr) +static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, + u64 addr) { - struct mm_struct *mm = kvm->mm; - struct page *page = NULL; - int locked = 1; + struct s390_map_info *map; - if (mmget_not_zero(mm)) { - mmap_read_lock(mm); - get_user_pages_remote(mm, uaddr, 1, FOLL_WRITE, - &page, &locked); - if (locked) - mmap_read_unlock(mm); - mmput(mm); + if (!adapter) + return NULL; + + list_for_each_entry(map, &adapter->maps, list) { + if (map->addr == addr) + return map; } - - return page; + return NULL; } static int adapter_indicators_set(struct kvm *kvm, @@ -2757,35 +2941,98 @@ static int adapter_indicators_set(struct kvm *kvm, { unsigned long bit; int summary_set, idx; - struct page *ind_page, *summary_page; + struct s390_map_info *ind_info, *summary_info; void *map; + struct page *ind_page, *summary_page; + unsigned long flags; - ind_page = get_map_page(kvm, adapter_int->ind_addr); - if (!ind_page) - return -1; - summary_page = get_map_page(kvm, adapter_int->summary_addr); - if (!summary_page) { - put_page(ind_page); - return -1; + ind_page = NULL; + + spin_lock_irqsave(&adapter->maps_lock, flags); + ind_info = get_map_info(adapter, adapter_int->ind_addr); + if (!ind_info) { + spin_unlock_irqrestore(&adapter->maps_lock, flags); + ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0); + if (!ind_page) + return -1; + idx = srcu_read_lock(&kvm->srcu); + map = page_address(ind_page); + bit = get_ind_bit(adapter_int->ind_addr, + adapter_int->ind_offset, adapter->swap); + set_bit(bit, map); + mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT); + set_page_dirty_lock(ind_page); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(ind_page); + } else { + map = page_address(ind_info->page); + bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); + set_bit(bit, map); + spin_unlock_irqrestore(&adapter->maps_lock, flags); } - idx = srcu_read_lock(&kvm->srcu); - map = page_address(ind_page); - bit = get_ind_bit(adapter_int->ind_addr, - adapter_int->ind_offset, adapter->swap); - set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT); - set_page_dirty_lock(ind_page); - map = page_address(summary_page); - bit = get_ind_bit(adapter_int->summary_addr, - adapter_int->summary_offset, adapter->swap); - summary_set = test_and_set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT); - set_page_dirty_lock(summary_page); - srcu_read_unlock(&kvm->srcu, idx); + spin_lock_irqsave(&adapter->maps_lock, flags); + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (!summary_info) { + spin_unlock_irqrestore(&adapter->maps_lock, flags); + summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0); + if (WARN_ON_ONCE(!summary_page)) + return -1; + idx = srcu_read_lock(&kvm->srcu); + map = page_address(summary_page); + bit = get_ind_bit(adapter_int->summary_addr, + adapter_int->summary_offset, adapter->swap); + summary_set = test_and_set_bit(bit, map); + mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT); + set_page_dirty_lock(summary_page); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(summary_page); + } else { + map = page_address(summary_info->page); + bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, + adapter->swap); + summary_set = test_and_set_bit(bit, map); + spin_unlock_irqrestore(&adapter->maps_lock, flags); + } - put_page(ind_page); - put_page(summary_page); + return summary_set ? 0 : 1; +} + +static int adapter_indicators_set_fast(struct kvm *kvm, + struct s390_io_adapter *adapter, + struct kvm_s390_adapter_int *adapter_int, + int setbit) +{ + unsigned long bit; + int summary_set; + struct s390_map_info *ind_info, *summary_info; + void *map; + + spin_lock(&adapter->maps_lock); + ind_info = get_map_info(adapter, adapter_int->ind_addr); + if (!ind_info) { + spin_unlock(&adapter->maps_lock); + return -EWOULDBLOCK; + } + map = page_address(ind_info->page); + bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); + if (setbit) + set_bit(bit, map); + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (!summary_info) { + spin_unlock(&adapter->maps_lock); + return -EWOULDBLOCK; + } + map = page_address(summary_info->page); + bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, + adapter->swap); + /* If setbit then set summary bit. Else if falling back to the slow path */ + /* with setbit==0 then clear the summary bit so the slow path re-injects */ + if (setbit) + summary_set = test_and_set_bit(bit, map); + else + summary_set = test_and_clear_bit(bit, map); + spin_unlock(&adapter->maps_lock); return summary_set ? 0 : 1; } @@ -2801,6 +3048,8 @@ static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, int ret; struct s390_io_adapter *adapter; + kvm->stat.io_set_adapter_int++; + /* We're only interested in the 0->1 transition. */ if (!level) return 0; @@ -2869,7 +3118,6 @@ int kvm_set_routing_entry(struct kvm *kvm, int idx; switch (ue->type) { - /* we store the userspace addresses instead of the guest addresses */ case KVM_IRQ_ROUTING_S390_ADAPTER: if (kvm_is_ucontrol(kvm)) return -EINVAL; @@ -3459,3 +3707,86 @@ int __init kvm_s390_gib_init(u8 nisc) out: return rc; } + +/* + * kvm_arch_set_irq_inatomic: fast-path for irqfd injection + */ +int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, int level, + bool line_status) +{ + int ret, setbit; + struct s390_io_adapter *adapter; + struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + struct kvm_s390_interrupt_info *inti; + struct kvm_s390_interrupt s390int = { + .type = KVM_S390_INT_IO(1, 0, 0, 0), + .parm = 0, + }; + + kvm->stat.io_390_inatomic++; + + /* We're only interested in the 0->1 transition. */ + if (!level) + return 0; + if (e->type != KVM_IRQ_ROUTING_S390_ADAPTER) + return -EWOULDBLOCK; + + adapter = get_io_adapter(kvm, e->adapter.adapter_id); + if (!adapter) + return -EWOULDBLOCK; + + s390int.parm64 = isc_to_int_word(adapter->isc); + setbit = 1; + ret = adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + if (ret < 0) + return -EWOULDBLOCK; + if (!ret || adapter->masked) { + kvm->stat.io_390_inatomic_no_inject++; + return 0; + } + + inti = kzalloc_obj(*inti, GFP_ATOMIC); + if (!inti) { + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + return -EWOULDBLOCK; + } + + if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) { + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (ret == 0) { + return ret; + } else { + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + kfree(inti); + return -EWOULDBLOCK; + } + } + + spin_lock(&fi->ais_lock); + if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { + trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); + spin_unlock(&fi->ais_lock); + kfree(inti); + kvm->stat.io_390_inatomic_no_inject++; + return 0; + } + + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { + fi->nimm |= AIS_MODE_MASK(adapter->isc); + trace_kvm_s390_modify_ais_mode(adapter->isc, + KVM_S390_AIS_MODE_SINGLE, 2); + } else if (ret) { + spin_unlock(&fi->ais_lock); + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + kfree(inti); + return -EWOULDBLOCK; + } + + spin_unlock(&fi->ais_lock); + return 0; +} diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index ffb20a64d328..dc9c7451fe97 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -68,6 +68,12 @@ const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_COUNTER(VM, inject_io), + STATS_DESC_COUNTER(VM, io_390_adapter_map), + STATS_DESC_COUNTER(VM, io_390_adapter_unmap), + STATS_DESC_COUNTER(VM, io_390_inatomic), + STATS_DESC_COUNTER(VM, io_flic_inject_airq), + STATS_DESC_COUNTER(VM, io_set_adapter_int), + STATS_DESC_COUNTER(VM, io_390_inatomic_no_inject), STATS_DESC_COUNTER(VM, inject_float_mchk), STATS_DESC_COUNTER(VM, inject_pfault_done), STATS_DESC_COUNTER(VM, inject_service_signal), @@ -204,11 +210,16 @@ static int nested; module_param(nested, int, S_IRUGO); MODULE_PARM_DESC(nested, "Nested virtualization support"); -/* allow 1m huge page guest backing, if !nested */ +/* allow 1m huge page guest backing */ static int hpage; module_param(hpage, int, 0444); MODULE_PARM_DESC(hpage, "1m huge page backing support"); +/* allow 2g huge page guest backing */ +static int hpage_2g; +module_param(hpage_2g, int, 0444); +MODULE_PARM_DESC(hpage_2g, "2g huge page backing support"); + /* maximum percentage of steal time for polling. >100 is treated like 100 */ static u8 halt_poll_max_steal = 10; module_param(halt_poll_max_steal, byte, 0644); @@ -232,33 +243,25 @@ static int async_destroy = 1; module_param(async_destroy, int, 0444); MODULE_PARM_DESC(async_destroy, "Asynchronous destroy for protected guests"); -/* - * For now we handle at most 16 double words as this is what the s390 base - * kernel handles and stores in the prefix page. If we ever need to go beyond - * this, this requires changes to code, but the external uapi can stay. - */ -#define SIZE_INTERNAL 16 - +#define HMFAI_DWORDS 16 /* * Base feature mask that defines default mask for facilities. Consists of the * defines in FACILITIES_KVM and the non-hypervisor managed bits. */ -static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM }; +static unsigned long kvm_s390_fac_base[HMFAI_DWORDS] = { FACILITIES_KVM }; +static_assert(ARRAY_SIZE(((long[]){ FACILITIES_KVM })) <= HMFAI_DWORDS); +static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_MASK_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_LIST_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= ARRAY_SIZE(stfle_fac_list)); + /* * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL * and defines the facilities that can be enabled via a cpu model. */ -static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL }; - -static unsigned long kvm_s390_fac_size(void) -{ - BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64); - BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64); - BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) > - sizeof(stfle_fac_list)); - - return SIZE_INTERNAL; -} +static const unsigned long kvm_s390_fac_ext[] = { FACILITIES_KVM_CPUMODEL }; +static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_MASK_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_LIST_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= ARRAY_SIZE(stfle_fac_list)); /* available cpu features supported by kvm */ static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); @@ -468,6 +471,8 @@ static void __init kvm_s390_cpu_feat_init(void) allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS); if (sclp.has_kss) allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS); + if (sclp.has_astfleie2) + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ASTFLEIE2); /* * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make * all skey handling functions read/set the skey from the PGSTE @@ -630,6 +635,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_S390_USER_OPEREXEC: case KVM_CAP_S390_KEYOP: case KVM_CAP_S390_VSIE_ESAMODE: + case KVM_CAP_PRE_FAULT_MEMORY: r = 1; break; case KVM_CAP_SET_GUEST_DEBUG2: @@ -640,6 +646,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) if (hpage && !(kvm && kvm_is_ucontrol(kvm))) r = 1; break; + case KVM_CAP_S390_HPAGE_2G: + r = 0; + if (hpage_2g && !(kvm && kvm_is_ucontrol(kvm))) + r = 1; + break; case KVM_CAP_S390_MEM_OP: r = MEM_OP_MAX_SIZE; break; @@ -896,6 +907,27 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s", r ? "(not available)" : "(success)"); break; + case KVM_CAP_S390_HPAGE_2G: + mutex_lock(&kvm->lock); + if (kvm->created_vcpus) { + r = -EBUSY; + } else if (!hpage_2g || kvm->arch.use_cmma || kvm_is_ucontrol(kvm)) { + r = -EINVAL; + } else { + r = 0; + set_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &kvm->arch.gmap->flags); + /* + * We might have to create fake 4k page + * tables. To avoid that the hardware works on + * stale PGSTEs, we emulate these instructions. + */ + kvm->arch.use_skf = 0; + kvm->arch.use_pfmfi = 0; + } + mutex_unlock(&kvm->lock); + VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE_2G %s", + r ? "(not available)" : "(success)"); + break; case KVM_CAP_S390_USER_STSI: VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI"); kvm->arch.user_stsi = 1; @@ -2252,7 +2284,7 @@ static int kvm_s390_get_cmma_bits(struct kvm *kvm, return 0; } - values = vmalloc(args->count); + values = vzalloc(args->count); if (!values) return -ENOMEM; @@ -2534,6 +2566,7 @@ static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd) if (kvm_s390_pv_is_protected(kvm)) break; + kvm_s390_unmap_all_adapters(kvm); mmap_write_lock(kvm->mm); /* * Disable creation of new THPs. Existing THPs can stay, they @@ -2869,6 +2902,7 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) void __user *argp = (void __user *)arg; struct kvm_device_attr attr; int r; + struct kvm_s390_interrupt_info *inti; switch (ioctl) { case KVM_S390_INTERRUPT: { @@ -2877,7 +2911,12 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) r = -EFAULT; if (copy_from_user(&s390int, argp, sizeof(s390int))) break; - r = kvm_s390_inject_vm(kvm, &s390int); + inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); + if (!inti) + return -ENOMEM; + r = kvm_s390_inject_vm(kvm, &s390int, inti); + if (r) + kfree(inti); break; } case KVM_CREATE_IRQCHIP: { @@ -3238,13 +3277,16 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm->arch.sie_page2->kvm = kvm; kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list; - for (i = 0; i < kvm_s390_fac_size(); i++) { + for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_base); i++) { kvm->arch.model.fac_mask[i] = stfle_fac_list[i] & - (kvm_s390_fac_base[i] | - kvm_s390_fac_ext[i]); + kvm_s390_fac_base[i]; kvm->arch.model.fac_list[i] = stfle_fac_list[i] & kvm_s390_fac_base[i]; } + for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_ext); i++) { + kvm->arch.model.fac_mask[i] |= stfle_fac_list[i] & + kvm_s390_fac_ext[i]; + } kvm->arch.model.subfuncs = kvm_s390_available_subfunc; /* we are always in czam mode - even on pre z14 machines */ @@ -3275,7 +3317,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) mutex_unlock(&kvm->lock); } - mutex_init(&kvm->arch.float_int.ais_lock); + spin_lock_init(&kvm->arch.float_int.ais_lock); spin_lock_init(&kvm->arch.float_int.lock); for (i = 0; i < FIRQ_LIST_COUNT; i++) INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]); @@ -4397,19 +4439,28 @@ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clo } static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, - unsigned long token) + unsigned long token) { struct kvm_s390_interrupt inti; struct kvm_s390_irq irq; + struct kvm_s390_interrupt_info *inti_mem = NULL; + int ret = 0; if (start_token) { irq.u.ext.ext_params2 = token; irq.type = KVM_S390_INT_PFAULT_INIT; WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq)); } else { + inti_mem = kzalloc_obj(*inti_mem, GFP_KERNEL_ACCOUNT); + if (WARN_ON_ONCE(!inti_mem)) + return; + inti.type = KVM_S390_INT_PFAULT_DONE; inti.parm64 = token; - WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti)); + ret = kvm_s390_inject_vm(vcpu->kvm, &inti, inti_mem); + if (ret) + kfree(inti_mem); + WARN_ON_ONCE(ret); } } @@ -5759,6 +5810,50 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, return; } +/** + * kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables + * @vcpu: the vcpu that shall appear to have generated the fault-in. + * @range: the range that needs to be faulted in. + * + * The first page of the given range is faulted in and the corresponding gmap + * page tables are created, as if the given vCPU had performed a read + * operation. + * If the range starts outside any memslots, an error is returned. An error is + * also returned for UCONTROL VMs, which should instead use the + * KVM_S390_VCPU_FAULT ioctl. + * + * Return: + * * %-ENOENT if the range lies outside of a memslot. + * * %-EINVAL in case of invalid state (for example if the VM is UCONTROL). + * * %-EIO if errors happen while faulting-in the page (will trigger a warning + * in the caller). + * * other error codes < 0 in case of other errors. + * * otherwise a number > 0 of bytes that have been faulted in successfully. + */ +long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) +{ + struct guest_fault f = { .gfn = gpa_to_gfn(range->gpa), }; + gpa_t end; + int rc; + + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; + + rc = kvm_s390_faultin_gfn(vcpu, NULL, &f); + if (rc == PGM_ADDRESSING) + return -ENOENT; + if (rc > 0) + return -EIO; + if (rc < 0) + return rc; + + if (f.ptep) + return PAGE_SIZE; + + end = ALIGN(range->gpa + PAGE_SIZE, f.crste_region3 ? _REGION3_SIZE : HPAGE_SIZE); + return min(range->size, end - range->gpa); +} + /** * kvm_test_age_gfn() - test young * @kvm: the kvm instance @@ -5820,9 +5915,13 @@ static int __init kvm_s390_init(void) return -ENODEV; } - for (i = 0; i < 16; i++) - kvm_s390_fac_base[i] |= - stfle_fac_list[i] & nonhyp_mask(i); + if (hpage_2g && !hpage) { + hpage_2g = 0; + pr_info("Disabling 2G hugepage support, since 1M hugepage support is not enabled.\n"); + } + + for (i = 0; i < HMFAI_DWORDS; i++) + kvm_s390_fac_base[i] |= nonhyp_mask(i); r = __kvm_s390_init(); if (r) diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index dc0573b7aa4b..6d2842fb71a3 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -376,7 +376,8 @@ int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu); void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu); void kvm_s390_clear_float_irqs(struct kvm *kvm); int __must_check kvm_s390_inject_vm(struct kvm *kvm, - struct kvm_s390_interrupt *s390int); + struct kvm_s390_interrupt *s390int, + struct kvm_s390_interrupt_info *inti); int __must_check kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq); static inline int kvm_s390_inject_prog_irq(struct kvm_vcpu *vcpu, @@ -560,6 +561,8 @@ void kvm_s390_gisa_disable(struct kvm *kvm); void kvm_s390_gisa_enable(struct kvm *kvm); int __init kvm_s390_gib_init(u8 nisc); void kvm_s390_gib_destroy(void); +void kvm_s390_unmap_all_adapters(struct kvm *kvm); + /* implemented in guestdbg.c */ void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu); diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c index 4b865e75351c..1beacc841ca8 100644 --- a/arch/s390/kvm/pv.c +++ b/arch/s390/kvm/pv.c @@ -740,7 +740,10 @@ int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc) uvcb.flags.ap_allow_instr = kvm->arch.model.uv_feat_guest.ap; uvcb.flags.ap_instr_intr = kvm->arch.model.uv_feat_guest.ap_intr; - clear_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags); + scoped_guard(write_lock, &kvm->mmu_lock) { + clear_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags); + clear_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &kvm->arch.gmap->flags); + } gmap_split_huge_pages(kvm->arch.gmap); cc = uv_call_sched(0, (u64)&uvcb); diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index e5a23f1c9749..eea24562e7db 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -6,12 +6,15 @@ * * Author(s): David Hildenbrand */ +#include #include #include #include +#include #include #include #include +#include #include #include @@ -62,9 +65,9 @@ struct vsie_page { gpa_t scb_gpa; /* 0x0258 */ /* the shadow gmap in use by the vsie_page */ struct gmap_cache gmap_cache; /* 0x0260 */ - __u8 reserved[0x0700 - 0x0278]; /* 0x0278 */ - struct kvm_s390_crypto_cb crycb; /* 0x0700 */ - __u8 fac[S390_ARCH_FAC_LIST_SIZE_BYTE]; /* 0x0800 */ + __u8 reserved[0x06f8 - 0x0278]; /* 0x0278 */ + struct kvm_s390_crypto_cb crycb; /* 0x06f8 */ + __u8 fac[8 + S390_ARCH_FAC_LIST_SIZE_BYTE];/* 0x07f8 */ }; static_assert(sizeof(struct vsie_page) == PAGE_SIZE); @@ -1000,6 +1003,45 @@ static void retry_vsie_icpt(struct vsie_page *vsie_page) clear_vsie_icpt(vsie_page); } +static int handle_stfle_0(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, + u32 fac_list_origin) +{ + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; + + /* + * format-0 -> size of nested guest's facility list == guest's size + * guest's size == host's size, since STFLE is interpretatively executed + * using a format-0 for the guest, too. + */ + if (read_guest_real(vcpu, fac_list_origin, &vsie_page->fac, + stfle_size() * sizeof(u64))) + return set_validity_icpt(scb_s, 0x1090U); + scb_s->fac = (u32)virt_to_phys(&vsie_page->fac); + return 0; +} + +static int handle_stfle_2(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, u32 fac_list_origin) +{ + struct kvm_s390_flcb2 *flcb_s = (struct kvm_s390_flcb2 *)vsie_page->fac; + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; + u64 len; + + if (read_guest_real(vcpu, fac_list_origin, &len, sizeof(len))) + return set_validity_icpt(scb_s, 0x1090U); + + /* discard reserved bits */ + len = (len & U8_MAX); + flcb_s->header_val = len; + len += 1; + + if (read_guest_real(vcpu, fac_list_origin + offsetof(struct kvm_s390_flcb2, facilities), + &flcb_s->facilities, len * sizeof(u64))) + return set_validity_icpt(scb_s, 0x1090U); + + scb_s->fac = (u32)virt_to_phys(&vsie_page->fac) | S390_ARCH_FAC_FORMAT_2; + return 0; +} + /* * Try to shadow + enable the guest 2 provided facility list. * Retry instruction execution if enabled for and provided by guest 2. @@ -1009,29 +1051,34 @@ static void retry_vsie_icpt(struct vsie_page *vsie_page) */ static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) { - struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; - __u32 fac = READ_ONCE(vsie_page->scb_o->fac); + bool has_astfleie2 = test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_ASTFLEIE2); + u32 fac = READ_ONCE(vsie_page->scb_o->fac); + int format_mask, format; + u32 origin; + + /* assert no overflow with maximum len */ + BUILD_BUG_ON(sizeof(vsie_page->fac) < ((S390_ARCH_FAC_LIST_SIZE_U64 + 1) * sizeof(u64))); + BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct vsie_page, fac), 8)); - /* - * Alternate-STFLE-Interpretive-Execution facilities are not supported - * -> format-0 flcb - */ if (fac && test_kvm_facility(vcpu->kvm, 7)) { retry_vsie_icpt(vsie_page); /* * The facility list origin (FLO) is in bits 1 - 28 of the FLD * so we need to mask here before reading. */ - fac = fac & 0x7ffffff8U; - /* - * format-0 -> size of nested guest's facility list == guest's size - * guest's size == host's size, since STFLE is interpretatively executed - * using a format-0 for the guest, too. - */ - if (read_guest_real(vcpu, fac, &vsie_page->fac, - stfle_size() * sizeof(u64))) - return set_validity_icpt(scb_s, 0x1090U); - scb_s->fac = (u32)virt_to_phys(&vsie_page->fac); + origin = fac & 0x7ffffff8U; + format_mask = has_astfleie2 ? 3 : 0; + format = fac & format_mask; + switch (format) { + case 0: + return handle_stfle_0(vcpu, vsie_page, origin); + case 1: + return set_validity_icpt(&vsie_page->scb_s, 0x1330U); + case 2: + return handle_stfle_2(vcpu, vsie_page, origin); + case 3: + return set_validity_icpt(&vsie_page->scb_s, 0x1330U); + } } return 0; } diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index 6bf501ad8ff0..22dd797e6229 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -61,8 +61,10 @@ static void __init sclp_early_facilities_detect(void) sclp.has_sipl = !!(sccb->cbl & 0x4000); sclp.has_sipl_eckd = !!(sccb->cbl & 0x2000); } - if (sccb->cpuoff > 139) + if (sccb->cpuoff > 139) { sclp.has_diag324 = !!(sccb->byte_139 & 0x80); + sclp.has_astfleie2 = !!(sccb->byte_139 & 0x40); + } sclp.rnmax = sccb->rnmax ? sccb->rnmax : sccb->rnmax2; sclp.rzm = sccb->rnsize ? sccb->rnsize : sccb->rnsize2; sclp.rzm <<= 20; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 6c8afa2047bf..419011097fa8 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -996,6 +996,7 @@ struct kvm_enable_cap { #define KVM_CAP_S390_USER_OPEREXEC 246 #define KVM_CAP_S390_KEYOP 247 #define KVM_CAP_S390_VSIE_ESAMODE 248 +#define KVM_CAP_S390_HPAGE_2G 249 struct kvm_irq_routing_irqchip { __u32 irqchip; diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index e0ddd3ff9472..d28a057fa6c2 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -213,6 +213,7 @@ TEST_GEN_PROGS_s390 += s390/keyop TEST_GEN_PROGS_s390 += rseq_test TEST_GEN_PROGS_s390 += s390/irq_routing TEST_GEN_PROGS_s390 += mmu_stress_test +TEST_GEN_PROGS_s390 += pre_fault_memory_test TEST_GEN_PROGS_riscv = $(TEST_GEN_PROGS_COMMON) TEST_GEN_PROGS_riscv += riscv/sbi_pmu_test diff --git a/tools/testing/selftests/kvm/pre_fault_memory_test.c b/tools/testing/selftests/kvm/pre_fault_memory_test.c index fcb57fd034e6..a0fcae3cb7a8 100644 --- a/tools/testing/selftests/kvm/pre_fault_memory_test.c +++ b/tools/testing/selftests/kvm/pre_fault_memory_test.c @@ -11,6 +11,7 @@ #include #include #include +#include /* Arbitrarily chosen values */ #define TEST_SIZE (SZ_2M + PAGE_SIZE) @@ -167,7 +168,6 @@ static void __test_pre_fault_memory(unsigned long vm_type, bool private) .type = vm_type, }; struct kvm_vcpu *vcpu; - struct kvm_run *run; struct kvm_vm *vm; struct ucall uc; @@ -193,11 +193,6 @@ static void __test_pre_fault_memory(unsigned long vm_type, bool private) vcpu_args_set(vcpu, 1, gva); vcpu_run(vcpu); - run = vcpu->run; - TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, - "Wanted KVM_EXIT_IO, got exit reason: %u (%s)", - run->exit_reason, exit_reason_str(run->exit_reason)); - switch (get_ucall(vcpu, &uc)) { case UCALL_ABORT: REPORT_GUEST_ASSERT(uc);