diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 4ff1ede90873..eaa34c5bd3c1 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -359,7 +359,7 @@ struct kvm_s390_float_interrupt { struct kvm_s390_mchk_info mchk; struct kvm_s390_ext_info srv_signal; int last_sleep_cpu; - struct mutex ais_lock; + spinlock_t ais_lock; u8 simm; u8 nimm; }; @@ -450,6 +450,10 @@ struct kvm_vm_stat { u64 inject_io; u64 io_390_adapter_map; u64 io_390_adapter_unmap; + u64 io_390_inatomic; + u64 io_flic_inject_airq; + u64 io_set_adapter_int; + u64 io_390_inatomic_no_inject; u64 inject_float_mchk; u64 inject_pfault_done; u64 inject_service_signal; diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c index 39aff324203e..1980df61ef30 100644 --- a/arch/s390/kvm/intercept.c +++ b/arch/s390/kvm/intercept.c @@ -517,8 +517,9 @@ static int handle_pv_spx(struct kvm_vcpu *vcpu) static int handle_pv_sclp(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); /* * 2 cases: * a: an sccb answering interrupt was already pending or in flight. @@ -534,7 +535,7 @@ static int handle_pv_sclp(struct kvm_vcpu *vcpu) fi->srv_signal.ext_params |= 0x43000; set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); clear_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index f276f76f814f..51ea576bb445 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -624,8 +624,9 @@ static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) struct kvm_s390_mchk_info mchk = {}; int deliver = 0; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); spin_lock(&li->lock); if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) || test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) { @@ -654,7 +655,7 @@ static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) deliver = 1; } spin_unlock(&li->lock); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (deliver) { VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx", @@ -941,11 +942,12 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_ext_info ext; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) || !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } ext = fi->srv_signal; @@ -954,7 +956,7 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu) clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); if (kvm_s390_pv_cpu_is_protected(vcpu)) set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (!ext.ext_params) return 0; @@ -972,17 +974,18 @@ static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_ext_info ext; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } ext = fi->srv_signal; /* only clear the event bits */ fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING; clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event"); vcpu->stat.deliver_service_signal++; @@ -997,8 +1000,9 @@ static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_interrupt_info *inti; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT], struct kvm_s390_interrupt_info, list); @@ -1008,7 +1012,7 @@ static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) } if (list_empty(&fi->lists[FIRQ_LIST_PFAULT])) clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, @@ -1039,8 +1043,9 @@ static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_interrupt_info *inti; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO], struct kvm_s390_interrupt_info, list); @@ -1058,7 +1063,7 @@ static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) } if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO])) clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, @@ -1116,10 +1121,11 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info io; u32 isc; int rc = 0; + unsigned long flags; fi = &vcpu->kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); isc = irq_type_to_isc(irq_type); isc_list = &fi->lists[isc]; inti = list_first_entry_or_null(isc_list, @@ -1146,7 +1152,7 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu, } if (list_empty(isc_list)) clear_bit(irq_type, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { rc = __do_deliver_io(vcpu, &(inti->io)); @@ -1662,8 +1668,9 @@ static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, struct kvm_s390_interrupt_info *iter; u16 id = (schid & 0xffff0000U) >> 16; u16 nr = schid & 0x0000ffffU; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); list_for_each_entry(iter, isc_list, list) { if (schid && (id != iter->io.subchannel_id || nr != iter->io.subchannel_nr)) @@ -1673,10 +1680,10 @@ static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, fi->counters[FIRQ_CNTR_IO] -= 1; if (list_empty(isc_list)) clear_bit(isc_to_irq_type(isc), &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return iter; } - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return NULL; } @@ -1769,9 +1776,10 @@ static int __inject_service(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_service_signal++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING; /* We always allow events, track them separately from the sccb ints */ @@ -1791,7 +1799,7 @@ static int __inject_service(struct kvm *kvm, fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK; set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); out: - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kfree(inti); return 0; } @@ -1800,17 +1808,18 @@ static int __inject_virtio(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_virtio++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_VIRTIO] += 1; list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]); set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1818,18 +1827,19 @@ static int __inject_pfault_done(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_pfault_done++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_PFAULT] >= (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_PFAULT] += 1; list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]); set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1838,13 +1848,14 @@ static int __inject_float_mchk(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_float_mchk++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS); fi->mchk.mcic |= inti->mchk.mcic; set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kfree(inti); return 0; } @@ -1855,6 +1866,7 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) struct kvm_s390_float_interrupt *fi; struct list_head *list; int isc; + unsigned long flags; kvm->stat.inject_io++; isc = int_word_to_isc(inti->io.io_int_word); @@ -1873,9 +1885,9 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) } fi = &kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_IO] += 1; @@ -1890,7 +1902,7 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; list_add_tail(&inti->list, list); set_bit(isc_to_irq_type(isc), &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1966,15 +1978,10 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) } int kvm_s390_inject_vm(struct kvm *kvm, - struct kvm_s390_interrupt *s390int) + struct kvm_s390_interrupt *s390int, struct kvm_s390_interrupt_info *inti) { - struct kvm_s390_interrupt_info *inti; int rc; - inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); - if (!inti) - return -ENOMEM; - inti->type = s390int->type; switch (inti->type) { case KVM_S390_INT_VIRTIO: @@ -2003,15 +2010,13 @@ int kvm_s390_inject_vm(struct kvm *kvm, inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull; break; default: - kfree(inti); return -EINVAL; } trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64, 2); rc = __inject_vm(kvm, inti); - if (rc) - kfree(inti); + return rc; } @@ -2176,12 +2181,13 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; int i; + unsigned long flags; mutex_lock(&kvm->lock); if (!kvm_s390_pv_is_protected(kvm)) fi->masked_irqs = 0; mutex_unlock(&kvm->lock); - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->pending_irqs = 0; memset(&fi->srv_signal, 0, sizeof(fi->srv_signal)); memset(&fi->mchk, 0, sizeof(fi->mchk)); @@ -2189,7 +2195,7 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm) clear_irq_list(&fi->lists[i]); for (i = 0; i < FIRQ_MAX_COUNT; i++) fi->counters[i] = 0; - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kvm_s390_gisa_clear(kvm); }; @@ -2204,6 +2210,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) int ret = 0; int n = 0; int i; + unsigned long flags; if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0) return -EINVAL; @@ -2235,7 +2242,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) } } fi = &kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); for (i = 0; i < FIRQ_LIST_COUNT; i++) { list_for_each_entry(inti, &fi->lists[i], list) { if (n == max_irqs) { @@ -2272,7 +2279,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) } out: - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); out_nolock: if (!ret && n > 0) { if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n)) @@ -2287,6 +2294,7 @@ static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_all ais; + unsigned long flags; if (attr->attr < sizeof(ais)) return -EINVAL; @@ -2294,10 +2302,10 @@ static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); ais.simm = fi->simm; ais.nimm = fi->nimm; - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais))) return -EFAULT; @@ -2683,6 +2691,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_req req; int ret = 0; + unsigned long flags; if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; @@ -2699,7 +2708,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) 2 : KVM_S390_AIS_MODE_SINGLE : KVM_S390_AIS_MODE_ALL, req.mode); - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); switch (req.mode) { case KVM_S390_AIS_MODE_ALL: fi->simm &= ~AIS_MODE_MASK(req.isc); @@ -2712,7 +2721,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) default: ret = -EINVAL; } - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); return ret; } @@ -2726,25 +2735,41 @@ static int kvm_s390_inject_airq(struct kvm *kvm, .parm = 0, .parm64 = isc_to_int_word(adapter->isc), }; + struct kvm_s390_interrupt_info *inti; + unsigned long flags; + int ret = 0; - if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) - return kvm_s390_inject_vm(kvm, &s390int); + inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); + if (!inti) + return -ENOMEM; - mutex_lock(&fi->ais_lock); - if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { - trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); - goto out; + if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) { + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (ret) + kfree(inti); + return ret; } - ret = kvm_s390_inject_vm(kvm, &s390int); + spin_lock_irqsave(&fi->ais_lock, flags); + if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { + trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); + spin_unlock_irqrestore(&fi->ais_lock, flags); + kfree(inti); + return ret; + } + + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { fi->nimm |= AIS_MODE_MASK(adapter->isc); trace_kvm_s390_modify_ais_mode(adapter->isc, KVM_S390_AIS_MODE_SINGLE, 2); } -out: - mutex_unlock(&fi->ais_lock); + + spin_unlock_irqrestore(&fi->ais_lock, flags); + if (ret) + kfree(inti); return ret; } @@ -2753,6 +2778,8 @@ static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr) unsigned int id = attr->attr; struct s390_io_adapter *adapter = get_io_adapter(kvm, id); + kvm->stat.io_flic_inject_airq++; + if (!adapter) return -EINVAL; @@ -2763,6 +2790,7 @@ static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_all ais; + unsigned long flags; if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; @@ -2770,10 +2798,10 @@ static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais))) return -EFAULT; - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); fi->simm = ais.simm; fi->nimm = ais.nimm; - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); return 0; } @@ -2942,6 +2970,7 @@ static int adapter_indicators_set(struct kvm *kvm, set_bit(bit, map); spin_unlock_irqrestore(&adapter->maps_lock, flags); } + spin_lock_irqsave(&adapter->maps_lock, flags); summary_info = get_map_info(adapter, adapter_int->summary_addr); if (!summary_info) { @@ -2969,6 +2998,44 @@ static int adapter_indicators_set(struct kvm *kvm, return summary_set ? 0 : 1; } +static int adapter_indicators_set_fast(struct kvm *kvm, + struct s390_io_adapter *adapter, + struct kvm_s390_adapter_int *adapter_int, + int setbit) +{ + unsigned long bit; + int summary_set; + struct s390_map_info *ind_info, *summary_info; + void *map; + + spin_lock(&adapter->maps_lock); + ind_info = get_map_info(adapter, adapter_int->ind_addr); + if (!ind_info) { + spin_unlock(&adapter->maps_lock); + return -EWOULDBLOCK; + } + map = page_address(ind_info->page); + bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); + if (setbit) + set_bit(bit, map); + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (!summary_info) { + spin_unlock(&adapter->maps_lock); + return -EWOULDBLOCK; + } + map = page_address(summary_info->page); + bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, + adapter->swap); + /* If setbit then set summary bit. Else if falling back to the slow path */ + /* with setbit==0 then clear the summary bit so the slow path re-injects */ + if (setbit) + summary_set = test_and_set_bit(bit, map); + else + summary_set = test_and_clear_bit(bit, map); + spin_unlock(&adapter->maps_lock); + return summary_set ? 0 : 1; +} + /* * < 0 - not injected due to error * = 0 - coalesced, summary indicator already active @@ -2981,6 +3048,8 @@ static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, int ret; struct s390_io_adapter *adapter; + kvm->stat.io_set_adapter_int++; + /* We're only interested in the 0->1 transition. */ if (!level) return 0; @@ -3049,7 +3118,6 @@ int kvm_set_routing_entry(struct kvm *kvm, int idx; switch (ue->type) { - /* we store the userspace addresses instead of the guest addresses */ case KVM_IRQ_ROUTING_S390_ADAPTER: if (kvm_is_ucontrol(kvm)) return -EINVAL; @@ -3640,3 +3708,86 @@ int __init kvm_s390_gib_init(u8 nisc) out: return rc; } + +/* + * kvm_arch_set_irq_inatomic: fast-path for irqfd injection + */ +int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, int level, + bool line_status) +{ + int ret, setbit; + struct s390_io_adapter *adapter; + struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + struct kvm_s390_interrupt_info *inti; + struct kvm_s390_interrupt s390int = { + .type = KVM_S390_INT_IO(1, 0, 0, 0), + .parm = 0, + }; + + kvm->stat.io_390_inatomic++; + + /* We're only interested in the 0->1 transition. */ + if (!level) + return 0; + if (e->type != KVM_IRQ_ROUTING_S390_ADAPTER) + return -EWOULDBLOCK; + + adapter = get_io_adapter(kvm, e->adapter.adapter_id); + if (!adapter) + return -EWOULDBLOCK; + + s390int.parm64 = isc_to_int_word(adapter->isc); + setbit = 1; + ret = adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + if (ret < 0) + return -EWOULDBLOCK; + if (!ret || adapter->masked) { + kvm->stat.io_390_inatomic_no_inject++; + return 0; + } + + inti = kzalloc_obj(*inti, GFP_ATOMIC); + if (!inti) { + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + return -EWOULDBLOCK; + } + + if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) { + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (ret == 0) { + return ret; + } else { + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + kfree(inti); + return -EWOULDBLOCK; + } + } + + spin_lock(&fi->ais_lock); + if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { + trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); + spin_unlock(&fi->ais_lock); + kfree(inti); + kvm->stat.io_390_inatomic_no_inject++; + return 0; + } + + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { + fi->nimm |= AIS_MODE_MASK(adapter->isc); + trace_kvm_s390_modify_ais_mode(adapter->isc, + KVM_S390_AIS_MODE_SINGLE, 2); + } else if (ret) { + spin_unlock(&fi->ais_lock); + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + kfree(inti); + return -EWOULDBLOCK; + } + + spin_unlock(&fi->ais_lock); + return 0; +} diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 9de6edeb5ae0..80b4bc497494 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -70,6 +70,10 @@ const struct kvm_stats_desc kvm_vm_stats_desc[] = { STATS_DESC_COUNTER(VM, inject_io), STATS_DESC_COUNTER(VM, io_390_adapter_map), STATS_DESC_COUNTER(VM, io_390_adapter_unmap), + STATS_DESC_COUNTER(VM, io_390_inatomic), + STATS_DESC_COUNTER(VM, io_flic_inject_airq), + STATS_DESC_COUNTER(VM, io_set_adapter_int), + STATS_DESC_COUNTER(VM, io_390_inatomic_no_inject), STATS_DESC_COUNTER(VM, inject_float_mchk), STATS_DESC_COUNTER(VM, inject_pfault_done), STATS_DESC_COUNTER(VM, inject_service_signal), @@ -2877,6 +2881,7 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) void __user *argp = (void __user *)arg; struct kvm_device_attr attr; int r; + struct kvm_s390_interrupt_info *inti; switch (ioctl) { case KVM_S390_INTERRUPT: { @@ -2885,7 +2890,12 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) r = -EFAULT; if (copy_from_user(&s390int, argp, sizeof(s390int))) break; - r = kvm_s390_inject_vm(kvm, &s390int); + inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); + if (!inti) + return -ENOMEM; + r = kvm_s390_inject_vm(kvm, &s390int, inti); + if (r) + kfree(inti); break; } case KVM_CREATE_IRQCHIP: { @@ -3286,7 +3296,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) mutex_unlock(&kvm->lock); } - mutex_init(&kvm->arch.float_int.ais_lock); + spin_lock_init(&kvm->arch.float_int.ais_lock); spin_lock_init(&kvm->arch.float_int.lock); for (i = 0; i < FIRQ_LIST_COUNT; i++) INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]); @@ -4408,19 +4418,28 @@ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clo } static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, - unsigned long token) + unsigned long token) { struct kvm_s390_interrupt inti; struct kvm_s390_irq irq; + struct kvm_s390_interrupt_info *inti_mem = NULL; + int ret = 0; if (start_token) { irq.u.ext.ext_params2 = token; irq.type = KVM_S390_INT_PFAULT_INIT; WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq)); } else { + inti_mem = kzalloc_obj(*inti_mem, GFP_KERNEL_ACCOUNT); + if (WARN_ON_ONCE(!inti_mem)) + return; + inti.type = KVM_S390_INT_PFAULT_DONE; inti.parm64 = token; - WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti)); + ret = kvm_s390_inject_vm(vcpu->kvm, &inti, inti_mem); + if (ret) + kfree(inti_mem); + WARN_ON_ONCE(ret); } } diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index 7ba885cb6bd1..6d2842fb71a3 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -376,7 +376,8 @@ int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu); void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu); void kvm_s390_clear_float_irqs(struct kvm *kvm); int __must_check kvm_s390_inject_vm(struct kvm *kvm, - struct kvm_s390_interrupt *s390int); + struct kvm_s390_interrupt *s390int, + struct kvm_s390_interrupt_info *inti); int __must_check kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq); static inline int kvm_s390_inject_prog_irq(struct kvm_vcpu *vcpu,