From 499d2e4b75c44365a40a5fe623f3bd5bc1844197 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 27 May 2026 16:43:54 +0200 Subject: [PATCH 01/17] KVM: s390: Track page size in struct guest_fault Until now, the members of struct guest_fault are always accessed while holding the required locks, and thus the ptep and crstep pointers can be dereferenced safely. There will be some new cases where callers of kvm_s390_faultin_gfn() need to know the size of the page used to solve the fault, at which point no locks are held anymore, and dereferencing the crstep field is not possible. Introduce a new crste_region3 flag for struct guest_fault to indicate whether the crstep used to solve the fault was a region 3 entry with FC=1 (large pud). This allows to disambiguate all three possible scenarios: * If ptep is not NULL, the fault was solved with a pte. * If ptep is NULL and crste_region3 is 0, a segment entry with FC=1 (large pmd) was used. * If ptep is NULL and crste_region3 is 1, a region 3 entry with FC=1 (large pud) was used. Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda Message-ID: <20260527144358.186359-2-imbrenda@linux.ibm.com> --- arch/s390/kvm/dat.h | 1 + arch/s390/kvm/gmap.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h index 8f8278c44879..5d848e74e382 100644 --- a/arch/s390/kvm/dat.h +++ b/arch/s390/kvm/dat.h @@ -500,6 +500,7 @@ struct guest_fault { bool write_attempt; /* Write access attempted */ bool attempt_pfault; /* Attempt a pfault first */ bool valid; /* This entry contains valid data */ + bool crste_region3; /* Whether crstep refers to a region3 entry */ void (*callback)(struct guest_fault *f); void *priv; }; diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index 3c26e35af0ef..fe138d17caaf 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -531,6 +531,7 @@ static int gmap_handle_minor_crste_fault(struct gmap *gmap, struct guest_fault * f->pfn = PHYS_PFN(large_crste_to_phys(oldcrste, f->gfn)); f->writable = oldcrste.s.fc1.w; + f->crste_region3 = is_pud(oldcrste); /* Appropriate permissions already (race with another handler), nothing to do. */ if (!oldcrste.h.i && !(f->write_attempt && oldcrste.h.p)) return 0; @@ -690,6 +691,7 @@ static int _gmap_link(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, int leve if (oldval.val != _CRSTE_EMPTY(oldval.h.tt).val && crste_origin_large(oldval) != crste_origin_large(newval)) return -EAGAIN; + f->crste_region3 = is_pud(newval); } while (!gmap_crstep_xchg_atomic(gmap, f->crstep, oldval, newval, f->gfn)); if (f->callback) f->callback(f); From bea77e83c0b9e1d98482a0c24f9ee9eb8dd61edc Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 27 May 2026 16:43:55 +0200 Subject: [PATCH 02/17] KVM: s390: Implement KVM_PRE_FAULT_MEMORY Implement and enable the KVM_PRE_FAULT_MEMORY ioctl for s390. Faulted-in pages will be marked as accessed, unlike x86, otherwise they will trigger a minor fault when accessed. Avoiding such faults is one of the points of KVM_PRE_FAULT_MEMORY. Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda Message-ID: <20260527144358.186359-3-imbrenda@linux.ibm.com> --- arch/s390/kvm/Kconfig | 1 + arch/s390/kvm/kvm-s390.c | 45 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index 5b835bc6a194..8d3ee17a1bcb 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -30,6 +30,7 @@ config KVM select KVM_VFIO select VIRT_XFER_TO_GUEST_WORK select KVM_MMU_LOCKLESS_AGING + select KVM_GENERIC_PRE_FAULT_MEMORY help Support hosting paravirtualized guest machines using the SIE virtualization capability on the mainframe. This should work diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index e09960c2e6ed..f6521f16532a 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -630,6 +630,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_S390_USER_OPEREXEC: case KVM_CAP_S390_KEYOP: case KVM_CAP_S390_VSIE_ESAMODE: + case KVM_CAP_PRE_FAULT_MEMORY: r = 1; break; case KVM_CAP_SET_GUEST_DEBUG2: @@ -5736,6 +5737,50 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, return; } +/** + * kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables + * @vcpu: the vcpu that shall appear to have generated the fault-in. + * @range: the range that needs to be faulted in. + * + * The first page of the given range is faulted in and the corresponding gmap + * page tables are created, as if the given vCPU had performed a read + * operation. + * If the range starts outside any memslots, an error is returned. An error is + * also returned for UCONTROL VMs, which should instead use the + * KVM_S390_VCPU_FAULT ioctl. + * + * Return: + * * %-ENOENT if the range lies outside of a memslot. + * * %-EINVAL in case of invalid state (for example if the VM is UCONTROL). + * * %-EIO if errors happen while faulting-in the page (will trigger a warning + * in the caller). + * * other error codes < 0 in case of other errors. + * * otherwise a number > 0 of bytes that have been faulted in successfully. + */ +long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) +{ + struct guest_fault f = { .gfn = gpa_to_gfn(range->gpa), }; + gpa_t end; + int rc; + + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; + + rc = kvm_s390_faultin_gfn(vcpu, NULL, &f); + if (rc == PGM_ADDRESSING) + return -ENOENT; + if (rc > 0) + return -EIO; + if (rc < 0) + return rc; + + if (f.ptep) + return PAGE_SIZE; + + end = ALIGN(range->gpa + PAGE_SIZE, f.crste_region3 ? _REGION3_SIZE : HPAGE_SIZE); + return min(range->size, end - range->gpa); +} + /** * kvm_test_age_gfn() - test young * @kvm: the kvm instance From a5aa761e73074266fef95fd3ee2ff75ea0372f16 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 27 May 2026 16:43:56 +0200 Subject: [PATCH 03/17] KVM: s390: Update KVM_PRE_FAULT_MEMORY API documentation Update the API documentation for KVM_PRE_FAULT_MEMORY to account for its s390 implementation. Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda Message-ID: <20260527144358.186359-4-imbrenda@linux.ibm.com> --- Documentation/virt/kvm/api.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 52bbbb553ce1..e7998feaa940 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6471,7 +6471,8 @@ Errors: ========== =============================================================== EINVAL The specified `gpa` and `size` were invalid (e.g. not - page aligned, causes an overflow, or size is zero). + page aligned, causes an overflow, or size is zero), or the VM + is UCONTROL (s390). ENOENT The specified `gpa` is outside defined memslots. EINTR An unmasked signal is pending and no page was processed. EFAULT The parameter address was invalid. @@ -6494,7 +6495,7 @@ Errors: KVM_PRE_FAULT_MEMORY populates KVM's stage-2 page tables used to map memory for the current vCPU state. KVM maps memory as if the vCPU generated a stage-2 read page fault, e.g. faults in memory as needed, but doesn't break -CoW. However, KVM does not mark any newly created stage-2 PTE as Accessed. +CoW. On x86, KVM does not mark any newly created stage-2 PTE as Accessed. In the case of confidential VM types where there is an initial set up of private guest memory before the guest is 'finalized'/measured, this ioctl From c847704619da45d5a161887afcfc3701ae18f971 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 27 May 2026 16:43:57 +0200 Subject: [PATCH 04/17] KVM: selftests: Fix pre_fault_memory_test to run on s390 Add a missing #include which is needed and otherwise not included on s390. Remove the assertion vcpu->run->exit_reason == KVM_EXIT_IO since it is x86-specific and redundant anyway. Acked-by: Sean Christopherson Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda Message-ID: <20260527144358.186359-5-imbrenda@linux.ibm.com> --- tools/testing/selftests/kvm/pre_fault_memory_test.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/testing/selftests/kvm/pre_fault_memory_test.c b/tools/testing/selftests/kvm/pre_fault_memory_test.c index fcb57fd034e6..a0fcae3cb7a8 100644 --- a/tools/testing/selftests/kvm/pre_fault_memory_test.c +++ b/tools/testing/selftests/kvm/pre_fault_memory_test.c @@ -11,6 +11,7 @@ #include #include #include +#include /* Arbitrarily chosen values */ #define TEST_SIZE (SZ_2M + PAGE_SIZE) @@ -167,7 +168,6 @@ static void __test_pre_fault_memory(unsigned long vm_type, bool private) .type = vm_type, }; struct kvm_vcpu *vcpu; - struct kvm_run *run; struct kvm_vm *vm; struct ucall uc; @@ -193,11 +193,6 @@ static void __test_pre_fault_memory(unsigned long vm_type, bool private) vcpu_args_set(vcpu, 1, gva); vcpu_run(vcpu); - run = vcpu->run; - TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, - "Wanted KVM_EXIT_IO, got exit reason: %u (%s)", - run->exit_reason, exit_reason_str(run->exit_reason)); - switch (get_ucall(vcpu, &uc)) { case UCALL_ABORT: REPORT_GUEST_ASSERT(uc); From 68f8c9565c4391343aa3b77f7af642a366991405 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 27 May 2026 16:43:58 +0200 Subject: [PATCH 05/17] KVM: selftests: Enable pre_fault_memory_test for s390 Enable the pre_fault_memory_test to run on s390. Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda Message-ID: <20260527144358.186359-6-imbrenda@linux.ibm.com> --- tools/testing/selftests/kvm/Makefile.kvm | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 9118a5a51b89..fff939db89cd 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -210,6 +210,7 @@ TEST_GEN_PROGS_s390 += s390/keyop TEST_GEN_PROGS_s390 += rseq_test TEST_GEN_PROGS_s390 += s390/irq_routing TEST_GEN_PROGS_s390 += mmu_stress_test +TEST_GEN_PROGS_s390 += pre_fault_memory_test TEST_GEN_PROGS_riscv = $(TEST_GEN_PROGS_COMMON) TEST_GEN_PROGS_riscv += riscv/sbi_pmu_test From 782fec4ab945e1f3b9f0ba383fc00c67641d9767 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Tue, 9 Jun 2026 17:09:27 +0200 Subject: [PATCH 06/17] KVM: s390: Add module parameter to fence 2G hugepages Add the hpage_2g module parameter to KVM to allow enabling or disabling 2G hugepages in KVM. If hpage_2g is enabled but hpage is not enabled, print a message and disable hpage_2g. Opportunistically fix the comment for the hpage module parameter. Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda Message-ID: <20260609150930.665370-2-imbrenda@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index f6521f16532a..abc7941d7bb4 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -204,11 +204,16 @@ static int nested; module_param(nested, int, S_IRUGO); MODULE_PARM_DESC(nested, "Nested virtualization support"); -/* allow 1m huge page guest backing, if !nested */ +/* allow 1m huge page guest backing */ static int hpage; module_param(hpage, int, 0444); MODULE_PARM_DESC(hpage, "1m huge page backing support"); +/* allow 2g huge page guest backing */ +static int hpage_2g; +module_param(hpage_2g, int, 0444); +MODULE_PARM_DESC(hpage_2g, "2g huge page backing support"); + /* maximum percentage of steal time for polling. >100 is treated like 100 */ static u8 halt_poll_max_steal = 10; module_param(halt_poll_max_steal, byte, 0644); @@ -5842,6 +5847,11 @@ static int __init kvm_s390_init(void) return -ENODEV; } + if (hpage_2g && !hpage) { + hpage_2g = 0; + pr_info("Disabling 2G hugepage support, since 1M hugepage support is not enabled.\n"); + } + for (i = 0; i < 16; i++) kvm_s390_fac_base[i] |= stfle_fac_list[i] & nonhyp_mask(i); From 103ff3a50e3a50a97e2b123d4dccbc8665eb1552 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Tue, 9 Jun 2026 17:09:28 +0200 Subject: [PATCH 07/17] KVM: s390: Add capability to support 2G hugepages Add KVM_CAP_S390_HPAGE_2G to signal to userspace that 2G hugepages may be used to back the guest; restrictions apply similar to 1M hugepages. Enable the (for now still ignored) GMAP_FLAG_ALLOW_HPAGE_2G flag for the guest gmap, and propagate / disable it as necessary. Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda Message-ID: <20260609150930.665370-3-imbrenda@linux.ibm.com> --- arch/s390/kvm/gmap.c | 5 +++++ arch/s390/kvm/kvm-s390.c | 26 ++++++++++++++++++++++++++ arch/s390/kvm/pv.c | 5 ++++- include/uapi/linux/kvm.h | 1 + 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index fe138d17caaf..f8bd6c135e3c 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -105,6 +105,11 @@ static void gmap_add_child(struct gmap *parent, struct gmap *child) else clear_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &child->flags); + if (test_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &parent->flags)) + set_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &child->flags); + else + clear_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &child->flags); + if (kvm_is_ucontrol(parent->kvm)) clear_bit(GMAP_FLAG_OWNS_PAGETABLES, &child->flags); list_add(&child->list, &parent->children); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index abc7941d7bb4..6de36421548c 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -646,6 +646,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) if (hpage && !(kvm && kvm_is_ucontrol(kvm))) r = 1; break; + case KVM_CAP_S390_HPAGE_2G: + r = 0; + if (hpage_2g && !(kvm && kvm_is_ucontrol(kvm))) + r = 1; + break; case KVM_CAP_S390_MEM_OP: r = MEM_OP_MAX_SIZE; break; @@ -902,6 +907,27 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s", r ? "(not available)" : "(success)"); break; + case KVM_CAP_S390_HPAGE_2G: + mutex_lock(&kvm->lock); + if (kvm->created_vcpus) { + r = -EBUSY; + } else if (!hpage_2g || kvm->arch.use_cmma || kvm_is_ucontrol(kvm)) { + r = -EINVAL; + } else { + r = 0; + set_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &kvm->arch.gmap->flags); + /* + * We might have to create fake 4k page + * tables. To avoid that the hardware works on + * stale PGSTEs, we emulate these instructions. + */ + kvm->arch.use_skf = 0; + kvm->arch.use_pfmfi = 0; + } + mutex_unlock(&kvm->lock); + VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE_2G %s", + r ? "(not available)" : "(success)"); + break; case KVM_CAP_S390_USER_STSI: VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI"); kvm->arch.user_stsi = 1; diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c index c2dafd812a3b..b9a23df96f7d 100644 --- a/arch/s390/kvm/pv.c +++ b/arch/s390/kvm/pv.c @@ -721,7 +721,10 @@ int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc) uvcb.flags.ap_allow_instr = kvm->arch.model.uv_feat_guest.ap; uvcb.flags.ap_instr_intr = kvm->arch.model.uv_feat_guest.ap_intr; - clear_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags); + scoped_guard(write_lock, &kvm->mmu_lock) { + clear_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags); + clear_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &kvm->arch.gmap->flags); + } gmap_split_huge_pages(kvm->arch.gmap); cc = uv_call_sched(0, (u64)&uvcb); diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 6c8afa2047bf..419011097fa8 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -996,6 +996,7 @@ struct kvm_enable_cap { #define KVM_CAP_S390_USER_OPEREXEC 246 #define KVM_CAP_S390_KEYOP 247 #define KVM_CAP_S390_VSIE_ESAMODE 248 +#define KVM_CAP_S390_HPAGE_2G 249 struct kvm_irq_routing_irqchip { __u32 irqchip; From 46756c086310057c92cf17c8d07e0c63a7ba687d Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Tue, 9 Jun 2026 17:09:29 +0200 Subject: [PATCH 08/17] KVM: s390: Allow for 2G hugepages Change gmap_2g_allowed() to perform the necessary checks to allow for 2G hugepages to be used, instead of returning false. The GMAP_FLAG_ALLOW_HPAGE_2G gmap flag is now taken into account. Also add appropriate kerneldoc comments. Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda Message-ID: <20260609150930.665370-4-imbrenda@linux.ibm.com> --- arch/s390/kvm/gmap.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index f8bd6c135e3c..402573cb2590 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -624,10 +624,27 @@ int gmap_try_fixup_minor(struct gmap *gmap, struct guest_fault *fault) return rc; } +/** + * gmap_2g_allowed() - Check whether a 2G hugepage is allowed. + * @gmap: The gmap of the guest. + * @f: Describes the fault that is being resolved. + * @slot: The memslot the faulting address belongs to. + * + * The function checks whether the GMAP_FLAG_ALLOW_HPAGE_2G flag is set for + * @gmap, whether the offset of the address in the 2G virtual frame is the + * same as the offset in the physical 2G frame, and finally whether the whole + * 2G page would fit in the given memslot. + * + * Return: true if a 2G hugepage is allowed to back the faulting address, false + * otherwise. + */ static inline bool gmap_2g_allowed(struct gmap *gmap, struct guest_fault *f, struct kvm_memory_slot *slot) { - return false; + return test_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &gmap->flags) && + !((f->gfn ^ f->pfn) & ~_REGION3_FR_MASK) && + slot->base_gfn <= ALIGN_DOWN(f->gfn, _PAGES_PER_REGION3) && + slot->base_gfn + slot->npages >= ALIGN(f->gfn + 1, _PAGES_PER_REGION3); } /** From 9030da050c9fe7d10150c9f93765e464f6c7da3c Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Tue, 9 Jun 2026 17:09:30 +0200 Subject: [PATCH 09/17] KVM: s390: Document the KVM_CAP_S390_HPAGE_2G capability Document the KVM_CAP_S390_HPAGE_2G capability, which behaves very similarly to the existing KVM_CAP_S390_HPAGE_1M. Signed-off-by: Claudio Imbrenda Message-ID: <20260609150930.665370-5-imbrenda@linux.ibm.com> --- Documentation/virt/kvm/api.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e7998feaa940..dfde0bfcbce6 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -8905,6 +8905,21 @@ helpful if user space wants to emulate instructions which are not This capability can be enabled dynamically even if VCPUs were already created and are running. +7.47 KVM_CAP_S390_HPAGE_2G +-------------------------- + +:Architectures: s390 +:Parameters: none +:Returns: 0 on success; -EINVAL if hpage_2g module parameter was not set, + cmma is enabled, or the VM has the KVM_VM_S390_UCONTROL + flag set; -EBUSY if vCPUs were already created for the VM. + +With this capability the KVM support for memory backing with 2g pages +through hugetlbfs can be enabled for a VM. After the capability is +enabled, cmma can't be enabled anymore and pfmfi and the storage key +interpretation are disabled. If cmma has already been enabled or the +hpage_2g module parameter is not set to 1, -EINVAL is returned. + 8. Other capabilities. ====================== From c7dda3d0f869dc97223448a06c9a2e5235928e48 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Thu, 11 Jun 2026 12:50:36 +0200 Subject: [PATCH 10/17] KVM: s390: Initialize KVM_S390_GET_CMMA_BITS memory kvm_s390_get_cmma_bits() allocates its output buffer with vmalloc(), which does not zero the returned pages: values = vmalloc(args->count); In the non-peek (migration) path, dat_get_cmma() reports a byte count spanning from the first to the last dirty page, but __dat_get_cmma_pte() writes values[gfn - start] only for pages whose CMMA dirty bit is set. The walk uses DAT_WALK_IGN_HOLES, so clean and unmapped pages that lie between two dirty pages within the reported span are visited but never store their byte. Those gaps (up to KVM_S390_MAX_BIT_DISTANCE pages each) stay uninitialized yet fall inside [0, count) and are copied out by copy_to_user(), disclosing stale kernel memory to user space. Before the switch to the new gmap implementation the buffer was fully populated for every gfn in the span, so no uninitialized bytes were exposed; the dirty-only walk introduced the leak. Use vzalloc() so the gaps read back as zero. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260611105036.11491-1-borntraeger@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 6de36421548c..d503e4d0072a 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -2263,7 +2263,7 @@ static int kvm_s390_get_cmma_bits(struct kvm *kvm, return 0; } - values = vmalloc(args->count); + values = vzalloc(args->count); if (!values) return -ENOMEM; From 20dcfaf2d3261ef84631114ae37213f5ebd3eb37 Mon Sep 17 00:00:00 2001 From: Nina Schoetterl-Glausch Date: Fri, 12 Jun 2026 14:23:01 +0200 Subject: [PATCH 11/17] KVM: s390: Minor refactor of base/ext facility lists Directly use the size of the arrays instead of going through the indirection of kvm_s390_fac_size(). Don't use magic number for the number of entries in the non hypervisor managed facility bit mask list. Make the constraint of that number on kvm_s390_fac_base obvious. Get rid of implicit double anding of stfle_fac_list. Signed-off-by: Nina Schoetterl-Glausch Co-developed-by: Christoph Schlameuss Signed-off-by: Christoph Schlameuss Reviewed-by: Janosch Frank Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260612-vsie-alter-stfle-fac-v4-1-74f0e1559929@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 44 +++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index d503e4d0072a..7334c160d019 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -237,33 +237,25 @@ static int async_destroy = 1; module_param(async_destroy, int, 0444); MODULE_PARM_DESC(async_destroy, "Asynchronous destroy for protected guests"); -/* - * For now we handle at most 16 double words as this is what the s390 base - * kernel handles and stores in the prefix page. If we ever need to go beyond - * this, this requires changes to code, but the external uapi can stay. - */ -#define SIZE_INTERNAL 16 - +#define HMFAI_DWORDS 16 /* * Base feature mask that defines default mask for facilities. Consists of the * defines in FACILITIES_KVM and the non-hypervisor managed bits. */ -static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM }; +static unsigned long kvm_s390_fac_base[HMFAI_DWORDS] = { FACILITIES_KVM }; +static_assert(ARRAY_SIZE(((long[]){ FACILITIES_KVM })) <= HMFAI_DWORDS); +static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_MASK_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_LIST_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= ARRAY_SIZE(stfle_fac_list)); + /* * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL * and defines the facilities that can be enabled via a cpu model. */ -static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL }; - -static unsigned long kvm_s390_fac_size(void) -{ - BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64); - BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64); - BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) > - sizeof(stfle_fac_list)); - - return SIZE_INTERNAL; -} +static const unsigned long kvm_s390_fac_ext[] = { FACILITIES_KVM_CPUMODEL }; +static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_MASK_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_LIST_SIZE_U64); +static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= ARRAY_SIZE(stfle_fac_list)); /* available cpu features supported by kvm */ static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); @@ -3249,13 +3241,16 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm->arch.sie_page2->kvm = kvm; kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list; - for (i = 0; i < kvm_s390_fac_size(); i++) { + for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_base); i++) { kvm->arch.model.fac_mask[i] = stfle_fac_list[i] & - (kvm_s390_fac_base[i] | - kvm_s390_fac_ext[i]); + kvm_s390_fac_base[i]; kvm->arch.model.fac_list[i] = stfle_fac_list[i] & kvm_s390_fac_base[i]; } + for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_ext); i++) { + kvm->arch.model.fac_mask[i] |= stfle_fac_list[i] & + kvm_s390_fac_ext[i]; + } kvm->arch.model.subfuncs = kvm_s390_available_subfunc; /* we are always in czam mode - even on pre z14 machines */ @@ -5878,9 +5873,8 @@ static int __init kvm_s390_init(void) pr_info("Disabling 2G hugepage support, since 1M hugepage support is not enabled.\n"); } - for (i = 0; i < 16; i++) - kvm_s390_fac_base[i] |= - stfle_fac_list[i] & nonhyp_mask(i); + for (i = 0; i < HMFAI_DWORDS; i++) + kvm_s390_fac_base[i] |= nonhyp_mask(i); r = __kvm_s390_init(); if (r) From a8ceca7d8c00479a650bc5fb1372d40dd5e535a2 Mon Sep 17 00:00:00 2001 From: Nina Schoetterl-Glausch Date: Fri, 12 Jun 2026 14:23:02 +0200 Subject: [PATCH 12/17] s390/sclp: Detect ASTFLEIE 2 facility Detect alternate STFLE interpretive execution facility 2. Signed-off-by: Nina Schoetterl-Glausch Signed-off-by: Christoph Schlameuss Reviewed-by: Janosch Frank Reviewed-by: Hendrik Brueckner Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260612-vsie-alter-stfle-fac-v4-2-74f0e1559929@linux.ibm.com> --- arch/s390/include/asm/sclp.h | 1 + drivers/s390/char/sclp_early.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 0f184dbdbe5e..0f21501d3e86 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -104,6 +104,7 @@ struct sclp_info { unsigned char has_aisii : 1; unsigned char has_aeni : 1; unsigned char has_aisi : 1; + unsigned char has_astfleie2 : 1; unsigned int ibc; unsigned int mtid; unsigned int mtid_cp; diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index 6bf501ad8ff0..22dd797e6229 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -61,8 +61,10 @@ static void __init sclp_early_facilities_detect(void) sclp.has_sipl = !!(sccb->cbl & 0x4000); sclp.has_sipl_eckd = !!(sccb->cbl & 0x2000); } - if (sccb->cpuoff > 139) + if (sccb->cpuoff > 139) { sclp.has_diag324 = !!(sccb->byte_139 & 0x80); + sclp.has_astfleie2 = !!(sccb->byte_139 & 0x40); + } sclp.rnmax = sccb->rnmax ? sccb->rnmax : sccb->rnmax2; sclp.rzm = sccb->rnsize ? sccb->rnsize : sccb->rnsize2; sclp.rzm <<= 20; From bf8f3cec939db68e6dc32705327acfabdbcf9e69 Mon Sep 17 00:00:00 2001 From: Nina Schoetterl-Glausch Date: Fri, 12 Jun 2026 14:23:03 +0200 Subject: [PATCH 13/17] KVM: s390: vsie: Refactor handle_stfle Use switch case in anticipation of handling format-1 and format-2 facility list designations in the future. As the alternate STFLE facilities are not enabled, only case 0 is possible. No functional change intended. Signed-off-by: Nina Schoetterl-Glausch Co-developed-by: Christoph Schlameuss Signed-off-by: Christoph Schlameuss Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260612-vsie-alter-stfle-fac-v4-3-74f0e1559929@linux.ibm.com> --- arch/s390/include/uapi/asm/kvm.h | 1 + arch/s390/kvm/vsie.c | 53 ++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/arch/s390/include/uapi/asm/kvm.h b/arch/s390/include/uapi/asm/kvm.h index 60345dd2cba2..4192769b5ce0 100644 --- a/arch/s390/include/uapi/asm/kvm.h +++ b/arch/s390/include/uapi/asm/kvm.h @@ -444,6 +444,7 @@ struct kvm_s390_vm_cpu_machine { #define KVM_S390_VM_CPU_FEAT_PFMFI 11 #define KVM_S390_VM_CPU_FEAT_SIGPIF 12 #define KVM_S390_VM_CPU_FEAT_KSS 13 +#define KVM_S390_VM_CPU_FEAT_ASTFLEIE2 14 struct kvm_s390_vm_cpu_feat { __u64 feat[16]; }; diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index e5a23f1c9749..c7dcdd460dd1 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -6,12 +6,15 @@ * * Author(s): David Hildenbrand */ +#include #include #include #include +#include #include #include #include +#include #include #include @@ -1000,6 +1003,23 @@ static void retry_vsie_icpt(struct vsie_page *vsie_page) clear_vsie_icpt(vsie_page); } +static int handle_stfle_0(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, + u32 fac_list_origin) +{ + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; + + /* + * format-0 -> size of nested guest's facility list == guest's size + * guest's size == host's size, since STFLE is interpretatively executed + * using a format-0 for the guest, too. + */ + if (read_guest_real(vcpu, fac_list_origin, &vsie_page->fac, + stfle_size() * sizeof(u64))) + return set_validity_icpt(scb_s, 0x1090U); + scb_s->fac = (u32)virt_to_phys(&vsie_page->fac); + return 0; +} + /* * Try to shadow + enable the guest 2 provided facility list. * Retry instruction execution if enabled for and provided by guest 2. @@ -1009,29 +1029,30 @@ static void retry_vsie_icpt(struct vsie_page *vsie_page) */ static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) { - struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; - __u32 fac = READ_ONCE(vsie_page->scb_o->fac); + bool has_astfleie2 = test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_ASTFLEIE2); + u32 fac = READ_ONCE(vsie_page->scb_o->fac); + int format_mask, format; + u32 origin; + + BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct vsie_page, fac), 8)); - /* - * Alternate-STFLE-Interpretive-Execution facilities are not supported - * -> format-0 flcb - */ if (fac && test_kvm_facility(vcpu->kvm, 7)) { retry_vsie_icpt(vsie_page); /* * The facility list origin (FLO) is in bits 1 - 28 of the FLD * so we need to mask here before reading. */ - fac = fac & 0x7ffffff8U; - /* - * format-0 -> size of nested guest's facility list == guest's size - * guest's size == host's size, since STFLE is interpretatively executed - * using a format-0 for the guest, too. - */ - if (read_guest_real(vcpu, fac, &vsie_page->fac, - stfle_size() * sizeof(u64))) - return set_validity_icpt(scb_s, 0x1090U); - scb_s->fac = (u32)virt_to_phys(&vsie_page->fac); + origin = fac & 0x7ffffff8U; + format_mask = has_astfleie2 ? 3 : 0; + format = fac & format_mask; + switch (format) { + case 0: + return handle_stfle_0(vcpu, vsie_page, origin); + case 1: + case 2: + case 3: + unreachable(); + } } return 0; } From 0e9704f1bca006ea09cab4a73dbcbeec8c77573e Mon Sep 17 00:00:00 2001 From: Nina Schoetterl-Glausch Date: Fri, 12 Jun 2026 14:23:04 +0200 Subject: [PATCH 14/17] KVM: s390: vsie: Implement ASTFLEIE facility 2 Implement shadowing of format-2 facility list when running in VSIE. ASTFLEIE2 is available since IBM z16. To function G1 has to run this KVM code and G1 and G2 have to run QEMU with ASTFLEIE2 support. Signed-off-by: Nina Schoetterl-Glausch Co-developed-by: Christoph Schlameuss Signed-off-by: Christoph Schlameuss Reviewed-by: Claudio Imbrenda [imbrenda@linux.ibm.com: Fix typo in comment] Signed-off-by: Claudio Imbrenda Message-ID: <20260612-vsie-alter-stfle-fac-v4-4-74f0e1559929@linux.ibm.com> --- arch/s390/include/asm/kvm_host.h | 12 +++++++++++ arch/s390/kvm/kvm-s390.c | 2 ++ arch/s390/kvm/vsie.c | 34 ++++++++++++++++++++++++++++---- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 8a4f4a39f7a2..aa4c4685f95c 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -504,6 +504,18 @@ struct kvm_s390_cpu_model { struct kvm_s390_vm_cpu_uv_feat uv_feat_guest; }; +#define S390_ARCH_FAC_FORMAT_2 2 +struct kvm_s390_flcb2 { + union { + struct { + u8 reserved0[7]; + u8 length; + }; + u64 header_val; + }; + u64 facilities[S390_ARCH_FAC_LIST_SIZE_U64]; +}; + typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); struct kvm_s390_crypto { diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 7334c160d019..de28ee1f7882 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -465,6 +465,8 @@ static void __init kvm_s390_cpu_feat_init(void) allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS); if (sclp.has_kss) allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS); + if (sclp.has_astfleie2) + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ASTFLEIE2); /* * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make * all skey handling functions read/set the skey from the PGSTE diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index c7dcdd460dd1..eea24562e7db 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -65,9 +65,9 @@ struct vsie_page { gpa_t scb_gpa; /* 0x0258 */ /* the shadow gmap in use by the vsie_page */ struct gmap_cache gmap_cache; /* 0x0260 */ - __u8 reserved[0x0700 - 0x0278]; /* 0x0278 */ - struct kvm_s390_crypto_cb crycb; /* 0x0700 */ - __u8 fac[S390_ARCH_FAC_LIST_SIZE_BYTE]; /* 0x0800 */ + __u8 reserved[0x06f8 - 0x0278]; /* 0x0278 */ + struct kvm_s390_crypto_cb crycb; /* 0x06f8 */ + __u8 fac[8 + S390_ARCH_FAC_LIST_SIZE_BYTE];/* 0x07f8 */ }; static_assert(sizeof(struct vsie_page) == PAGE_SIZE); @@ -1020,6 +1020,28 @@ static int handle_stfle_0(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, return 0; } +static int handle_stfle_2(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page, u32 fac_list_origin) +{ + struct kvm_s390_flcb2 *flcb_s = (struct kvm_s390_flcb2 *)vsie_page->fac; + struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s; + u64 len; + + if (read_guest_real(vcpu, fac_list_origin, &len, sizeof(len))) + return set_validity_icpt(scb_s, 0x1090U); + + /* discard reserved bits */ + len = (len & U8_MAX); + flcb_s->header_val = len; + len += 1; + + if (read_guest_real(vcpu, fac_list_origin + offsetof(struct kvm_s390_flcb2, facilities), + &flcb_s->facilities, len * sizeof(u64))) + return set_validity_icpt(scb_s, 0x1090U); + + scb_s->fac = (u32)virt_to_phys(&vsie_page->fac) | S390_ARCH_FAC_FORMAT_2; + return 0; +} + /* * Try to shadow + enable the guest 2 provided facility list. * Retry instruction execution if enabled for and provided by guest 2. @@ -1034,6 +1056,8 @@ static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) int format_mask, format; u32 origin; + /* assert no overflow with maximum len */ + BUILD_BUG_ON(sizeof(vsie_page->fac) < ((S390_ARCH_FAC_LIST_SIZE_U64 + 1) * sizeof(u64))); BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct vsie_page, fac), 8)); if (fac && test_kvm_facility(vcpu->kvm, 7)) { @@ -1049,9 +1073,11 @@ static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) case 0: return handle_stfle_0(vcpu, vsie_page, origin); case 1: + return set_validity_icpt(&vsie_page->scb_s, 0x1330U); case 2: + return handle_stfle_2(vcpu, vsie_page, origin); case 3: - unreachable(); + return set_validity_icpt(&vsie_page->scb_s, 0x1330U); } } return 0; From c9a5688380865b968f7aef6534de632857ab5be0 Mon Sep 17 00:00:00 2001 From: Douglas Freimuth Date: Thu, 4 Jun 2026 21:27:53 +0200 Subject: [PATCH 15/17] KVM: s390: Add map/unmap ioctl and clean mappings post-guest s390 needs map/unmap ioctls, which map the adapter set indicator pages, so the pages can be accessed when interrupts are disabled. The mappings are cleaned up when the guest is removed. pin_user_pages_remote is used for both the ioctl as well as the pin-on-demand logic in adapter_indicators_set(). Map/Unmap ioctls are fenced in order to avoid the longterm pinning in Secure Execution environments. In Secure Execution environments the path of execution available before this patch is followed. Statistical counters to count map/unmap functions for adapter indicator pages are added. The counters can be used to analyze map/unmap functions in non-Secure Execution environments and similarly can be used to analyze Secure Execution environments where the counters will not be incremented as the adapter indicator pages are not mapped. Reviewed-by: Matthew Rosato Signed-off-by: Douglas Freimuth Acked-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260604192755.203143-2-freimuth@linux.ibm.com> --- arch/s390/include/asm/kvm_host.h | 5 + arch/s390/kvm/interrupt.c | 227 +++++++++++++++++++++++++------ arch/s390/kvm/kvm-s390.c | 3 + arch/s390/kvm/kvm-s390.h | 2 + 4 files changed, 194 insertions(+), 43 deletions(-) diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index aa4c4685f95c..4ff1ede90873 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -448,6 +448,8 @@ struct kvm_vcpu_arch { struct kvm_vm_stat { struct kvm_vm_stat_generic generic; u64 inject_io; + u64 io_390_adapter_map; + u64 io_390_adapter_unmap; u64 inject_float_mchk; u64 inject_pfault_done; u64 inject_service_signal; @@ -479,6 +481,9 @@ struct s390_io_adapter { bool masked; bool swap; bool suppressible; + spinlock_t maps_lock; + struct list_head maps; + unsigned int nr_maps; }; #define MAX_S390_IO_ADAPTERS ((MAX_ISC + 1) * 8) diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 07f59c3b9a7b..d11dea4cb0d3 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -2411,34 +2411,46 @@ static int register_io_adapter(struct kvm_device *dev, { struct s390_io_adapter *adapter; struct kvm_s390_io_adapter adapter_info; + int rc = 0; + mutex_lock(&dev->kvm->lock); if (copy_from_user(&adapter_info, - (void __user *)attr->addr, sizeof(adapter_info))) - return -EFAULT; - - if (adapter_info.id >= MAX_S390_IO_ADAPTERS) - return -EINVAL; - + (void __user *)attr->addr, sizeof(adapter_info))) { + rc = -EFAULT; + goto out; + } + if (adapter_info.id >= MAX_S390_IO_ADAPTERS) { + rc = -EINVAL; + goto out; + } adapter_info.id = array_index_nospec(adapter_info.id, MAX_S390_IO_ADAPTERS); - if (dev->kvm->arch.adapters[adapter_info.id] != NULL) - return -EINVAL; - + if (dev->kvm->arch.adapters[adapter_info.id] != NULL) { + rc = -EINVAL; + goto out; + } adapter = kzalloc_obj(*adapter, GFP_KERNEL_ACCOUNT); - if (!adapter) - return -ENOMEM; + if (!adapter) { + rc = -ENOMEM; + goto out; + } + INIT_LIST_HEAD(&adapter->maps); + spin_lock_init(&adapter->maps_lock); + adapter->nr_maps = 0; adapter->id = adapter_info.id; adapter->isc = adapter_info.isc; adapter->maskable = adapter_info.maskable; adapter->masked = false; adapter->swap = adapter_info.swap; - adapter->suppressible = (adapter_info.flags) & + adapter->suppressible = adapter_info.flags & KVM_S390_ADAPTER_SUPPRESSIBLE; dev->kvm->arch.adapters[adapter->id] = adapter; - return 0; +out: + mutex_unlock(&dev->kvm->lock); + return rc; } int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) @@ -2453,12 +2465,151 @@ int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) return ret; } +static struct page *pin_map_page(struct kvm *kvm, u64 uaddr, + unsigned int gup_flags) +{ + struct mm_struct *mm = kvm->mm; + struct page *page = NULL; + int locked = 1; + + if (mmget_not_zero(mm)) { + mmap_read_lock(mm); + pin_user_pages_remote(mm, uaddr, 1, FOLL_WRITE | gup_flags, + &page, &locked); + if (locked) + mmap_read_unlock(mm); + mmput(mm); + } + + return page; +} + +static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) +{ + struct s390_io_adapter *adapter = get_io_adapter(kvm, id); + struct s390_map_info *map; + unsigned long flags; + __u64 host_addr; + int ret, idx; + + if (!adapter || !addr) + return -EINVAL; + + map = kzalloc_obj(*map, GFP_KERNEL_ACCOUNT); + if (!map) + return -ENOMEM; + + INIT_LIST_HEAD(&map->list); + idx = srcu_read_lock(&kvm->srcu); + host_addr = gpa_to_hva(kvm, addr); + if (kvm_is_error_hva(host_addr)) { + srcu_read_unlock(&kvm->srcu, idx); + ret = -EFAULT; + goto out; + } + srcu_read_unlock(&kvm->srcu, idx); + map->guest_addr = addr; + map->addr = host_addr; + map->page = pin_map_page(kvm, host_addr, FOLL_LONGTERM); + if (!map->page) { + ret = -EINVAL; + goto out; + } + spin_lock_irqsave(&adapter->maps_lock, flags); + if (adapter->nr_maps < MAX_S390_ADAPTER_MAPS) { + list_add_tail(&map->list, &adapter->maps); + adapter->nr_maps++; + ret = 0; + } else { + ret = -EINVAL; + } + spin_unlock_irqrestore(&adapter->maps_lock, flags); + if (ret) + unpin_user_page(map->page); +out: + if (ret) + kfree(map); + return ret; +} + +static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) +{ + struct s390_io_adapter *adapter = get_io_adapter(kvm, id); + struct s390_map_info *map, *tmp, *map_to_free; + struct page *map_page_to_put = NULL; + u64 map_addr_to_mark = 0; + unsigned long flags; + int found = 0, idx; + + if (!adapter || !addr) + return -EINVAL; + + spin_lock_irqsave(&adapter->maps_lock, flags); + list_for_each_entry_safe(map, tmp, &adapter->maps, list) { + if (map->guest_addr == addr) { + found = 1; + adapter->nr_maps--; + list_del(&map->list); + map_page_to_put = map->page; + map_addr_to_mark = map->guest_addr; + map_to_free = map; + break; + } + } + spin_unlock_irqrestore(&adapter->maps_lock, flags); + + if (found) { + kfree(map_to_free); + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map_addr_to_mark >> PAGE_SHIFT); + set_page_dirty_lock(map_page_to_put); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map_page_to_put); + } + + return found ? 0 : -ENOENT; +} + +void kvm_s390_unmap_all_adapters(struct kvm *kvm) +{ + struct s390_map_info *map, *tmp; + unsigned long flags; + int i, idx; + + for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { + struct s390_io_adapter *adapter = kvm->arch.adapters[i]; + LIST_HEAD(local_list); + + if (!adapter) + continue; + + spin_lock_irqsave(&adapter->maps_lock, flags); + list_splice_init(&adapter->maps, &local_list); + adapter->nr_maps = 0; + spin_unlock_irqrestore(&adapter->maps_lock, flags); + + list_for_each_entry_safe(map, tmp, &local_list, list) { + list_del(&map->list); + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT); + set_page_dirty_lock(map->page); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map->page); + kfree(map); + } + } +} + void kvm_s390_destroy_adapters(struct kvm *kvm) { int i; - for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) + kvm_s390_unmap_all_adapters(kvm); + + for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { kfree(kvm->arch.adapters[i]); + kvm->arch.adapters[i] = NULL; + } } static int modify_io_adapter(struct kvm_device *dev, @@ -2480,14 +2631,22 @@ static int modify_io_adapter(struct kvm_device *dev, if (ret > 0) ret = 0; break; - /* - * The following operations are no longer needed and therefore no-ops. - * The gpa to hva translation is done when an IRQ route is set up. The - * set_irq code uses get_user_pages_remote() to do the actual write. - */ case KVM_S390_IO_ADAPTER_MAP: case KVM_S390_IO_ADAPTER_UNMAP: - ret = 0; + /* If in Secure Execution mode do not long term pin. */ + mutex_lock(&dev->kvm->lock); + if (kvm_s390_pv_is_protected(dev->kvm)) { + mutex_unlock(&dev->kvm->lock); + return 0; + } + if (req.type == KVM_S390_IO_ADAPTER_MAP) { + dev->kvm->stat.io_390_adapter_map++; + ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr); + } else { + dev->kvm->stat.io_390_adapter_unmap++; + ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr); + } + mutex_unlock(&dev->kvm->lock); break; default: ret = -EINVAL; @@ -2733,24 +2892,6 @@ static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; } -static struct page *get_map_page(struct kvm *kvm, u64 uaddr) -{ - struct mm_struct *mm = kvm->mm; - struct page *page = NULL; - int locked = 1; - - if (mmget_not_zero(mm)) { - mmap_read_lock(mm); - get_user_pages_remote(mm, uaddr, 1, FOLL_WRITE, - &page, &locked); - if (locked) - mmap_read_unlock(mm); - mmput(mm); - } - - return page; -} - static int adapter_indicators_set(struct kvm *kvm, struct s390_io_adapter *adapter, struct kvm_s390_adapter_int *adapter_int) @@ -2760,12 +2901,12 @@ static int adapter_indicators_set(struct kvm *kvm, struct page *ind_page, *summary_page; void *map; - ind_page = get_map_page(kvm, adapter_int->ind_addr); + ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0); if (!ind_page) return -1; - summary_page = get_map_page(kvm, adapter_int->summary_addr); + summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0); if (!summary_page) { - put_page(ind_page); + unpin_user_page(ind_page); return -1; } @@ -2784,8 +2925,8 @@ static int adapter_indicators_set(struct kvm *kvm, set_page_dirty_lock(summary_page); srcu_read_unlock(&kvm->srcu, idx); - put_page(ind_page); - put_page(summary_page); + unpin_user_page(ind_page); + unpin_user_page(summary_page); return summary_set ? 0 : 1; } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index de28ee1f7882..9de6edeb5ae0 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -68,6 +68,8 @@ const struct kvm_stats_desc kvm_vm_stats_desc[] = { KVM_GENERIC_VM_STATS(), STATS_DESC_COUNTER(VM, inject_io), + STATS_DESC_COUNTER(VM, io_390_adapter_map), + STATS_DESC_COUNTER(VM, io_390_adapter_unmap), STATS_DESC_COUNTER(VM, inject_float_mchk), STATS_DESC_COUNTER(VM, inject_pfault_done), STATS_DESC_COUNTER(VM, inject_service_signal), @@ -2539,6 +2541,7 @@ static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd) if (kvm_s390_pv_is_protected(kvm)) break; + kvm_s390_unmap_all_adapters(kvm); mmap_write_lock(kvm->mm); /* * Disable creation of new THPs. Existing THPs can stay, they diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index dc0573b7aa4b..7ba885cb6bd1 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -560,6 +560,8 @@ void kvm_s390_gisa_disable(struct kvm *kvm); void kvm_s390_gisa_enable(struct kvm *kvm); int __init kvm_s390_gib_init(u8 nisc); void kvm_s390_gib_destroy(void); +void kvm_s390_unmap_all_adapters(struct kvm *kvm); + /* implemented in guestdbg.c */ void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu); From 1e95e3bc6b0572e60a98c7dde52e27259a802d4d Mon Sep 17 00:00:00 2001 From: Douglas Freimuth Date: Thu, 4 Jun 2026 21:27:54 +0200 Subject: [PATCH 16/17] KVM: s390: Enable adapter_indicators_set to use mapped pages The s390 adapter_indicators_set function can now be optimized to use long-term mapped pages when available so that work can be processed on a fast path when interrupts are disabled. If adapter indicator pages are not mapped then local mapping is done on a slow path as it is prior to this patch. For example, Secure Execution environments will take the local mapping path as it does prior to this patch. Reviewed-by: Matthew Rosato Signed-off-by: Douglas Freimuth Acked-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260604192755.203143-3-freimuth@linux.ibm.com> --- arch/s390/kvm/interrupt.c | 87 ++++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 24 deletions(-) diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index d11dea4cb0d3..f276f76f814f 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -2892,41 +2892,80 @@ static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; } +static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, + u64 addr) +{ + struct s390_map_info *map; + + if (!adapter) + return NULL; + + list_for_each_entry(map, &adapter->maps, list) { + if (map->addr == addr) + return map; + } + return NULL; +} + static int adapter_indicators_set(struct kvm *kvm, struct s390_io_adapter *adapter, struct kvm_s390_adapter_int *adapter_int) { unsigned long bit; int summary_set, idx; - struct page *ind_page, *summary_page; + struct s390_map_info *ind_info, *summary_info; void *map; + struct page *ind_page, *summary_page; + unsigned long flags; - ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0); - if (!ind_page) - return -1; - summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0); - if (!summary_page) { + ind_page = NULL; + + spin_lock_irqsave(&adapter->maps_lock, flags); + ind_info = get_map_info(adapter, adapter_int->ind_addr); + if (!ind_info) { + spin_unlock_irqrestore(&adapter->maps_lock, flags); + ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0); + if (!ind_page) + return -1; + idx = srcu_read_lock(&kvm->srcu); + map = page_address(ind_page); + bit = get_ind_bit(adapter_int->ind_addr, + adapter_int->ind_offset, adapter->swap); + set_bit(bit, map); + mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT); + set_page_dirty_lock(ind_page); + srcu_read_unlock(&kvm->srcu, idx); unpin_user_page(ind_page); - return -1; + } else { + map = page_address(ind_info->page); + bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); + set_bit(bit, map); + spin_unlock_irqrestore(&adapter->maps_lock, flags); + } + spin_lock_irqsave(&adapter->maps_lock, flags); + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (!summary_info) { + spin_unlock_irqrestore(&adapter->maps_lock, flags); + summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0); + if (WARN_ON_ONCE(!summary_page)) + return -1; + idx = srcu_read_lock(&kvm->srcu); + map = page_address(summary_page); + bit = get_ind_bit(adapter_int->summary_addr, + adapter_int->summary_offset, adapter->swap); + summary_set = test_and_set_bit(bit, map); + mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT); + set_page_dirty_lock(summary_page); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(summary_page); + } else { + map = page_address(summary_info->page); + bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, + adapter->swap); + summary_set = test_and_set_bit(bit, map); + spin_unlock_irqrestore(&adapter->maps_lock, flags); } - idx = srcu_read_lock(&kvm->srcu); - map = page_address(ind_page); - bit = get_ind_bit(adapter_int->ind_addr, - adapter_int->ind_offset, adapter->swap); - set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT); - set_page_dirty_lock(ind_page); - map = page_address(summary_page); - bit = get_ind_bit(adapter_int->summary_addr, - adapter_int->summary_offset, adapter->swap); - summary_set = test_and_set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT); - set_page_dirty_lock(summary_page); - srcu_read_unlock(&kvm->srcu, idx); - - unpin_user_page(ind_page); - unpin_user_page(summary_page); return summary_set ? 0 : 1; } From a868b30492c59f398359b7891293bbde8d126a51 Mon Sep 17 00:00:00 2001 From: Douglas Freimuth Date: Thu, 4 Jun 2026 21:27:55 +0200 Subject: [PATCH 17/17] KVM: s390: Introducing kvm_arch_set_irq_inatomic fast inject s390 needs a fast path for irq injection, and along those lines we introduce kvm_arch_set_irq_inatomic. Instead of placing all interrupts on the global work queue as it does today, this patch provides a fast path for irq injection. The inatomic fast path cannot lose control since it is running with interrupts disabled. This meant making the following changes that exist on the slow path today. First, the adapter_indicators page needs to be mapped since it is accessed with interrupts disabled, so we added map/unmap functions. Second, access to shared resources between the fast and slow paths needed to be changed from mutex and semaphores to spin_lock's. Finally, the memory allocation on the slow path utilizes GFP_KERNEL_ACCOUNT but we had to implement the fast path with GFP_ATOMIC allocation. Each of these enhancements were required to prevent blocking on the fast inject path. Fencing of Fast Inject in Secure Execution environments is enabled in the patch series by not mapping adapter indicator pages. In Secure Execution environments the path of execution available before this patch is followed. Statistical counters have been added to enable analysis of irq injection on the fast path and slow path including io_390_inatomic, io_flic_inject_airq, io_set_adapter_int and io_390_inatomic_no_inject. The no inject counter captures adapter masked, coalesced and suppressed interrupts. Reviewed-by: Matthew Rosato Signed-off-by: Douglas Freimuth Acked-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260604192755.203143-4-freimuth@linux.ibm.com> --- arch/s390/include/asm/kvm_host.h | 6 +- arch/s390/kvm/intercept.c | 5 +- arch/s390/kvm/interrupt.c | 269 ++++++++++++++++++++++++------- arch/s390/kvm/kvm-s390.c | 27 +++- arch/s390/kvm/kvm-s390.h | 3 +- 5 files changed, 243 insertions(+), 67 deletions(-) diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 4ff1ede90873..eaa34c5bd3c1 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -359,7 +359,7 @@ struct kvm_s390_float_interrupt { struct kvm_s390_mchk_info mchk; struct kvm_s390_ext_info srv_signal; int last_sleep_cpu; - struct mutex ais_lock; + spinlock_t ais_lock; u8 simm; u8 nimm; }; @@ -450,6 +450,10 @@ struct kvm_vm_stat { u64 inject_io; u64 io_390_adapter_map; u64 io_390_adapter_unmap; + u64 io_390_inatomic; + u64 io_flic_inject_airq; + u64 io_set_adapter_int; + u64 io_390_inatomic_no_inject; u64 inject_float_mchk; u64 inject_pfault_done; u64 inject_service_signal; diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c index 39aff324203e..1980df61ef30 100644 --- a/arch/s390/kvm/intercept.c +++ b/arch/s390/kvm/intercept.c @@ -517,8 +517,9 @@ static int handle_pv_spx(struct kvm_vcpu *vcpu) static int handle_pv_sclp(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); /* * 2 cases: * a: an sccb answering interrupt was already pending or in flight. @@ -534,7 +535,7 @@ static int handle_pv_sclp(struct kvm_vcpu *vcpu) fi->srv_signal.ext_params |= 0x43000; set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); clear_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index f276f76f814f..51ea576bb445 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -624,8 +624,9 @@ static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) struct kvm_s390_mchk_info mchk = {}; int deliver = 0; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); spin_lock(&li->lock); if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) || test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) { @@ -654,7 +655,7 @@ static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) deliver = 1; } spin_unlock(&li->lock); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (deliver) { VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx", @@ -941,11 +942,12 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_ext_info ext; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) || !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } ext = fi->srv_signal; @@ -954,7 +956,7 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu) clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); if (kvm_s390_pv_cpu_is_protected(vcpu)) set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (!ext.ext_params) return 0; @@ -972,17 +974,18 @@ static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu) { struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_ext_info ext; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } ext = fi->srv_signal; /* only clear the event bits */ fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING; clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event"); vcpu->stat.deliver_service_signal++; @@ -997,8 +1000,9 @@ static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_interrupt_info *inti; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT], struct kvm_s390_interrupt_info, list); @@ -1008,7 +1012,7 @@ static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) } if (list_empty(&fi->lists[FIRQ_LIST_PFAULT])) clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, @@ -1039,8 +1043,9 @@ static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; struct kvm_s390_interrupt_info *inti; int rc = 0; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO], struct kvm_s390_interrupt_info, list); @@ -1058,7 +1063,7 @@ static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) } if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO])) clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, @@ -1116,10 +1121,11 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info io; u32 isc; int rc = 0; + unsigned long flags; fi = &vcpu->kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); isc = irq_type_to_isc(irq_type); isc_list = &fi->lists[isc]; inti = list_first_entry_or_null(isc_list, @@ -1146,7 +1152,7 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu, } if (list_empty(isc_list)) clear_bit(irq_type, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); if (inti) { rc = __do_deliver_io(vcpu, &(inti->io)); @@ -1662,8 +1668,9 @@ static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, struct kvm_s390_interrupt_info *iter; u16 id = (schid & 0xffff0000U) >> 16; u16 nr = schid & 0x0000ffffU; + unsigned long flags; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); list_for_each_entry(iter, isc_list, list) { if (schid && (id != iter->io.subchannel_id || nr != iter->io.subchannel_nr)) @@ -1673,10 +1680,10 @@ static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, fi->counters[FIRQ_CNTR_IO] -= 1; if (list_empty(isc_list)) clear_bit(isc_to_irq_type(isc), &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return iter; } - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return NULL; } @@ -1769,9 +1776,10 @@ static int __inject_service(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_service_signal++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING; /* We always allow events, track them separately from the sccb ints */ @@ -1791,7 +1799,7 @@ static int __inject_service(struct kvm *kvm, fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK; set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); out: - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kfree(inti); return 0; } @@ -1800,17 +1808,18 @@ static int __inject_virtio(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_virtio++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_VIRTIO] += 1; list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]); set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1818,18 +1827,19 @@ static int __inject_pfault_done(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_pfault_done++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_PFAULT] >= (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_PFAULT] += 1; list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]); set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1838,13 +1848,14 @@ static int __inject_float_mchk(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + unsigned long flags; kvm->stat.inject_float_mchk++; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS); fi->mchk.mcic |= inti->mchk.mcic; set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kfree(inti); return 0; } @@ -1855,6 +1866,7 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) struct kvm_s390_float_interrupt *fi; struct list_head *list; int isc; + unsigned long flags; kvm->stat.inject_io++; isc = int_word_to_isc(inti->io.io_int_word); @@ -1873,9 +1885,9 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) } fi = &kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) { - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return -EBUSY; } fi->counters[FIRQ_CNTR_IO] += 1; @@ -1890,7 +1902,7 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; list_add_tail(&inti->list, list); set_bit(isc_to_irq_type(isc), &fi->pending_irqs); - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); return 0; } @@ -1966,15 +1978,10 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) } int kvm_s390_inject_vm(struct kvm *kvm, - struct kvm_s390_interrupt *s390int) + struct kvm_s390_interrupt *s390int, struct kvm_s390_interrupt_info *inti) { - struct kvm_s390_interrupt_info *inti; int rc; - inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); - if (!inti) - return -ENOMEM; - inti->type = s390int->type; switch (inti->type) { case KVM_S390_INT_VIRTIO: @@ -2003,15 +2010,13 @@ int kvm_s390_inject_vm(struct kvm *kvm, inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull; break; default: - kfree(inti); return -EINVAL; } trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64, 2); rc = __inject_vm(kvm, inti); - if (rc) - kfree(inti); + return rc; } @@ -2176,12 +2181,13 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; int i; + unsigned long flags; mutex_lock(&kvm->lock); if (!kvm_s390_pv_is_protected(kvm)) fi->masked_irqs = 0; mutex_unlock(&kvm->lock); - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); fi->pending_irqs = 0; memset(&fi->srv_signal, 0, sizeof(fi->srv_signal)); memset(&fi->mchk, 0, sizeof(fi->mchk)); @@ -2189,7 +2195,7 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm) clear_irq_list(&fi->lists[i]); for (i = 0; i < FIRQ_MAX_COUNT; i++) fi->counters[i] = 0; - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); kvm_s390_gisa_clear(kvm); }; @@ -2204,6 +2210,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) int ret = 0; int n = 0; int i; + unsigned long flags; if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0) return -EINVAL; @@ -2235,7 +2242,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) } } fi = &kvm->arch.float_int; - spin_lock(&fi->lock); + spin_lock_irqsave(&fi->lock, flags); for (i = 0; i < FIRQ_LIST_COUNT; i++) { list_for_each_entry(inti, &fi->lists[i], list) { if (n == max_irqs) { @@ -2272,7 +2279,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) } out: - spin_unlock(&fi->lock); + spin_unlock_irqrestore(&fi->lock, flags); out_nolock: if (!ret && n > 0) { if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n)) @@ -2287,6 +2294,7 @@ static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_all ais; + unsigned long flags; if (attr->attr < sizeof(ais)) return -EINVAL; @@ -2294,10 +2302,10 @@ static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); ais.simm = fi->simm; ais.nimm = fi->nimm; - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais))) return -EFAULT; @@ -2683,6 +2691,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_req req; int ret = 0; + unsigned long flags; if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; @@ -2699,7 +2708,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) 2 : KVM_S390_AIS_MODE_SINGLE : KVM_S390_AIS_MODE_ALL, req.mode); - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); switch (req.mode) { case KVM_S390_AIS_MODE_ALL: fi->simm &= ~AIS_MODE_MASK(req.isc); @@ -2712,7 +2721,7 @@ static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) default: ret = -EINVAL; } - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); return ret; } @@ -2726,25 +2735,41 @@ static int kvm_s390_inject_airq(struct kvm *kvm, .parm = 0, .parm64 = isc_to_int_word(adapter->isc), }; + struct kvm_s390_interrupt_info *inti; + unsigned long flags; + int ret = 0; - if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) - return kvm_s390_inject_vm(kvm, &s390int); + inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); + if (!inti) + return -ENOMEM; - mutex_lock(&fi->ais_lock); - if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { - trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); - goto out; + if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) { + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (ret) + kfree(inti); + return ret; } - ret = kvm_s390_inject_vm(kvm, &s390int); + spin_lock_irqsave(&fi->ais_lock, flags); + if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { + trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); + spin_unlock_irqrestore(&fi->ais_lock, flags); + kfree(inti); + return ret; + } + + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { fi->nimm |= AIS_MODE_MASK(adapter->isc); trace_kvm_s390_modify_ais_mode(adapter->isc, KVM_S390_AIS_MODE_SINGLE, 2); } -out: - mutex_unlock(&fi->ais_lock); + + spin_unlock_irqrestore(&fi->ais_lock, flags); + if (ret) + kfree(inti); return ret; } @@ -2753,6 +2778,8 @@ static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr) unsigned int id = attr->attr; struct s390_io_adapter *adapter = get_io_adapter(kvm, id); + kvm->stat.io_flic_inject_airq++; + if (!adapter) return -EINVAL; @@ -2763,6 +2790,7 @@ static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) { struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; struct kvm_s390_ais_all ais; + unsigned long flags; if (!test_kvm_facility(kvm, 72)) return -EOPNOTSUPP; @@ -2770,10 +2798,10 @@ static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais))) return -EFAULT; - mutex_lock(&fi->ais_lock); + spin_lock_irqsave(&fi->ais_lock, flags); fi->simm = ais.simm; fi->nimm = ais.nimm; - mutex_unlock(&fi->ais_lock); + spin_unlock_irqrestore(&fi->ais_lock, flags); return 0; } @@ -2942,6 +2970,7 @@ static int adapter_indicators_set(struct kvm *kvm, set_bit(bit, map); spin_unlock_irqrestore(&adapter->maps_lock, flags); } + spin_lock_irqsave(&adapter->maps_lock, flags); summary_info = get_map_info(adapter, adapter_int->summary_addr); if (!summary_info) { @@ -2969,6 +2998,44 @@ static int adapter_indicators_set(struct kvm *kvm, return summary_set ? 0 : 1; } +static int adapter_indicators_set_fast(struct kvm *kvm, + struct s390_io_adapter *adapter, + struct kvm_s390_adapter_int *adapter_int, + int setbit) +{ + unsigned long bit; + int summary_set; + struct s390_map_info *ind_info, *summary_info; + void *map; + + spin_lock(&adapter->maps_lock); + ind_info = get_map_info(adapter, adapter_int->ind_addr); + if (!ind_info) { + spin_unlock(&adapter->maps_lock); + return -EWOULDBLOCK; + } + map = page_address(ind_info->page); + bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); + if (setbit) + set_bit(bit, map); + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (!summary_info) { + spin_unlock(&adapter->maps_lock); + return -EWOULDBLOCK; + } + map = page_address(summary_info->page); + bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, + adapter->swap); + /* If setbit then set summary bit. Else if falling back to the slow path */ + /* with setbit==0 then clear the summary bit so the slow path re-injects */ + if (setbit) + summary_set = test_and_set_bit(bit, map); + else + summary_set = test_and_clear_bit(bit, map); + spin_unlock(&adapter->maps_lock); + return summary_set ? 0 : 1; +} + /* * < 0 - not injected due to error * = 0 - coalesced, summary indicator already active @@ -2981,6 +3048,8 @@ static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, int ret; struct s390_io_adapter *adapter; + kvm->stat.io_set_adapter_int++; + /* We're only interested in the 0->1 transition. */ if (!level) return 0; @@ -3049,7 +3118,6 @@ int kvm_set_routing_entry(struct kvm *kvm, int idx; switch (ue->type) { - /* we store the userspace addresses instead of the guest addresses */ case KVM_IRQ_ROUTING_S390_ADAPTER: if (kvm_is_ucontrol(kvm)) return -EINVAL; @@ -3640,3 +3708,86 @@ int __init kvm_s390_gib_init(u8 nisc) out: return rc; } + +/* + * kvm_arch_set_irq_inatomic: fast-path for irqfd injection + */ +int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, + struct kvm *kvm, int irq_source_id, int level, + bool line_status) +{ + int ret, setbit; + struct s390_io_adapter *adapter; + struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + struct kvm_s390_interrupt_info *inti; + struct kvm_s390_interrupt s390int = { + .type = KVM_S390_INT_IO(1, 0, 0, 0), + .parm = 0, + }; + + kvm->stat.io_390_inatomic++; + + /* We're only interested in the 0->1 transition. */ + if (!level) + return 0; + if (e->type != KVM_IRQ_ROUTING_S390_ADAPTER) + return -EWOULDBLOCK; + + adapter = get_io_adapter(kvm, e->adapter.adapter_id); + if (!adapter) + return -EWOULDBLOCK; + + s390int.parm64 = isc_to_int_word(adapter->isc); + setbit = 1; + ret = adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + if (ret < 0) + return -EWOULDBLOCK; + if (!ret || adapter->masked) { + kvm->stat.io_390_inatomic_no_inject++; + return 0; + } + + inti = kzalloc_obj(*inti, GFP_ATOMIC); + if (!inti) { + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + return -EWOULDBLOCK; + } + + if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) { + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (ret == 0) { + return ret; + } else { + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + kfree(inti); + return -EWOULDBLOCK; + } + } + + spin_lock(&fi->ais_lock); + if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { + trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); + spin_unlock(&fi->ais_lock); + kfree(inti); + kvm->stat.io_390_inatomic_no_inject++; + return 0; + } + + ret = kvm_s390_inject_vm(kvm, &s390int, inti); + if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { + fi->nimm |= AIS_MODE_MASK(adapter->isc); + trace_kvm_s390_modify_ais_mode(adapter->isc, + KVM_S390_AIS_MODE_SINGLE, 2); + } else if (ret) { + spin_unlock(&fi->ais_lock); + setbit = 0; + adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); + kfree(inti); + return -EWOULDBLOCK; + } + + spin_unlock(&fi->ais_lock); + return 0; +} diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 9de6edeb5ae0..80b4bc497494 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -70,6 +70,10 @@ const struct kvm_stats_desc kvm_vm_stats_desc[] = { STATS_DESC_COUNTER(VM, inject_io), STATS_DESC_COUNTER(VM, io_390_adapter_map), STATS_DESC_COUNTER(VM, io_390_adapter_unmap), + STATS_DESC_COUNTER(VM, io_390_inatomic), + STATS_DESC_COUNTER(VM, io_flic_inject_airq), + STATS_DESC_COUNTER(VM, io_set_adapter_int), + STATS_DESC_COUNTER(VM, io_390_inatomic_no_inject), STATS_DESC_COUNTER(VM, inject_float_mchk), STATS_DESC_COUNTER(VM, inject_pfault_done), STATS_DESC_COUNTER(VM, inject_service_signal), @@ -2877,6 +2881,7 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) void __user *argp = (void __user *)arg; struct kvm_device_attr attr; int r; + struct kvm_s390_interrupt_info *inti; switch (ioctl) { case KVM_S390_INTERRUPT: { @@ -2885,7 +2890,12 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) r = -EFAULT; if (copy_from_user(&s390int, argp, sizeof(s390int))) break; - r = kvm_s390_inject_vm(kvm, &s390int); + inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); + if (!inti) + return -ENOMEM; + r = kvm_s390_inject_vm(kvm, &s390int, inti); + if (r) + kfree(inti); break; } case KVM_CREATE_IRQCHIP: { @@ -3286,7 +3296,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) mutex_unlock(&kvm->lock); } - mutex_init(&kvm->arch.float_int.ais_lock); + spin_lock_init(&kvm->arch.float_int.ais_lock); spin_lock_init(&kvm->arch.float_int.lock); for (i = 0; i < FIRQ_LIST_COUNT; i++) INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]); @@ -4408,19 +4418,28 @@ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clo } static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, - unsigned long token) + unsigned long token) { struct kvm_s390_interrupt inti; struct kvm_s390_irq irq; + struct kvm_s390_interrupt_info *inti_mem = NULL; + int ret = 0; if (start_token) { irq.u.ext.ext_params2 = token; irq.type = KVM_S390_INT_PFAULT_INIT; WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq)); } else { + inti_mem = kzalloc_obj(*inti_mem, GFP_KERNEL_ACCOUNT); + if (WARN_ON_ONCE(!inti_mem)) + return; + inti.type = KVM_S390_INT_PFAULT_DONE; inti.parm64 = token; - WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti)); + ret = kvm_s390_inject_vm(vcpu->kvm, &inti, inti_mem); + if (ret) + kfree(inti_mem); + WARN_ON_ONCE(ret); } } diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index 7ba885cb6bd1..6d2842fb71a3 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -376,7 +376,8 @@ int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu); void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu); void kvm_s390_clear_float_irqs(struct kvm *kvm); int __must_check kvm_s390_inject_vm(struct kvm *kvm, - struct kvm_s390_interrupt *s390int); + struct kvm_s390_interrupt *s390int, + struct kvm_s390_interrupt_info *inti); int __must_check kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq); static inline int kvm_s390_inject_prog_irq(struct kvm_vcpu *vcpu,