From 8fa01be5a6149404adb82c0979a78f6347edd3ef Mon Sep 17 00:00:00 2001 From: Farhan Ali Date: Thu, 23 Jul 2026 15:14:04 -0700 Subject: [PATCH 01/82] KVM: s390: pci: Reject adapter interrupt forwarding if already enabled The MPCIFC instruction doesn't allow registering adapter interrupts without first unregistering. So reject any request to enable interrupt forwarding if its already enabled for the zPCI device. This also fixes overwriting and thus leaking resources when the ioctl is called multiple times for the same device. Fixes: 3c5a1b6f0a18 ("KVM: s390: pci: provide routines for enabling/disabling interrupt forwarding") Cc: stable@vger.kernel.org Reviewed-by: Christian Borntraeger Reviewed-by: Matthew Rosato Signed-off-by: Farhan Ali Tested-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- arch/s390/kvm/pci.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 720bb58cabe2..d2a11cdf6941 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -237,6 +237,10 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, if (zdev->gisa == 0) return -EINVAL; + /* AIF already enabled for the device */ + if (zdev->kzdev->fib.fmt0.aibv != 0) + return -EINVAL; + kvm = zdev->kzdev->kvm; msi_vecs = min_t(unsigned int, fib->fmt0.noi, zdev->max_msi); From 36f6999ecde3976731a8bfc0b8e667da6f593069 Mon Sep 17 00:00:00 2001 From: Farhan Ali Date: Thu, 23 Jul 2026 15:14:05 -0700 Subject: [PATCH 02/82] KVM: s390: pci: Fix memory accounting for pinned/unpinned pages The account_mem() and unaccount_mem() functions call get_uid() which increments the reference count of struct user_struct on every invocation. But we don't decrement the count by calling free_uid(). It also accounted/unaccounted the pages against the current->mm. But its possible the unaccount_mem() can be called from a different process context than the one that originally pinned the pages. Let's fix this by storing the pinning process user_struct and mm_struct when accounting for pinned pages, and subsequently free these resources when the pages are unpinned. Fixes: 3c5a1b6f0a18 ("KVM: s390: pci: provide routines for enabling/disabling interrupt forwarding") Cc: stable@vger.kernel.org Reviewed-by: Christian Borntraeger Reviewed-by: Matthew Rosato Signed-off-by: Farhan Ali Tested-by: Matthew Rosato [borntraeger@linux.ibm.com: Fixed whitespace] Signed-off-by: Christian Borntraeger --- arch/s390/kvm/pci.c | 43 ++++++++++++++++++++++++++++++++----------- arch/s390/kvm/pci.h | 2 ++ 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index d2a11cdf6941..0741aed442bc 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -190,33 +190,54 @@ static int kvm_zpci_clear_airq(struct zpci_dev *zdev) return cc ? -EIO : 0; } -static inline void unaccount_mem(unsigned long nr_pages) +static inline void unaccount_mem(struct kvm_zdev *kzdev, unsigned long nr_pages) { - struct user_struct *user = get_uid(current_user()); + struct user_struct *user = kzdev->user_account; + struct mm_struct *mm_account = kzdev->mm_account; - if (user) + if (user) { atomic_long_sub(nr_pages, &user->locked_vm); - if (current->mm) - atomic64_sub(nr_pages, ¤t->mm->pinned_vm); + free_uid(user); + kzdev->user_account = NULL; + } + + if (mm_account) { + atomic64_sub(nr_pages, &mm_account->pinned_vm); + mmdrop(mm_account); + kzdev->mm_account = NULL; + } } -static inline int account_mem(unsigned long nr_pages) +static inline int account_mem(struct kvm_zdev *kzdev, unsigned long nr_pages) { struct user_struct *user = get_uid(current_user()); unsigned long page_limit, cur_pages, new_pages; + int rc = 0; page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; cur_pages = atomic_long_read(&user->locked_vm); do { new_pages = cur_pages + nr_pages; - if (new_pages > page_limit) - return -ENOMEM; + if (new_pages > page_limit) { + rc = -ENOMEM; + goto out; + } } while (!atomic_long_try_cmpxchg(&user->locked_vm, &cur_pages, new_pages)); - atomic64_add(nr_pages, ¤t->mm->pinned_vm); + if (current->mm) { + mmgrab(current->mm); + atomic64_add(nr_pages, ¤t->mm->pinned_vm); + } + + kzdev->user_account = user; + kzdev->mm_account = current->mm; return 0; + +out: + free_uid(user); + return rc; } static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, @@ -279,7 +300,7 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, } /* Account for pinned pages, roll back on failure */ - if (account_mem(pcount)) + if (account_mem(zdev->kzdev, pcount)) goto unpin2; /* AISB must be allocated before we can fill in GAITE */ @@ -400,7 +421,7 @@ static int kvm_s390_pci_aif_disable(struct zpci_dev *zdev, bool force) pcount++; } if (pcount > 0) - unaccount_mem(pcount); + unaccount_mem(kzdev, pcount); out: mutex_unlock(&aift->aift_lock); diff --git a/arch/s390/kvm/pci.h b/arch/s390/kvm/pci.h index ff0972dd5e71..fdf8c7bf4ed0 100644 --- a/arch/s390/kvm/pci.h +++ b/arch/s390/kvm/pci.h @@ -22,6 +22,8 @@ struct kvm_zdev { struct kvm *kvm; struct zpci_fib fib; struct list_head entry; + struct user_struct *user_account; + struct mm_struct *mm_account; }; struct zpci_gaite { From f86842e4d6c482300f4567f492d512c9ccf5bc4f Mon Sep 17 00:00:00 2001 From: Farhan Ali Date: Thu, 23 Jul 2026 15:14:06 -0700 Subject: [PATCH 03/82] KVM: s390: pci: Fix missing error codes and memory unaccounting In kvm_s390_pci_aif_enable() two error paths failed to set an error code, causing the function to return 0 on failure. It also failed to rollback memory accounting on failure. Fix both by propagating an error code on failure and calling unaccount_mem() in the cleanup path. Fixes: 3c5a1b6f0a18 ("KVM: s390: pci: provide routines for enabling/disabling interrupt forwarding") Cc: stable@vger.kernel.org Reviewed-by: Christian Borntraeger Reviewed-by: Matthew Rosato Signed-off-by: Farhan Ali Tested-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- arch/s390/kvm/pci.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 0741aed442bc..36eb30953bb5 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -300,14 +300,17 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, } /* Account for pinned pages, roll back on failure */ - if (account_mem(zdev->kzdev, pcount)) + rc = account_mem(zdev->kzdev, pcount); + if (rc) goto unpin2; /* AISB must be allocated before we can fill in GAITE */ mutex_lock(&aift->aift_lock); bit = airq_iv_alloc_bit(aift->sbv); - if (bit == -1UL) + if (bit == -1UL) { + rc = -ENOMEM; goto unlock; + } zdev->aisb = bit; /* store the summary bit number */ zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK | @@ -351,6 +354,8 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, return rc; unlock: + if (pcount > 0) + unaccount_mem(zdev->kzdev, pcount); mutex_unlock(&aift->aift_lock); unpin2: if (fib->fmt0.sum == 1) From 8bf09b9b7d3232806df95f409581f8a9fd99a3fa Mon Sep 17 00:00:00 2001 From: Farhan Ali Date: Thu, 23 Jul 2026 15:14:07 -0700 Subject: [PATCH 04/82] KVM: s390: pci: Fix NULL dereference on AIBV allocation failure The airq_iv_create() can return NULL on failure, but the return value was never checked. If it fails, zdev->aibv will be NULL and fail when dereferenced in kvm_zpci_set_airq(). Add a NULL check and free the previously allocated AISB bit and zdev->aisb on failure. Fixes: 3c5a1b6f0a18 ("KVM: s390: pci: provide routines for enabling/disabling interrupt forwarding") Cc: stable@vger.kernel.org Reviewed-by: Christian Borntraeger Reviewed-by: Matthew Rosato Signed-off-by: Farhan Ali Tested-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- arch/s390/kvm/pci.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 36eb30953bb5..1eb127fc9f89 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -317,6 +317,11 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, AIRQ_IV_GUESTVEC, phys_to_virt(fib->fmt0.aibv)); + if (!zdev->aibv) { + rc = -ENOMEM; + goto free_aisb; + } + spin_lock_irq(&aift->gait_lock); gaite = aift->gait + zdev->aisb; @@ -353,6 +358,9 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, rc = kvm_zpci_set_airq(zdev); return rc; +free_aisb: + airq_iv_free_bit(aift->sbv, zdev->aisb); + zdev->aisb = 0; unlock: if (pcount > 0) unaccount_mem(zdev->kzdev, pcount); From 5580c9858f1e00f60191eb09c3add359836d60b6 Mon Sep 17 00:00:00 2001 From: Farhan Ali Date: Thu, 23 Jul 2026 15:14:08 -0700 Subject: [PATCH 05/82] KVM: s390: pci: Fix resource leak on IRQ registration failure Currently if kvm_zpci_set_airq() fails, kvm_s390_pci_aif_enable() returns the error code but doesn't do any resource cleanup thus leaking resources. Fix this by cleaning up all the resources such as the GAITE, AIBV, AISB and unpinning any pinned pages. While at it, remove dead code that stored FIB values that were never referenced. As part of the cleanup, we are also holding the aift_lock a bit longer, as we hold the lock while executing the MPCIFC instruction. Though this is not strictly necessary, it means we don't have to drop and re-acquire in the error case. Fixes: 3c5a1b6f0a18 ("KVM: s390: pci: provide routines for enabling/disabling interrupt forwarding") Cc: stable@vger.kernel.org Reviewed-by: Matthew Rosato Reviewed-by: Christian Borntraeger Signed-off-by: Farhan Ali Tested-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- arch/s390/kvm/pci.c | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 1eb127fc9f89..50f495bc8303 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -344,19 +344,32 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, aift->kzdev[zdev->aisb] = zdev->kzdev; spin_unlock_irq(&aift->gait_lock); - /* Update guest FIB for re-issue */ - fib->fmt0.aisbo = zdev->aisb & 63; - fib->fmt0.aisb = virt_to_phys(aift->sbv->vector) + (zdev->aisb / 64) * 8; - fib->fmt0.isc = gisc; - /* Save some guest fib values in the host for later use */ - zdev->kzdev->fib.fmt0.isc = fib->fmt0.isc; + zdev->kzdev->fib.fmt0.isc = gisc; zdev->kzdev->fib.fmt0.aibv = fib->fmt0.aibv; - mutex_unlock(&aift->aift_lock); /* Issue the clp to setup the irq now */ rc = kvm_zpci_set_airq(zdev); - return rc; + if (!rc) { + mutex_unlock(&aift->aift_lock); + return rc; + } + + /* Start cleanup */ + zdev->kzdev->fib.fmt0.isc = 0; + zdev->kzdev->fib.fmt0.aibv = 0; + + spin_lock_irq(&aift->gait_lock); + gaite->count--; + gaite->aisb = 0; + gaite->gisc = 0; + gaite->aisbo = 0; + gaite->gisa = 0; + aift->kzdev[zdev->aisb] = NULL; + spin_unlock_irq(&aift->gait_lock); + + airq_iv_release(zdev->aibv); + zdev->aibv = NULL; free_aisb: airq_iv_free_bit(aift->sbv, zdev->aisb); From 868d32ac72cba21c5c6d8a66a814b7c25a3a5c01 Mon Sep 17 00:00:00 2001 From: Farhan Ali Date: Thu, 23 Jul 2026 15:14:09 -0700 Subject: [PATCH 06/82] KVM: s390: pci: Validate AIBV and AISB before pinning guest pages The AIBV holds one bit per MSI-X vector for a given function. The size of the bit vector is derived from the NOI and the AIBVO. If the size of the AIBV exceeds a single page boundary, then reject the request as we cannot safely pin the guest AIBV. Similarly reject the request if the AISB address is not 8-byte aligned as the architecture requires doubleword alignment for the summary bit address. Since the AISBO can address up to 64 bits, the size of the AISB can only be 8 bytes for the function. This also ensures the AISB doesn't exceed a single page boundary. Fixes: 3c5a1b6f0a18 ("KVM: s390: pci: provide routines for enabling/disabling interrupt forwarding") Cc: stable@vger.kernel.org Reviewed-by: Christian Borntraeger Reviewed-by: Matthew Rosato Signed-off-by: Farhan Ali Tested-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- arch/s390/kvm/pci.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 50f495bc8303..50f5ec79600e 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -244,7 +244,7 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, bool assist) { struct page *pages[1], *aibv_page, *aisb_page = NULL; - unsigned int msi_vecs, idx; + unsigned int msi_vecs, idx, size; struct zpci_gaite *gaite; unsigned long hva, bit; struct kvm *kvm; @@ -271,6 +271,14 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, return gisc; /* Replace AIBV address */ + size = BITS_TO_LONGS(msi_vecs + fib->fmt0.aibvo) * sizeof(unsigned long); + npages = DIV_ROUND_UP((fib->fmt0.aibv & ~PAGE_MASK) + size, PAGE_SIZE); + /* AIBV cannot span more than 1 page */ + if (npages > 1) { + rc = -EINVAL; + goto out; + } + idx = srcu_read_lock(&kvm->srcu); hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aibv)); npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, pages); @@ -286,6 +294,12 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, /* Pin the guest AISB if one was specified */ if (fib->fmt0.sum == 1) { + /* AISB must be dword aligned */ + if (fib->fmt0.aisb & 0x7) { + rc = -EINVAL; + goto unpin1; + } + idx = srcu_read_lock(&kvm->srcu); hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aisb)); npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, From 9579cb7fcceaba393f291d502500b76cc8b52df0 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Mon, 30 Mar 2026 13:56:12 +0200 Subject: [PATCH 07/82] KVM: s390: move some facilities from FACILITIES_KVM_CPUMODEL to FACILITIES_KVM Some facilities have been put into FACILITIES_KVM_CPUMODEL to be on the safe side with older VMMs. Unfortunately this has some unwanted side effects for VMMs without a CPU model (like kvm unit test) and IBC/VAL is not used in that case. Ideally the guest visible STFLE bits, the behaviour when running interpreted (HW supported) and the behaviour when running emulated (kvm or qemu) should be in sync. For LPSWEY this was not the case. STFLE.193 was off, but interpretion did work, emulation did not. As emulation only happened in rare cases (e.g. deliver a machine check) the result was inconsistency for the guest. Move beareh to FACILITIES_KVM to fix the inconsistency. NNPA (facility 165) has no fencing and no KVM emulation. The instruction will work, despite STFLE.165 being off in the guest. Move also to FACILITIES_KVM. Facility 170 (ineffective-nonconstrained-transaction facility) is an anti facility and should be passed along as well as KVM cannot simulate the missing function. KVM also does not implement trapping for guest RDP and there is no additional hypervisor control. Move 194 to FACILITIES_KVM as well. Facilities 196 and 197 (PAI) also do not have a hypervisor control and need to be passed on as well. The PFCR is also not intercepted by KVM and needs to be moved (stfle.201). The other facilities are fine (stfle, emulation, interpretion in sync): Both AP related features (12 and 15) require a userspace added AP via vfio. 156 etoken facility is fenced off for interpretion via ECD_ETOKENF so everything is in sync Signed-off-by: Christian Borntraeger Cc: David Hildenbrand Cc: Hendrik Brueckner Cc: Janosch Frank Reviewed-by: Janosch Frank --- arch/s390/tools/gen_facilities.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c index 2d28a569f793..32dd5a57240d 100644 --- a/arch/s390/tools/gen_facilities.c +++ b/arch/s390/tools/gen_facilities.c @@ -96,6 +96,13 @@ static struct facility_def facility_defs[] = { 150, /* enhanced sort */ 151, /* deflate conversion */ 155, /* msa extension 9 */ + 165, /* nnpa facility */ + 170, /* ineffective-nonconstrained-transaction facility */ + 193, /* bear enhancement facility */ + 194, /* rdp enhancement facility */ + 196, /* processor activity instrumentation facility */ + 197, /* processor activity instrumentation extension 1 */ + 201, /* concurrent-functions facility */ -1 /* END */ } }, @@ -112,13 +119,6 @@ static struct facility_def facility_defs[] = { 12, /* AP Query Configuration Information */ 15, /* AP Facilities Test */ 156, /* etoken facility */ - 165, /* nnpa facility */ - 170, /* ineffective-nonconstrained-transaction facility */ - 193, /* bear enhancement facility */ - 194, /* rdp enhancement facility */ - 196, /* processor activity instrumentation facility */ - 197, /* processor activity instrumentation extension 1 */ - 201, /* concurrent-functions facility */ -1 /* END */ } }, From 9972befc3e34ff8b6847198c84f11bfc312dde40 Mon Sep 17 00:00:00 2001 From: Jaehoon Kim Date: Fri, 24 Jul 2026 08:39:43 -0500 Subject: [PATCH 08/82] KVM: s390: Fall back to short-term pinning in MAP ioctl FOLL_LONGTERM pinning fails for some memory types, such as file-backed guest memory. As a result, kvm_s390_adapter_map() returns -EINVAL and irqfd adapter registration fails even though interrupt delivery could still work via the existing non-atomic path. When FOLL_LONGTERM pinning fails, verify that the page is accessible using a short-term pin instead. If the short-term pin succeeds, unpin the page and add a map entry with pinned=false to preserve MAP/UNMAP symmetry. The non-atomic irqfd path already performs short-term pinning for interrupt delivery, so this restores the previous behavior for memory that cannot be pinned long-term. get_map_info() is updated to return NULL for unpinned entries so that the atomic irqfd fast path falls back to the non-atomic path. kvm_s390_adapter_unmap() and kvm_s390_unmap_all_adapters() skip dirty marking and unpin for unpinned entries. Update Documentation/virt/kvm/devices/s390_flic.rst to reflect the new MAP/UNMAP behavior. Fixes: c9a568838086 ("KVM: s390: Add map/unmap ioctl and clean mappings post-guest") Signed-off-by: Jaehoon Kim Reviewed-by: Douglas Freimuth Reviewed-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- Documentation/virt/kvm/devices/s390_flic.rst | 15 +++++- arch/s390/include/asm/kvm_host.h | 5 ++ arch/s390/kvm/interrupt.c | 56 +++++++++++++++----- 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/Documentation/virt/kvm/devices/s390_flic.rst b/Documentation/virt/kvm/devices/s390_flic.rst index b784f8016748..983c858b444a 100644 --- a/Documentation/virt/kvm/devices/s390_flic.rst +++ b/Documentation/virt/kvm/devices/s390_flic.rst @@ -112,9 +112,20 @@ Groups: mask or unmask the adapter, as specified in mask KVM_S390_IO_ADAPTER_MAP - This is now a no-op. The mapping is purely done by the irq route. + Map an adapter indicator or summary page for long-term pinning so that + interrupt injection can be performed in atomic context. If long-term + pinning is not possible (e.g. file-backed memory), the page is verified + via a short-term pin and the ioctl returns success; interrupt injection + will use the non-atomic irqfd path with short-term pinning on each + interrupt. In Secure Execution mode this is a no-op and the ioctl + returns success. + KVM_S390_IO_ADAPTER_UNMAP - This is now a no-op. The mapping is purely done by the irq route. + Unmap a previously mapped adapter indicator or summary page and release + the long-term pin. If the page was not long-term pinned (e.g. file-backed + memory), the map entry is removed and success is returned; if no prior + map entry exists, -ENOENT is returned. In Secure Execution mode this is + a no-op and the ioctl returns success. KVM_DEV_FLIC_AISM modify the adapter-interruption-suppression mode for a given isc if the diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index eaa34c5bd3c1..c172f9b212d1 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -476,6 +476,11 @@ struct s390_map_info { __u64 guest_addr; __u64 addr; struct page *page; + /* + * True if the page is long-term pinned. False if long-term pinning + * failed and this entry exists only to preserve MAP/UNMAP symmetry. + */ + bool pinned; }; struct s390_io_adapter { diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 9e3e6b0d72ad..7514d9e2403c 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -2520,8 +2520,22 @@ static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) map->addr = host_addr; map->page = pin_map_page(kvm, host_addr, FOLL_LONGTERM); if (!map->page) { - ret = -EINVAL; - goto out; + /* + * Long-term pinning may fail for memory types such as file-backed + * memory. Verify that short-term pinning succeeds so that the + * non-atomic irqfd path can handle interrupt injection. + */ + map->page = pin_map_page(kvm, host_addr, 0); + if (!map->page) { + ret = -EINVAL; + goto out; + } + unpin_user_page(map->page); + map->page = NULL; + map->pinned = false; + /* Add an entry to preserve MAP/UNMAP symmetry. */ + } else { + map->pinned = true; } spin_lock_irqsave(&adapter->maps_lock, flags); if (adapter->nr_maps < MAX_S390_ADAPTER_MAPS) { @@ -2532,7 +2546,7 @@ static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) ret = -EINVAL; } spin_unlock_irqrestore(&adapter->maps_lock, flags); - if (ret) + if (ret && map->page) unpin_user_page(map->page); out: if (ret) @@ -2546,6 +2560,7 @@ static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) struct s390_map_info *map, *tmp, *map_to_free; struct page *map_page_to_put = NULL; u64 map_addr_to_mark = 0; + bool map_pinned = false; unsigned long flags; int found = 0, idx; @@ -2560,6 +2575,7 @@ static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) list_del(&map->list); map_page_to_put = map->page; map_addr_to_mark = map->guest_addr; + map_pinned = map->pinned; map_to_free = map; break; } @@ -2568,11 +2584,18 @@ static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) if (found) { kfree(map_to_free); - idx = srcu_read_lock(&kvm->srcu); - mark_page_dirty(kvm, map_addr_to_mark >> PAGE_SHIFT); - set_page_dirty_lock(map_page_to_put); - srcu_read_unlock(&kvm->srcu, idx); - unpin_user_page(map_page_to_put); + if (map_pinned) { + /* + * Only long-term pinned pages need to be marked dirty + * and released. Fallback entries exist only for + * MAP/UNMAP symmetry. + */ + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map_addr_to_mark >> PAGE_SHIFT); + set_page_dirty_lock(map_page_to_put); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map_page_to_put); + } } return found ? 0 : -ENOENT; @@ -2598,11 +2621,13 @@ void kvm_s390_unmap_all_adapters(struct kvm *kvm) list_for_each_entry_safe(map, tmp, &local_list, list) { list_del(&map->list); - idx = srcu_read_lock(&kvm->srcu); - mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT); - set_page_dirty_lock(map->page); - srcu_read_unlock(&kvm->srcu, idx); - unpin_user_page(map->page); + if (map->pinned) { + idx = srcu_read_lock(&kvm->srcu); + mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT); + set_page_dirty_lock(map->page); + srcu_read_unlock(&kvm->srcu, idx); + unpin_user_page(map->page); + } kfree(map); } } @@ -2929,8 +2954,11 @@ static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, return NULL; list_for_each_entry(map, &adapter->maps, list) { - if (map->addr == addr) + if (map->addr == addr) { + if (!map->pinned) + return NULL; return map; + } } return NULL; } From eae9cb30cfa74c29d4b633d144f9f63af7aac94f Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 30 Jul 2026 07:34:37 +0200 Subject: [PATCH 09/82] MAINTAINERS: Update s390 specific vfio sections Update all three s390 vfio sections to clarify that s390 specific vfio code normally will go upstream via the kvms390 git tree. Add a corresponding T entry to reflect that. For consistency with the other s390 specfic vfio sections rename "VFIO AP" to "VFIO-AP" and add an L entry for the kvm mailing list. Acked-by: Claudio Imbrenda Acked-by: Matthew Rosato Acked-by: Eric Farman Acked-by: Anthony Krowiak Acked-by: Farhan Ali Signed-off-by: Heiko Carstens Acked-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- MAINTAINERS | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 4a8b0fd665ce..78f3f17f7b44 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23884,12 +23884,14 @@ S: Supported F: drivers/s390/block/scm* F: drivers/s390/cio/scm.c -S390 VFIO AP DRIVER +S390 VFIO-AP DRIVER M: Tony Krowiak M: Halil Pasic M: Jason Herne L: linux-s390@vger.kernel.org +L: kvm@vger.kernel.org S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git F: Documentation/arch/s390/vfio-ap* F: drivers/s390/crypto/vfio_ap* @@ -23900,6 +23902,7 @@ R: Halil Pasic L: linux-s390@vger.kernel.org L: kvm@vger.kernel.org S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git F: Documentation/arch/s390/vfio-ccw.rst F: drivers/s390/cio/vfio_ccw* F: include/uapi/linux/vfio_ccw.h @@ -23911,6 +23914,7 @@ R: Eric Farman L: linux-s390@vger.kernel.org L: kvm@vger.kernel.org S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git F: arch/s390/kvm/pci* F: drivers/vfio/pci/vfio_pci_zdev.c F: include/uapi/linux/vfio_zdev.h From a3928021b93006f857c365dc6ac3c1a84cb7a9ee Mon Sep 17 00:00:00 2001 From: Ciunas Bennett Date: Fri, 24 Jul 2026 12:39:57 +0200 Subject: [PATCH 10/82] KVM: s390: Refactor __diag_time_slice_end_directed for single exit point Refactor the DIAG 9c (directed yield) handler to use a unified exit path, improving code maintainability and reducing duplication. Changes: - Consolidate all exit paths to use a single 'out' label - Replace multiple VCPU_EVENT logging calls with one unified call - Introduce 'result' string variable to track operation outcome - Initialize tcpu_cpu to -1 for safe handling across all code paths - Ensure statistics updates occur before the common exit point This refactoring maintains identical functionality while making the control flow clearer and easier to maintain. All three possible outcomes (yield forwarded, done, ignored) now converge at a single logging point Signed-off-by: Ciunas Bennett Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/diag.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c index d89d1c381522..85c84421b510 100644 --- a/arch/s390/kvm/diag.c +++ b/arch/s390/kvm/diag.c @@ -186,7 +186,8 @@ static int diag9c_forwarding_overrun(void) static int __diag_time_slice_end_directed(struct kvm_vcpu *vcpu) { struct kvm_vcpu *tcpu; - int tcpu_cpu; + const char *result; + int tcpu_cpu = -1; int tid; tid = vcpu->run->s.regs.gprs[(vcpu->arch.sie_block->ipa & 0xf0) >> 4]; @@ -211,21 +212,22 @@ static int __diag_time_slice_end_directed(struct kvm_vcpu *vcpu) if (!vcpu_is_preempted(tcpu_cpu)) goto no_yield; smp_yield_cpu(tcpu_cpu); - VCPU_EVENT(vcpu, 5, - "diag time slice end directed to %d: yield forwarded", - tid); vcpu->stat.diag_9c_forward++; - return 0; + result = "yield forwarded"; + goto out; } if (kvm_vcpu_yield_to(tcpu) <= 0) goto no_yield; - VCPU_EVENT(vcpu, 5, "diag time slice end directed to %d: done", tid); - return 0; + result = "done"; + goto out; no_yield: - VCPU_EVENT(vcpu, 5, "diag time slice end directed to %d: ignored", tid); vcpu->stat.diag_9c_ignored++; + result = "ignored"; +out: + VCPU_EVENT(vcpu, 5, "diag time slice end directed to %d: %s", tid, + result); return 0; } From 5ff53ce9a4771140336929ded9d16178cad05d95 Mon Sep 17 00:00:00 2001 From: Ciunas Bennett Date: Fri, 24 Jul 2026 12:39:58 +0200 Subject: [PATCH 11/82] KVM: s390: Add tracepoint for DIAG 9c directed yield operations Add a new tracepoint, kvm_s390_diag_9c, to provide visibility into directed yield operations. The tracepoint records: - Source vCPU context via the standard vCPU tracepoint fields - Target vCPU ID - Target physical CPU number - Operation result (done, ignored, or yield forwarded) This improves observability of vCPU scheduling behaviour and helps diagnose performance issues related to directed yields in virtualised s390 environments. Signed-off-by: Ciunas Bennett Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/diag.c | 1 + arch/s390/kvm/trace.h | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c index 85c84421b510..031ab6e5d6c4 100644 --- a/arch/s390/kvm/diag.c +++ b/arch/s390/kvm/diag.c @@ -228,6 +228,7 @@ static int __diag_time_slice_end_directed(struct kvm_vcpu *vcpu) out: VCPU_EVENT(vcpu, 5, "diag time slice end directed to %d: %s", tid, result); + trace_kvm_s390_diag_9c(vcpu, tid, tcpu_cpu, result); return 0; } diff --git a/arch/s390/kvm/trace.h b/arch/s390/kvm/trace.h index aa419eb6a0c8..dd2685c7df5b 100644 --- a/arch/s390/kvm/trace.h +++ b/arch/s390/kvm/trace.h @@ -283,6 +283,32 @@ TRACE_EVENT(kvm_s390_handle_diag, __print_symbolic(__entry->code, diagnose_codes)) ); +TRACE_EVENT(kvm_s390_diag_9c, + TP_PROTO(VCPU_PROTO_COMMON, int target_vcpu, int target_cpu, + const char *result), + TP_ARGS(VCPU_ARGS_COMMON, target_vcpu, target_cpu, result), + + TP_STRUCT__entry( + VCPU_FIELD_COMMON + __field(int, target_vcpu) + __field(int, target_cpu) + __string(result, result) + ), + + TP_fast_assign( + VCPU_ASSIGN_COMMON + __entry->target_vcpu = target_vcpu; + __entry->target_cpu = target_cpu; + __assign_str(result); + ), + + VCPU_TP_PRINTK( + "diag=9c target_vcpu=%d target_pcpu=%d result=%s", + __entry->target_vcpu, + __entry->target_cpu, + __get_str(result)) + ); + TRACE_EVENT(kvm_s390_handle_lctl, TP_PROTO(VCPU_PROTO_COMMON, int g, int reg1, int reg3, u64 addr), TP_ARGS(VCPU_ARGS_COMMON, g, reg1, reg3, addr), From 74186c2968f8f756ac3226b545b598457c910c75 Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 28 Jul 2026 05:30:13 +0200 Subject: [PATCH 12/82] s390/vfio_ccw: Free all memory if cp_init() fails The routine cp_free() is called to unpin/free any memory once an I/O is completed successfully, or if cp_prefetch() fails. But if cp_init() fails, and cp->initialized is not enabled, the same routine cannot be used to free all the memory. An attempt to address this exists in ccwchain_handle_ccw(), where a single call to ccwchain_free() is made for the currently-processed CCW segment. But this will leak other segments (created as a result of a Transfer in Channel) that had been allocated as part of the same channel program. Address this by performing the cleanup outside of the recursive ccwchain_handle_ccw()/ccwchain_loop_tic() logic. Fixes: 8b515be512a2 ("vfio-ccw: Fix memory leak and don't call cp_free in cp_init") Cc: stable@vger.kernel.org Reviewed-by: Farhan Ali Reviewed-by: Matthew Rosato Signed-off-by: Eric Farman Signed-off-by: Christian Borntraeger --- drivers/s390/cio/vfio_ccw_cp.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c index 7561aa7d3e01..086d1b54bdb0 100644 --- a/drivers/s390/cio/vfio_ccw_cp.c +++ b/drivers/s390/cio/vfio_ccw_cp.c @@ -455,9 +455,6 @@ static int ccwchain_handle_ccw(dma32_t cda, struct channel_program *cp) /* Loop for tics on this new chain. */ ret = ccwchain_loop_tic(chain, cp); - if (ret) - ccwchain_free(chain); - return ret; } @@ -486,6 +483,23 @@ static int ccwchain_loop_tic(struct ccwchain *chain, struct channel_program *cp) return 0; } +static int ccwchain_build_ccws(dma32_t cda, struct channel_program *cp) +{ + struct ccwchain *chain, *temp; + int ret; + + ret = ccwchain_handle_ccw(cda, cp); + + if (ret) { + /* Cleanup if an error occurred */ + list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) { + ccwchain_free(chain); + } + } + + return ret; +} + static int ccwchain_fetch_tic(struct ccw1 *ccw, struct channel_program *cp) { @@ -735,7 +749,7 @@ int cp_init(struct channel_program *cp, union orb *orb) memcpy(&cp->orb, orb, sizeof(*orb)); /* Build a ccwchain for the first CCW segment */ - ret = ccwchain_handle_ccw(orb->cmd.cpa, cp); + ret = ccwchain_build_ccws(orb->cmd.cpa, cp); if (!ret) cp->initialized = true; From 5405c90d6a47b3014e74ee0618a162449abbbc93 Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 28 Jul 2026 05:30:14 +0200 Subject: [PATCH 13/82] s390/vfio_ccw: Limit the number of channel program segments The processing of channel programs, and the CCWs within them, is done recursively. As such, there is an arbitrary (but not architectural) limit to the number of CCWs that can exist in a single channel program. The vfio-ccw logic breaks these channel programs into segments whenever it encounters a Transfer-In-Channel (TIC) CCW, and the combined number of segments count towards the global limit. Impose an equivalent limit to the number of segments until such logic can be made non-recursive. Fixes: 0a19e61e6d4c ("vfio: ccw: introduce channel program interfaces") Cc: stable@vger.kernel.org Reviewed-by: Matthew Rosato Signed-off-by: Eric Farman Signed-off-by: Christian Borntraeger --- drivers/s390/cio/vfio_ccw_cp.c | 6 ++++++ drivers/s390/cio/vfio_ccw_cp.h | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c index 086d1b54bdb0..1c2890d139c6 100644 --- a/drivers/s390/cio/vfio_ccw_cp.c +++ b/drivers/s390/cio/vfio_ccw_cp.c @@ -332,6 +332,7 @@ static struct ccwchain *ccwchain_alloc(struct channel_program *cp, int len) goto out_err; list_add_tail(&chain->next, &cp->ccwchain_list); + cp->ccwchain_count++; return chain; @@ -441,6 +442,10 @@ static int ccwchain_handle_ccw(dma32_t cda, struct channel_program *cp) if (len < 0) return len; + /* Limit number of chains in a single channel program */ + if (cp->ccwchain_count >= CCWCHAIN_COUNT_MAX) + return -EINVAL; + /* Need alloc a new chain for this one. */ chain = ccwchain_alloc(cp, len); if (!chain) @@ -745,6 +750,7 @@ int cp_init(struct channel_program *cp, union orb *orb) vdev->dev, "Prefetching channel program even though prefetch not specified in ORB"); + cp->ccwchain_count = 0; INIT_LIST_HEAD(&cp->ccwchain_list); memcpy(&cp->orb, orb, sizeof(*orb)); diff --git a/drivers/s390/cio/vfio_ccw_cp.h b/drivers/s390/cio/vfio_ccw_cp.h index fc31eb699807..a9b1d8dbc6f6 100644 --- a/drivers/s390/cio/vfio_ccw_cp.h +++ b/drivers/s390/cio/vfio_ccw_cp.h @@ -23,11 +23,18 @@ */ #define CCWCHAIN_LEN_MAX 256 +/* + * Maximum number of chains + */ +#define CCWCHAIN_COUNT_MAX 16 + /** * struct channel_program - manage information for channel program * @ccwchain_list: list head of ccwchains * @orb: orb for the currently processed ssch request * @initialized: whether this instance is actually initialized + * @guest_cp: copy of guest channel program + * @ccwchain_count: number of channel program segments (linked by TIC) * * @ccwchain_list is the head of a ccwchain list, that contents the * translated result of the guest channel program that pointed out by @@ -38,6 +45,7 @@ struct channel_program { union orb orb; bool initialized; struct ccw1 *guest_cp; + unsigned int ccwchain_count; }; int cp_init(struct channel_program *cp, union orb *orb); From a005b7f1a491ffda61bff0fd0f6548f8986fb977 Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 28 Jul 2026 05:30:15 +0200 Subject: [PATCH 14/82] s390/vfio_ccw: Fix out of bounds check on CCW array The routine ccwchain_calc_length() counts the number of channel command words (CCWs) that are chained together in a single channel program, and rejects anything larger than CCWCHAIN_LEN_MAX (256) CCWs. The loop itself is "do..while (count < 257)", and while the logic in is_cpa_within_range() correctly adjusts between the 0-index array of CCWs and the count of CCWs starting at 1, this means it would look at a possible 257th CCW before ending the loop and (correctly) returning an error. Fix this by restructuring the loop to break as soon as 256 CCWs (thus indexes 0-255) are examined, without looking at memory outside the range. Fixes: 0a19e61e6d4c ("vfio: ccw: introduce channel program interfaces") Cc: stable@vger.kernel.org Reviewed-by: Matthew Rosato Signed-off-by: Eric Farman Signed-off-by: Christian Borntraeger --- drivers/s390/cio/vfio_ccw_cp.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c index 1c2890d139c6..af632f9d5453 100644 --- a/drivers/s390/cio/vfio_ccw_cp.c +++ b/drivers/s390/cio/vfio_ccw_cp.c @@ -377,11 +377,9 @@ static void ccwchain_cda_free(struct ccwchain *chain, int idx) static int ccwchain_calc_length(u64 iova, struct channel_program *cp) { struct ccw1 *ccw = cp->guest_cp; - int cnt = 0; - - do { - cnt++; + int cnt; + for (cnt = 1; cnt <= CCWCHAIN_LEN_MAX; cnt++, ccw++) { /* * We want to keep counting if the current CCW has the * command-chaining flag enabled, or if it is a TIC CCW @@ -391,15 +389,10 @@ static int ccwchain_calc_length(u64 iova, struct channel_program *cp) * after the TIC, depending on the results of its operation. */ if (!ccw_is_chain(ccw) && !is_tic_within_range(ccw, iova, cnt)) - break; + return cnt; + } - ccw++; - } while (cnt < CCWCHAIN_LEN_MAX + 1); - - if (cnt == CCWCHAIN_LEN_MAX + 1) - cnt = -EINVAL; - - return cnt; + return -EINVAL; } static int tic_target_chain_exists(struct ccw1 *tic, struct channel_program *cp) From 565bef268d75bf7df665bce6923a88cd0eb74592 Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 28 Jul 2026 05:30:16 +0200 Subject: [PATCH 15/82] s390/vfio_ccw: Ensure first IDAW remains constant The first IDAW in a list does not need to be on a 2K/4K boundary like all others, and so is read separately to accurately calculate the size of the buffer needed to read the full IDAL. Verify that the address found in the first IDAW is unchanged between reads, to ensure a consistent set of IDAWs being worked with. Fixes: 01aa26c672c0 ("s390/cio: Combine direct and indirect CCW paths") Cc: stable@vger.kernel.org Reviewed-by: Matthew Rosato Signed-off-by: Eric Farman Signed-off-by: Christian Borntraeger --- drivers/s390/cio/vfio_ccw_cp.c | 16 ++++++++++++++++ drivers/s390/cio/vfio_ccw_cp.h | 2 ++ 2 files changed, 18 insertions(+) diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c index af632f9d5453..6275794751cb 100644 --- a/drivers/s390/cio/vfio_ccw_cp.c +++ b/drivers/s390/cio/vfio_ccw_cp.c @@ -523,6 +523,7 @@ static dma64_t *get_guest_idal(struct ccw1 *ccw, struct channel_program *cp, int &container_of(cp, struct vfio_ccw_private, cp)->vdev; dma64_t *idaws; dma32_t *idaws_f1; + u64 first_idaw; int idal_len = idaw_nr * sizeof(*idaws); int idaw_size = idal_is_2k(cp) ? PAGE_SIZE / 2 : PAGE_SIZE; int idaw_mask = ~(idaw_size - 1); @@ -539,6 +540,18 @@ static dma64_t *get_guest_idal(struct ccw1 *ccw, struct channel_program *cp, int kfree(idaws); return ERR_PTR(ret); } + + idaws_f1 = (dma32_t *)idaws; + if (cp->orb.cmd.c64) + first_idaw = dma64_to_u64(idaws[0]); + else + first_idaw = dma32_to_u32(idaws_f1[0]); + + /* Unexpected mismatch from earlier read */ + if (first_idaw != cp->guest_iova) { + kfree(idaws); + return ERR_PTR(-EINVAL); + } } else { /* Fabricate an IDAL based off CCW data address */ if (cp->orb.cmd.c64) { @@ -604,6 +617,9 @@ static int ccw_count_idaws(struct ccw1 *ccw, iova = dma32_to_u32(ccw->cda); } + /* Save the read address for later */ + cp->guest_iova = iova; + /* Format-1 IDAWs operate on 2K each */ if (!cp->orb.cmd.c64) return idal_2k_nr_words((void *)iova, bytes); diff --git a/drivers/s390/cio/vfio_ccw_cp.h b/drivers/s390/cio/vfio_ccw_cp.h index a9b1d8dbc6f6..9af98ff12d67 100644 --- a/drivers/s390/cio/vfio_ccw_cp.h +++ b/drivers/s390/cio/vfio_ccw_cp.h @@ -35,6 +35,7 @@ * @initialized: whether this instance is actually initialized * @guest_cp: copy of guest channel program * @ccwchain_count: number of channel program segments (linked by TIC) + * @guest_iova: first data address of a guest channel program * * @ccwchain_list is the head of a ccwchain list, that contents the * translated result of the guest channel program that pointed out by @@ -46,6 +47,7 @@ struct channel_program { bool initialized; struct ccw1 *guest_cp; unsigned int ccwchain_count; + u64 guest_iova; }; int cp_init(struct channel_program *cp, union orb *orb); From 4f6fdc6e1a7fbfa36b945af33c65a417948feac0 Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 28 Jul 2026 05:30:17 +0200 Subject: [PATCH 16/82] s390/vfio_ccw: Calculate idal length based on idaw type Sashiko pointed out that get_guest_idal() unconditionally calculates the length of the IDAL presuming everything is a Format-2 IDAW. The output of vfio-ccw is always Format-2, but the input can be either Format-1 (31-bit addresses) or Format-2 (64-bit addresses). As a result, the size of the guest IDAL may be incorrect and should be trimmed down. Reported-by: sashiko-bot Link: https://lore.kernel.org/r/20260720203400.7328E1F000E9@smtp.kernel.org/ Fixes: 1b676fe3d9d3 ("vfio/ccw: handle a guest Format-1 IDAL") Cc: stable@vger.kernel.org Reviewed-by: Matthew Rosato Signed-off-by: Eric Farman Signed-off-by: Christian Borntraeger --- drivers/s390/cio/vfio_ccw_cp.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c index 6275794751cb..5ef082b8289a 100644 --- a/drivers/s390/cio/vfio_ccw_cp.c +++ b/drivers/s390/cio/vfio_ccw_cp.c @@ -233,6 +233,7 @@ static void convert_ccw0_to_ccw1(struct ccw1 *source, unsigned long len) } #define idal_is_2k(_cp) (!(_cp)->orb.cmd.c64 || (_cp)->orb.cmd.i2k) +#define get_idaw_size(_cp) ((_cp)->orb.cmd.c64 ? sizeof(u64) : sizeof(u32)) /* * Helpers to operate ccwchain. @@ -524,7 +525,7 @@ static dma64_t *get_guest_idal(struct ccw1 *ccw, struct channel_program *cp, int dma64_t *idaws; dma32_t *idaws_f1; u64 first_idaw; - int idal_len = idaw_nr * sizeof(*idaws); + int idal_len = idaw_nr * get_idaw_size(cp); int idaw_size = idal_is_2k(cp) ? PAGE_SIZE / 2 : PAGE_SIZE; int idaw_mask = ~(idaw_size - 1); int i, ret; @@ -593,7 +594,7 @@ static int ccw_count_idaws(struct ccw1 *ccw, struct vfio_device *vdev = &container_of(cp, struct vfio_ccw_private, cp)->vdev; u64 iova; - int size = cp->orb.cmd.c64 ? sizeof(u64) : sizeof(u32); + int size = get_idaw_size(cp); int ret; int bytes = 1; From 9f5f9a78fedc45bc29d6a0a64e3a3472361afae5 Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 28 Jul 2026 05:30:18 +0200 Subject: [PATCH 17/82] s390/vfio_ccw: Ensure index for read/write regions are within range The introduction of the capability chain rightly clamped the region indexes to the range of the capabilities itself, but neglected to do so for the existing read/write regions which should also be enforced. Fixes: db8e5d17ac03 ("vfio-ccw: add capabilities chain") Cc: stable@vger.kernel.org Cc: Cornelia Huck Reviewed-by: Matthew Rosato Signed-off-by: Eric Farman Signed-off-by: Christian Borntraeger --- drivers/s390/cio/vfio_ccw_async.c | 16 ++++++++++++++++ drivers/s390/cio/vfio_ccw_chp.c | 15 +++++++++++++++ drivers/s390/cio/vfio_ccw_ops.c | 7 +++---- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/drivers/s390/cio/vfio_ccw_async.c b/drivers/s390/cio/vfio_ccw_async.c index 420d89ba7f83..4aff0b58fa5d 100644 --- a/drivers/s390/cio/vfio_ccw_async.c +++ b/drivers/s390/cio/vfio_ccw_async.c @@ -8,6 +8,7 @@ */ #include +#include #include "vfio_ccw_private.h" @@ -24,11 +25,20 @@ static ssize_t vfio_ccw_async_region_read(struct vfio_ccw_private *private, return -EINVAL; mutex_lock(&private->io_mutex); + + if (i >= private->num_regions) { + ret = -EINVAL; + goto out_unlock; + } + + i = array_index_nospec(i, private->num_regions); region = private->region[i].data; if (copy_to_user(buf, (void *)region + pos, count)) ret = -EFAULT; else ret = count; + +out_unlock: mutex_unlock(&private->io_mutex); return ret; } @@ -48,6 +58,12 @@ static ssize_t vfio_ccw_async_region_write(struct vfio_ccw_private *private, if (!mutex_trylock(&private->io_mutex)) return -EAGAIN; + if (i >= private->num_regions) { + ret = -EINVAL; + goto out_unlock; + } + + i = array_index_nospec(i, private->num_regions); region = private->region[i].data; if (copy_from_user((void *)region + pos, buf, count)) { ret = -EFAULT; diff --git a/drivers/s390/cio/vfio_ccw_chp.c b/drivers/s390/cio/vfio_ccw_chp.c index 38c176cf6295..f3015132d4b5 100644 --- a/drivers/s390/cio/vfio_ccw_chp.c +++ b/drivers/s390/cio/vfio_ccw_chp.c @@ -9,6 +9,7 @@ */ #include +#include #include #include "vfio_ccw_private.h" @@ -26,6 +27,13 @@ static ssize_t vfio_ccw_schib_region_read(struct vfio_ccw_private *private, return -EINVAL; mutex_lock(&private->io_mutex); + + if (i >= private->num_regions) { + ret = -EINVAL; + goto out; + } + + i = array_index_nospec(i, private->num_regions); region = private->region[i].data; if (cio_update_schib(sch)) { @@ -97,6 +105,12 @@ static ssize_t vfio_ccw_crw_region_read(struct vfio_ccw_private *private, list_del(&crw->next); mutex_lock(&private->io_mutex); + if (i >= private->num_regions) { + ret = -EINVAL; + goto out; + } + + i = array_index_nospec(i, private->num_regions); region = private->region[i].data; if (crw) @@ -109,6 +123,7 @@ static ssize_t vfio_ccw_crw_region_read(struct vfio_ccw_private *private, region->crw = 0; +out: mutex_unlock(&private->io_mutex); kfree(crw); diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c index 45ec722d25ea..032a1cdf4df7 100644 --- a/drivers/s390/cio/vfio_ccw_ops.c +++ b/drivers/s390/cio/vfio_ccw_ops.c @@ -243,6 +243,7 @@ static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev, return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); default: index -= VFIO_CCW_NUM_REGIONS; + index = array_index_nospec(index, private->num_regions); return private->region[index].ops->read(private, buf, count, ppos); } @@ -295,6 +296,7 @@ static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev, return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); default: index -= VFIO_CCW_NUM_REGIONS; + index = array_index_nospec(index, private->num_regions); return private->region[index].ops->write(private, buf, count, ppos); } @@ -338,11 +340,8 @@ static int vfio_ccw_mdev_ioctl_get_region_info(struct vfio_device *vdev, VFIO_CCW_NUM_REGIONS + private->num_regions) return -EINVAL; - info->index = array_index_nospec(info->index, - VFIO_CCW_NUM_REGIONS + - private->num_regions); - i = info->index - VFIO_CCW_NUM_REGIONS; + i = array_index_nospec(i, private->num_regions); info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); info->size = private->region[i].size; From 79c60b2c61105368dcc8444eb45847e21734f7c4 Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 28 Jul 2026 05:30:19 +0200 Subject: [PATCH 18/82] s390/vfio_ccw: Cancel existing workqueues The initialization of the io_work and crw_work workqueues begs the question of whether they should be un-initialized. Add the corresponding cleanup tags in _release_dev to ensure work isn't dispatched after the private struct is free'd. Suggested-by: Matthew Rosato Fixes: e5f84dbaea59 ("vfio: ccw: return I/O results asynchronously") Fixes: 3f02cb2fd9d2 ("vfio-ccw: Wire up the CRW irq and CRW region") Cc: stable@vger.kernel.org Reviewed-by: Matthew Rosato Signed-off-by: Eric Farman Signed-off-by: Christian Borntraeger --- drivers/s390/cio/vfio_ccw_ops.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c index 032a1cdf4df7..d361d1fde3a0 100644 --- a/drivers/s390/cio/vfio_ccw_ops.c +++ b/drivers/s390/cio/vfio_ccw_ops.c @@ -131,6 +131,13 @@ static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) container_of(vdev, struct vfio_ccw_private, vdev); struct vfio_ccw_crw *crw, *temp; + /* + * Ensure these work items are fully drained, so none can + * fire after being released. + */ + cancel_work_sync(&private->io_work); + cancel_work_sync(&private->crw_work); + list_for_each_entry_safe(crw, temp, &private->crw, next) { list_del(&crw->next); kfree(crw); @@ -202,6 +209,14 @@ static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) container_of(vdev, struct vfio_ccw_private, vdev); vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); + + /* + * Ensure these work items are drained, in the event the + * device is re-opened instead of released. + */ + cancel_work_sync(&private->io_work); + cancel_work_sync(&private->crw_work); + vfio_ccw_unregister_dev_regions(private); } From 0c11f61a876ed6fcca53d442ed3f33ea8362a0f9 Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 28 Jul 2026 05:30:20 +0200 Subject: [PATCH 19/82] s390/vfio_ccw: Move cp cleanup out of not operational The fsm_notoper() routine is called when the device has been lost, and is (by definition) no longer operational. Since this can happen asynchronously from the normal behavior of the driver, the cleanup may happen when holding other locks in the calling sequence (notably, the cio subchannel lock). Push the cleanup of the private->cp resources to a workqueue, where it can be done out from under that lock sequence and a future patch can safely manage the locking requirements. Fixes: 204b394a23ad ("vfio/ccw: Move FSM open/close to MDEV open/close") Cc: stable@vger.kernel.org Signed-off-by: Eric Farman Reviewed-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- drivers/s390/cio/vfio_ccw_drv.c | 9 +++++++++ drivers/s390/cio/vfio_ccw_fsm.c | 3 +-- drivers/s390/cio/vfio_ccw_ops.c | 13 +++++++++++++ drivers/s390/cio/vfio_ccw_private.h | 3 +++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c index 1a095085bc72..c197ad5ab580 100644 --- a/drivers/s390/cio/vfio_ccw_drv.c +++ b/drivers/s390/cio/vfio_ccw_drv.c @@ -125,6 +125,15 @@ void vfio_ccw_crw_todo(struct work_struct *work) eventfd_signal(private->crw_trigger); } +void vfio_ccw_notoper_todo(struct work_struct *work) +{ + struct vfio_ccw_private *private; + + private = container_of(work, struct vfio_ccw_private, notoper_work); + + cp_free(&private->cp); +} + /* * Css driver callbacks */ diff --git a/drivers/s390/cio/vfio_ccw_fsm.c b/drivers/s390/cio/vfio_ccw_fsm.c index 4d7988ea47ef..4d47a3c7b9a0 100644 --- a/drivers/s390/cio/vfio_ccw_fsm.c +++ b/drivers/s390/cio/vfio_ccw_fsm.c @@ -170,8 +170,7 @@ static void fsm_notoper(struct vfio_ccw_private *private, css_sched_sch_todo(sch, SCH_TODO_UNREG); private->state = VFIO_CCW_STATE_NOT_OPER; - /* This is usually handled during CLOSE event */ - cp_free(&private->cp); + queue_work(vfio_ccw_work_q, &private->notoper_work); } /* diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c index d361d1fde3a0..1df6d649565b 100644 --- a/drivers/s390/cio/vfio_ccw_ops.c +++ b/drivers/s390/cio/vfio_ccw_ops.c @@ -54,6 +54,7 @@ static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev) INIT_LIST_HEAD(&private->crw); INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo); INIT_WORK(&private->crw_work, vfio_ccw_crw_todo); + INIT_WORK(&private->notoper_work, vfio_ccw_notoper_todo); private->cp.guest_cp = kzalloc_objs(struct ccw1, CCWCHAIN_LEN_MAX); if (!private->cp.guest_cp) @@ -134,9 +135,16 @@ static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) /* * Ensure these work items are fully drained, so none can * fire after being released. + * + * notoper_work should have nothing to do here, because only + * open devices could have channel_program resources in use + * and those would be released during close. Nevertheless, + * call flush here as well to be certain anything that was + * allocated is freed. */ cancel_work_sync(&private->io_work); cancel_work_sync(&private->crw_work); + flush_work(&private->notoper_work); list_for_each_entry_safe(crw, temp, &private->crw, next) { list_del(&crw->next); @@ -213,9 +221,14 @@ static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) /* * Ensure these work items are drained, in the event the * device is re-opened instead of released. + * + * notoper_work needs to be given a chance to run if it + * is queued, so any memory associated with the channel + * program can be returned. */ cancel_work_sync(&private->io_work); cancel_work_sync(&private->crw_work); + flush_work(&private->notoper_work); vfio_ccw_unregister_dev_regions(private); } diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h index 0501d4bbcdbd..e2256402b089 100644 --- a/drivers/s390/cio/vfio_ccw_private.h +++ b/drivers/s390/cio/vfio_ccw_private.h @@ -102,6 +102,7 @@ struct vfio_ccw_parent { * @req_trigger: eventfd ctx for signaling userspace to return device * @io_work: work for deferral process of I/O handling * @crw_work: work for deferral process of CRW handling + * @notoper_work: work for deferred processing in not-operational state */ struct vfio_ccw_private { struct vfio_device vdev; @@ -125,11 +126,13 @@ struct vfio_ccw_private { struct eventfd_ctx *req_trigger; struct work_struct io_work; struct work_struct crw_work; + struct work_struct notoper_work; } __aligned(8); int vfio_ccw_sch_quiesce(struct subchannel *sch); void vfio_ccw_sch_io_todo(struct work_struct *work); void vfio_ccw_crw_todo(struct work_struct *work); +void vfio_ccw_notoper_todo(struct work_struct *work); extern struct mdev_driver vfio_ccw_mdev_driver; From 34f4feff3e90bd09308fad0974e97113b23b812a Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 28 Jul 2026 05:30:21 +0200 Subject: [PATCH 20/82] s390/vfio_ccw: Selectively expand io_mutex The io_mutex was defined to serialize the io_regions, but then has also sort of been associated with the I/O themselves because of the close relationship they share. With the handful of races that are possible, the choices are either to: A) expand the scope of io_mutex to close these remaining windows, or B) reduce the scope of io_mutex to just io_region, and introduce a new lock mechanism for the remaining I/O resources This patch implements A, since B brings with it a lot more interactions that would need to be tracked and kept in a correct hierarchy. It also takes advantage of the workqueue element for cp_free() that now gets called out of fsm_notoper(), which could be invoked out of an interrupt context and thus cannot acquire a mutex itself. Fixes: 4f76617378ee ("vfio-ccw: protect the I/O region") Cc: stable@vger.kernel.org Signed-off-by: Eric Farman Reviewed-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- drivers/s390/cio/vfio_ccw_chp.c | 2 +- drivers/s390/cio/vfio_ccw_cp.c | 8 +++++++- drivers/s390/cio/vfio_ccw_drv.c | 6 ++++-- drivers/s390/cio/vfio_ccw_fsm.c | 5 +++++ drivers/s390/cio/vfio_ccw_private.h | 3 ++- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/drivers/s390/cio/vfio_ccw_chp.c b/drivers/s390/cio/vfio_ccw_chp.c index f3015132d4b5..9269b54f5cfd 100644 --- a/drivers/s390/cio/vfio_ccw_chp.c +++ b/drivers/s390/cio/vfio_ccw_chp.c @@ -98,13 +98,13 @@ static ssize_t vfio_ccw_crw_region_read(struct vfio_ccw_private *private, if (pos + count > sizeof(*region)) return -EINVAL; + mutex_lock(&private->io_mutex); crw = list_first_entry_or_null(&private->crw, struct vfio_ccw_crw, next); if (crw) list_del(&crw->next); - mutex_lock(&private->io_mutex); if (i >= private->num_regions) { ret = -EINVAL; goto out; diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c index 5ef082b8289a..58722c4baa25 100644 --- a/drivers/s390/cio/vfio_ccw_cp.c +++ b/drivers/s390/cio/vfio_ccw_cp.c @@ -977,17 +977,23 @@ void cp_update_scsw(struct channel_program *cp, union scsw *scsw) */ bool cp_iova_pinned(struct channel_program *cp, u64 iova, u64 length) { + struct vfio_ccw_private *private = + container_of(cp, struct vfio_ccw_private, cp); struct ccwchain *chain; int i; if (!cp->initialized) return false; + mutex_lock(&private->io_mutex); list_for_each_entry(chain, &cp->ccwchain_list, next) { for (i = 0; i < chain->ch_len; i++) - if (page_array_iova_pinned(&chain->ch_pa[i], iova, length)) + if (page_array_iova_pinned(&chain->ch_pa[i], iova, length)) { + mutex_unlock(&private->io_mutex); return true; + } } + mutex_unlock(&private->io_mutex); return false; } diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c index c197ad5ab580..757ff5b2556e 100644 --- a/drivers/s390/cio/vfio_ccw_drv.c +++ b/drivers/s390/cio/vfio_ccw_drv.c @@ -91,6 +91,7 @@ void vfio_ccw_sch_io_todo(struct work_struct *work) is_final = !(scsw_actl(&irb->scsw) & (SCSW_ACTL_DEVACT | SCSW_ACTL_SCHACT)); + mutex_lock(&private->io_mutex); if (scsw_is_solicited(&irb->scsw)) { cp_update_scsw(&private->cp, &irb->scsw); if (is_final && private->state == VFIO_CCW_STATE_CP_PENDING) { @@ -98,9 +99,7 @@ void vfio_ccw_sch_io_todo(struct work_struct *work) cp_is_finished = true; } } - mutex_lock(&private->io_mutex); memcpy(private->io_region->irb_area, irb, sizeof(*irb)); - mutex_unlock(&private->io_mutex); /* * Reset to IDLE only if processing of a channel program @@ -110,6 +109,7 @@ void vfio_ccw_sch_io_todo(struct work_struct *work) */ if (cp_is_finished) private->state = VFIO_CCW_STATE_IDLE; + mutex_unlock(&private->io_mutex); if (private->io_trigger) eventfd_signal(private->io_trigger); @@ -131,7 +131,9 @@ void vfio_ccw_notoper_todo(struct work_struct *work) private = container_of(work, struct vfio_ccw_private, notoper_work); + mutex_lock(&private->io_mutex); cp_free(&private->cp); + mutex_unlock(&private->io_mutex); } /* diff --git a/drivers/s390/cio/vfio_ccw_fsm.c b/drivers/s390/cio/vfio_ccw_fsm.c index 4d47a3c7b9a0..5fd94e9d5c61 100644 --- a/drivers/s390/cio/vfio_ccw_fsm.c +++ b/drivers/s390/cio/vfio_ccw_fsm.c @@ -170,6 +170,7 @@ static void fsm_notoper(struct vfio_ccw_private *private, css_sched_sch_todo(sch, SCH_TODO_UNREG); private->state = VFIO_CCW_STATE_NOT_OPER; + /* This routine could be called from IRQ context, so defer */ queue_work(vfio_ccw_work_q, &private->notoper_work); } @@ -409,7 +410,11 @@ static void fsm_close(struct vfio_ccw_private *private, private->state = VFIO_CCW_STATE_STANDBY; spin_unlock_irq(&sch->lock); + + mutex_lock(&private->io_mutex); cp_free(&private->cp); + mutex_unlock(&private->io_mutex); + return; err_unlock: diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h index e2256402b089..739121116ab6 100644 --- a/drivers/s390/cio/vfio_ccw_private.h +++ b/drivers/s390/cio/vfio_ccw_private.h @@ -88,7 +88,8 @@ struct vfio_ccw_parent { * @state: internal state of the device * @completion: synchronization helper of the I/O completion * @io_region: MMIO region to input/output I/O arguments/results - * @io_mutex: protect against concurrent update of I/O regions + * @io_mutex: protect against concurrent update of I/O resources + * and @cp lifecycle * @region: additional regions for other subchannel operations * @cmd_region: MMIO region for asynchronous I/O commands other than START * @schib_region: MMIO region for SCHIB information From 16b0798024c0e9117e395829ddbbe70981c79d9c Mon Sep 17 00:00:00 2001 From: Eric Farman Date: Tue, 28 Jul 2026 05:30:22 +0200 Subject: [PATCH 21/82] s390/vfio_ccw: Implement a crw lock Unlike the channel_program struct, which covers synchronous I/O submissions and asynchronous interrupts, the CRW region relies exclusively on asynchronous events coming from hardware. Implement a lock to manage the list of those payloads, to ensure they are read cohesively. Fixes: 3f02cb2fd9d2 ("vfio-ccw: Wire up the CRW irq and CRW region") Cc: stable@vger.kernel.org Reviewed-by: Matthew Rosato Reviewed-by: Farhan Ali Signed-off-by: Eric Farman Signed-off-by: Christian Borntraeger --- drivers/s390/cio/vfio_ccw_chp.c | 26 ++++++++++++++++---------- drivers/s390/cio/vfio_ccw_drv.c | 6 ++++++ drivers/s390/cio/vfio_ccw_ops.c | 4 ++++ drivers/s390/cio/vfio_ccw_private.h | 4 ++++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/drivers/s390/cio/vfio_ccw_chp.c b/drivers/s390/cio/vfio_ccw_chp.c index 9269b54f5cfd..7708eb4d6de0 100644 --- a/drivers/s390/cio/vfio_ccw_chp.c +++ b/drivers/s390/cio/vfio_ccw_chp.c @@ -93,18 +93,13 @@ static ssize_t vfio_ccw_crw_region_read(struct vfio_ccw_private *private, loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; struct ccw_crw_region *region; struct vfio_ccw_crw *crw; + unsigned long flags; int ret; if (pos + count > sizeof(*region)) return -EINVAL; mutex_lock(&private->io_mutex); - crw = list_first_entry_or_null(&private->crw, - struct vfio_ccw_crw, next); - - if (crw) - list_del(&crw->next); - if (i >= private->num_regions) { ret = -EINVAL; goto out; @@ -113,6 +108,16 @@ static ssize_t vfio_ccw_crw_region_read(struct vfio_ccw_private *private, i = array_index_nospec(i, private->num_regions); region = private->region[i].data; + spin_lock_irqsave(&private->crw_lock, flags); + crw = list_first_entry_or_null(&private->crw, + struct vfio_ccw_crw, next); + + if (crw) + list_del(&crw->next); + + /* Drop CRW lock while copying to userspace */ + spin_unlock_irqrestore(&private->crw_lock, flags); + if (crw) memcpy(®ion->crw, &crw->crw, sizeof(region->crw)); @@ -122,15 +127,16 @@ static ssize_t vfio_ccw_crw_region_read(struct vfio_ccw_private *private, ret = count; region->crw = 0; - -out: - mutex_unlock(&private->io_mutex); - kfree(crw); /* Notify the guest if more CRWs are on our queue */ + spin_lock_irqsave(&private->crw_lock, flags); if (!list_empty(&private->crw) && private->crw_trigger) eventfd_signal(private->crw_trigger); + spin_unlock_irqrestore(&private->crw_lock, flags); + +out: + mutex_unlock(&private->io_mutex); return ret; } diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c index 757ff5b2556e..ab6b518cc353 100644 --- a/drivers/s390/cio/vfio_ccw_drv.c +++ b/drivers/s390/cio/vfio_ccw_drv.c @@ -118,11 +118,14 @@ void vfio_ccw_sch_io_todo(struct work_struct *work) void vfio_ccw_crw_todo(struct work_struct *work) { struct vfio_ccw_private *private; + unsigned long flags; private = container_of(work, struct vfio_ccw_private, crw_work); + spin_lock_irqsave(&private->crw_lock, flags); if (!list_empty(&private->crw) && private->crw_trigger) eventfd_signal(private->crw_trigger); + spin_unlock_irqrestore(&private->crw_lock, flags); } void vfio_ccw_notoper_todo(struct work_struct *work) @@ -286,6 +289,7 @@ static void vfio_ccw_queue_crw(struct vfio_ccw_private *private, unsigned int rsid) { struct vfio_ccw_crw *crw; + unsigned long flags; /* * If unable to allocate a CRW, just drop the event and @@ -303,7 +307,9 @@ static void vfio_ccw_queue_crw(struct vfio_ccw_private *private, crw->crw.erc = erc; crw->crw.rsid = rsid; + spin_lock_irqsave(&private->crw_lock, flags); list_add_tail(&crw->next, &private->crw); + spin_unlock_irqrestore(&private->crw_lock, flags); queue_work(vfio_ccw_work_q, &private->crw_work); } diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c index 1df6d649565b..5ce91285c7d5 100644 --- a/drivers/s390/cio/vfio_ccw_ops.c +++ b/drivers/s390/cio/vfio_ccw_ops.c @@ -55,6 +55,7 @@ static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev) INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo); INIT_WORK(&private->crw_work, vfio_ccw_crw_todo); INIT_WORK(&private->notoper_work, vfio_ccw_notoper_todo); + spin_lock_init(&private->crw_lock); private->cp.guest_cp = kzalloc_objs(struct ccw1, CCWCHAIN_LEN_MAX); if (!private->cp.guest_cp) @@ -131,6 +132,7 @@ static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) struct vfio_ccw_private *private = container_of(vdev, struct vfio_ccw_private, vdev); struct vfio_ccw_crw *crw, *temp; + unsigned long flags; /* * Ensure these work items are fully drained, so none can @@ -146,10 +148,12 @@ static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) cancel_work_sync(&private->crw_work); flush_work(&private->notoper_work); + spin_lock_irqsave(&private->crw_lock, flags); list_for_each_entry_safe(crw, temp, &private->crw, next) { list_del(&crw->next); kfree(crw); } + spin_unlock_irqrestore(&private->crw_lock, flags); kmem_cache_free(vfio_ccw_crw_region, private->crw_region); kmem_cache_free(vfio_ccw_schib_region, private->schib_region); diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h index 739121116ab6..3bd0171d38d0 100644 --- a/drivers/s390/cio/vfio_ccw_private.h +++ b/drivers/s390/cio/vfio_ccw_private.h @@ -98,6 +98,8 @@ struct vfio_ccw_parent { * @cp: channel program for the current I/O operation * @irb: irb info received from interrupt * @scsw: scsw info + * @crw_lock: serialization of CRW list information + * @crw: list of Channel Report Word elements * @io_trigger: eventfd ctx for signaling userspace I/O results * @crw_trigger: eventfd ctx for signaling userspace CRW information * @req_trigger: eventfd ctx for signaling userspace to return device @@ -120,6 +122,8 @@ struct vfio_ccw_private { struct channel_program cp; struct irb irb; union scsw scsw; + + spinlock_t crw_lock; struct list_head crw; struct eventfd_ctx *io_trigger; From 496e0f706b8ad7831bf7189ff9deb8701d2ebc2b Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:28 +0200 Subject: [PATCH 22/82] KVM: s390: Fix unlikely NULL gmap dereference When creating a new vCPU, kvm_vm_ioctl_create_vcpu() will call kvm_arch_vcpu_postcreate() after the file descriptor for the new vCPU has been created. The new file descriptor has not been returned yet, but a malicious userspace program could try to guess it. If a malicious userspace program manages to start the newly created vCPU before kvm_arch_vcpu_postcreate() is called, __vcpu_run() will try to dereference vcpu->arch.gmap and trigger a NULL pointer dereference. Fix this by adding a new field to struct kvm_vcpu_arch to keep track of the initialization status of the vCPU. Refuse to run a vCPU that is not fully initialized. Fixes: dafd032a15f8 ("KVM: s390: move vcpu specific initalization to a later point") Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Reviewed-by: Steffen Eiden Reviewed-by: Janosch Frank Reviewed-by: Christian Borntraeger Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-2-imbrenda@linux.ibm.com> --- arch/s390/include/asm/kvm_host.h | 1 + arch/s390/kvm/kvm-s390.c | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index c172f9b212d1..b4182ca4435f 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -440,6 +440,7 @@ struct kvm_vcpu_arch { bool skey_enabled; /* Indicator if the access registers have been loaded from guest */ bool acrs_loaded; + bool initialized; struct kvm_s390_pv_vcpu pv; union diag318_info diag318_info; struct kvm_s390_mmu_cache *mc; diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 150b5dd2170e..f86b4b0b356f 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3613,6 +3613,9 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0 || vcpu->kvm->arch.user_operexec) vcpu->arch.sie_block->ictl |= ICTL_OPEREXC; + + /* Pairs with smp_load_acquire() in kvm_arch_vcpu_ioctl_run() and kvm_arch_vcpu_ioctl() */ + smp_store_release(&vcpu->arch.initialized, true); } static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr) @@ -5039,6 +5042,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS) return -EINVAL; + /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */ + if (!smp_load_acquire(&vcpu->arch.initialized)) + return -EINVAL; + vcpu_load(vcpu); if (guestdbg_exit_pending(vcpu)) { @@ -5523,6 +5530,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp, long r; u16 rc, rrc; + /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */ + if (!smp_load_acquire(&vcpu->arch.initialized)) + return -EINVAL; + vcpu_load(vcpu); switch (ioctl) { From f8e370058e9e1396fea1a8d11906fa92ec9bad88 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:29 +0200 Subject: [PATCH 23/82] KVM: s390: Do not free SCA if it was not allocated If VM creation fails early in kvm_arch_init_vm(), the cleanup code tries to free up the SCA, even though the address is 0. Due to using free_pages_exact(), only the first page is skipped, accidentally freeing pages 1, 2, and 3. Fix by checking whether the pointer is NULL before attempting to free the SCA in sca_dispose(). Fixes: e72753ed1267 ("KVM: s390: Use ESCA instead of BSCA at VM init") Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-3-imbrenda@linux.ibm.com> Reviewed-by: Christian Borntraeger Reviewed-by: Janosch Frank --- arch/s390/kvm/kvm-s390.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index f86b4b0b356f..1b3290a5ad1a 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3247,7 +3247,8 @@ static void kvm_s390_crypto_init(struct kvm *kvm) static void sca_dispose(struct kvm *kvm) { - free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca)); + if (kvm->arch.sca) + free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca)); kvm->arch.sca = NULL; } From b050f741fd0d636f2daab72b74aeccea97abdb5f Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:30 +0200 Subject: [PATCH 24/82] KVM: s390: Fix kvm_s390_vcpu_unsetup_cmma() In some cases kvm_s390_vcpu_unsetup_cmma() can be called with a 0 cbrlo; in such cases, if running with V != R, free_page() will attempt to free physical page 0. Fix by freeing cbrlo only if it's non-zero. Fixes: b31605c12f4e ("KVM: s390: make cmma usage conditionally") Fixes: 29b40f105ec8 ("KVM: s390: protvirt: Add initial vm and cpu lifecycle handling") Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-4-imbrenda@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 1b3290a5ad1a..9be27db0a21e 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3678,7 +3678,8 @@ static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu) void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) { - free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo)); + if (vcpu->arch.sie_block->cbrlo) + free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo)); vcpu->arch.sie_block->cbrlo = 0; } From a0496b40a4ab346052aaa59a5163c10224dda01f Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:31 +0200 Subject: [PATCH 25/82] KVM: s390: Fix overclearing ESCA in case of error If an attempt is made to create a vCPU with an already existing ID, the duplicated vCPU will be destroyed. When destroying a vCPU, its ESCA entry will be cleared. In the above scenario, the spurious duplicate vCPU is destroyed, but the ESCA entry corresponding to the original vCPU is cleared. Fix by skipping clearing the ESCA entry if the vCPU creation was not successful, i.e. if the vcpu->arch.initialized is still zero. Reviewed-by: Janosch Frank Signed-off-by: Claudio Imbrenda Fixes: abf4a71ed95f ("KVM: s390: Unlink vcpu on destroy - v2") [ Added Fixes tag while picking -- Claudio ] Message-ID: <20260803124040.126471-5-imbrenda@linux.ibm.com> --- arch/s390/kvm/interrupt.c | 2 +- arch/s390/kvm/kvm-s390.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 7514d9e2403c..bb4e72466043 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -86,7 +86,7 @@ static void sca_clear_ext_call(struct kvm_vcpu *vcpu) struct esca_block *sca = vcpu->kvm->arch.sca; union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; - if (!kvm_s390_use_sca_entries()) + if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized) return; kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 9be27db0a21e..5b2727d7dfd1 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3462,7 +3462,7 @@ static void sca_del_vcpu(struct kvm_vcpu *vcpu) { struct esca_block *sca = vcpu->kvm->arch.sca; - if (!kvm_s390_use_sca_entries()) + if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized) return; clear_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn); From d301ade41831e746783bdb697fae3afba47841ba Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:32 +0200 Subject: [PATCH 26/82] KVM: s390: ucontrol: Fix sca_clear_ext_call() When cleaning up a UCONTROL VM, sca_clear_ext_call() will touch memory outside of the allocated ESCA block, and UCONTROL VMs don't even use ESCA. Fix by not touching ESCA for UCONTROL VMs, and fence the KVM_S390_INTERRUPT ioctl altogether. Add extra checks in sca_ext_call_pending() and sca_inject_ext_call() to make sure UCONTROL VMs won't touch ESCA. Fencing does not cause regressions with userspace, since UCONTROL VMs never used KVM_S390_INTERRUPT ioctls. Fixes: 7d43bafcff17 ("KVM: s390: Make provisions for ESCA utilization") Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-6-imbrenda@linux.ibm.com> --- arch/s390/kvm/interrupt.c | 19 ++++++++++++++----- arch/s390/kvm/kvm-s390.c | 5 +++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index bb4e72466043..8f24bcd1a6d3 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -45,13 +45,16 @@ static struct kvm_s390_gib *gib; static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) { struct esca_block *sca = vcpu->kvm->arch.sca; - union esca_sigp_ctrl sigp_ctrl = sca->cpu[vcpu->vcpu_id].sigp_ctrl; + union esca_sigp_ctrl sigp_ctrl; if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND)) return 0; + if (kvm_is_ucontrol(vcpu->kvm)) + return 0; BUG_ON(!kvm_s390_use_sca_entries()); + sigp_ctrl = sca->cpu[vcpu->vcpu_id].sigp_ctrl; if (src_id) *src_id = sigp_ctrl.scn; @@ -60,13 +63,16 @@ static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) { - struct esca_block *sca = vcpu->kvm->arch.sca; - union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; union esca_sigp_ctrl old_val, new_val = {.scn = src_id, .c = 1}; + struct esca_block *sca = vcpu->kvm->arch.sca; + union esca_sigp_ctrl *sigp_ctrl; int expect, rc; BUG_ON(!kvm_s390_use_sca_entries()); + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; + sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; old_val = READ_ONCE(*sigp_ctrl); old_val.c = 0; @@ -84,10 +90,13 @@ static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) static void sca_clear_ext_call(struct kvm_vcpu *vcpu) { struct esca_block *sca = vcpu->kvm->arch.sca; - union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; + union esca_sigp_ctrl *sigp_ctrl; - if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized) + if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized || kvm_is_ucontrol(vcpu->kvm)) return; + + /* Initialize after the above check, to prevent going out of bounds */ + sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND); WRITE_ONCE(sigp_ctrl->value, 0); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 5b2727d7dfd1..21574f57be72 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -2934,6 +2934,9 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) case KVM_S390_INTERRUPT: { struct kvm_s390_interrupt s390int; + r = -EINVAL; + if (kvm_is_ucontrol(kvm)) + break; r = -EFAULT; if (copy_from_user(&s390int, argp, sizeof(s390int))) break; @@ -5456,6 +5459,8 @@ long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, struct kvm_s390_interrupt s390int; struct kvm_s390_irq s390irq = {}; + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; if (copy_from_user(&s390int, argp, sizeof(s390int))) return -EFAULT; if (s390int_to_s390irq(&s390int, &s390irq)) From d699986f11bf854bddecd6b1bfdf53e05ef945d6 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:33 +0200 Subject: [PATCH 27/82] KVM: s390: Fix leaking of PGM_ADDRESSING to userspace If kvm_s390_set_cmma_bits() is asked to set CMMA values outside of a memslot, PGM_ADDRESSING (5) is returned, instead of a negative error value. Same issue with kvm_s390_{g,s}et_skeys(), kvm_s390_keyop(), and dat_reset_reference_bit(). Fix by returning -EFAULT whenever the return value would be > 0, which is consistent with the behaviour before the gmap rewrite. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-7-imbrenda@linux.ibm.com> --- arch/s390/kvm/dat.c | 16 ++++++++++------ arch/s390/kvm/dat.h | 2 +- arch/s390/kvm/kvm-s390.c | 16 ++++++++-------- arch/s390/kvm/priv.c | 5 +++-- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index ed4259d17629..171b61959908 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -755,13 +755,15 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf return rc; } -int dat_reset_reference_bit(union asce asce, gfn_t gfn) +int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey) { union pgste pgste, old; union crste *crstep; union pte *ptep; int rc; + skey->skey = 0; + rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; @@ -771,21 +773,23 @@ int dat_reset_reference_bit(union asce asce, gfn_t gfn) if (!crste.h.fc || !crste.s.fc1.pr) return 0; - return page_reset_referenced(large_crste_to_phys(*crstep, gfn)); + skey->skey = page_reset_referenced(large_crste_to_phys(*crstep, gfn)) << 1; + return 0; } old = pgste_get_lock(ptep); pgste = old; if (!ptep->h.i) { - rc = page_reset_referenced(pte_origin(*ptep)); - pgste.hr = rc >> 1; + skey->skey = page_reset_referenced(pte_origin(*ptep)) << 1; + pgste.hr = skey->r; } - rc |= (pgste.gr << 1) | pgste.gc; + skey->r |= pgste.gr; + skey->c |= pgste.gc; pgste.gr = 0; dat_update_ptep_sd(old, pgste, ptep); pgste_set_unlock(ptep, pgste); - return rc; + return 0; } static long dat_reset_skeys_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h index fad605305e05..141ee7b9f019 100644 --- a/arch/s390/kvm/dat.h +++ b/arch/s390/kvm/dat.h @@ -537,7 +537,7 @@ int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gf union skey skey, bool nq); int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn, union skey skey, union skey *oldkey, bool nq, bool mr, bool mc); -int dat_reset_reference_bit(union asce asce, gfn_t gfn); +int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey); long dat_reset_skeys(union asce asce, gfn_t start); unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 21574f57be72..e162efaa35b8 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -571,7 +571,7 @@ static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op switch (op) { case KVM_S390_KEYOP_SSKE: r = dat_cond_set_storage_key(mc, asce, gfn, skey, &skey, 0, 0, 0); - if (r >= 0) + if (r == 0 || r == 1) return skey.skey; break; case KVM_S390_KEYOP_ISKE: @@ -580,14 +580,14 @@ static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op return skey.skey; break; case KVM_S390_KEYOP_RRBE: - r = dat_reset_reference_bit(asce, gfn); - if (r > 0) - return r << 1; + r = dat_reset_reference_bit(asce, gfn, &skey); + if (!r) + return skey.skey; break; default: return -EINVAL; } - return r; + return r > 0 ? -EFAULT : r; } /* Section: device related */ @@ -2214,7 +2214,7 @@ static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) } kvfree(keys); - return r; + return r <= 0 ? r : -EFAULT; } static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) @@ -2276,7 +2276,7 @@ static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) kvm_s390_free_mmu_cache(mc); out: kvfree(keys); - return r; + return r <= 0 ? r : -EFAULT; } /* @@ -2386,7 +2386,7 @@ static int kvm_s390_set_cmma_bits(struct kvm *kvm, set_bit(GMAP_FLAG_USES_CMM, &kvm->arch.gmap->flags); - return r; + return r <= 0 ? r : -EFAULT; } /** diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index ad0ddc433a73..ea5a99537346 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -289,6 +289,7 @@ static int handle_iske(struct kvm_vcpu *vcpu) static int handle_rrbe(struct kvm_vcpu *vcpu) { unsigned long gaddr; + union skey skey; int reg1, reg2; int rc; @@ -307,12 +308,12 @@ static int handle_rrbe(struct kvm_vcpu *vcpu) gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); gaddr = kvm_s390_real_to_abs(vcpu, gaddr); scoped_guard(read_lock, &vcpu->kvm->mmu_lock) - rc = dat_reset_reference_bit(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr)); + rc = dat_reset_reference_bit(vcpu->arch.gmap->asce, gpa_to_gfn(gaddr), &skey); if (rc > 0) return kvm_s390_inject_program_int(vcpu, rc); if (rc < 0) return rc; - kvm_s390_set_psw_cc(vcpu, rc); + kvm_s390_set_psw_cc(vcpu, (skey.skey >> 1) & 3); return 0; } From 4db7207052874f13dec1e9be384cca279c6693e8 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:34 +0200 Subject: [PATCH 28/82] KVM: s390: Fix race in __do_essa() An unlikely race between __do_essa() and kvm_s390_vm_start_migration(), kvm_s390_vm_stop_migration(), or dat_get_cmma() was possible. Fix by locking kvm->slots_arch_lock. Since this is not a hot path, the overhead of an additional mutex is negligible. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-8-imbrenda@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 20 ++++++++++---------- arch/s390/kvm/priv.c | 5 +++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index e162efaa35b8..e5c5e9f61cb2 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -1219,8 +1219,8 @@ static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req) /* * Must be called with kvm->srcu held to avoid races on memslots, and with - * kvm->slots_lock to avoid races with ourselves, kvm_s390_vm_stop_migration(), - * and kvm_s390_get_cmma_bits(). + * kvm->slots_arch_lock to avoid races with ourselves, + * kvm_s390_vm_stop_migration(), and kvm_s390_get_cmma_bits(). */ static int kvm_s390_vm_start_migration(struct kvm *kvm) { @@ -1265,7 +1265,7 @@ static int kvm_s390_vm_start_migration(struct kvm *kvm) } /* - * Must be called with kvm->slots_lock to avoid races with ourselves, + * Must be called with kvm->slots_arch_lock to avoid races with ourselves, * kvm_s390_vm_start_migration() and kvm_s390_get_cmma_bits(). */ static int kvm_s390_vm_stop_migration(struct kvm *kvm) @@ -1300,7 +1300,9 @@ static int kvm_s390_vm_set_migration(struct kvm *kvm, { int res = -ENXIO; - mutex_lock(&kvm->slots_lock); + guard(srcu)(&kvm->srcu); + guard(mutex)(&kvm->slots_arch_lock); + switch (attr->attr) { case KVM_S390_VM_MIGRATION_START: res = kvm_s390_vm_start_migration(kvm); @@ -1311,7 +1313,6 @@ static int kvm_s390_vm_set_migration(struct kvm *kvm, default: break; } - mutex_unlock(&kvm->slots_lock); return res; } @@ -3001,9 +3002,8 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) r = -EFAULT; if (copy_from_user(&args, argp, sizeof(args))) break; - mutex_lock(&kvm->slots_lock); - r = kvm_s390_get_cmma_bits(kvm, &args); - mutex_unlock(&kvm->slots_lock); + scoped_guard(mutex, &kvm->slots_arch_lock) + r = kvm_s390_get_cmma_bits(kvm, &args); if (!r) { r = copy_to_user(argp, &args, sizeof(args)); if (r) @@ -3017,9 +3017,9 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) r = -EFAULT; if (copy_from_user(&args, argp, sizeof(args))) break; - mutex_lock(&kvm->slots_lock); + mutex_lock(&kvm->slots_arch_lock); r = kvm_s390_set_cmma_bits(kvm, &args); - mutex_unlock(&kvm->slots_lock); + mutex_unlock(&kvm->slots_arch_lock); break; } case KVM_S390_PV_COMMAND: { diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index ea5a99537346..b1ba24c346ef 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -1261,8 +1261,9 @@ static int handle_essa(struct kvm_vcpu *vcpu) /* Retry the ESSA instruction */ kvm_s390_retry_instr(vcpu); } else { - scoped_guard(read_lock, &vcpu->kvm->mmu_lock) - i = __do_essa(vcpu, orc); + scoped_guard(mutex, &vcpu->kvm->slots_arch_lock) + scoped_guard(read_lock, &vcpu->kvm->mmu_lock) + i = __do_essa(vcpu, orc); if (i < 0) return i; /* Account for the possible extra cbrl entry */ From ec215346270512db60478e45e8dff2de950225aa Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:35 +0200 Subject: [PATCH 29/82] KVM: s390: cmma: Fix dirty tracking when removing memslot When a memslot is removed, all ptes that mapped the slot are cleared or even deallocated. If this happens while the system is in migration mode, and if cmma-dirty pages are removed, the cmma-dirty counter will not reflect reality. Fix by appropriately decrementing the cmma-dirty counter when removing a memslot. Opportunistically improve kvm_arch_commit_memory_region() to use __free() for the struct kvm_s390_mmu_cache. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-9-imbrenda@linux.ibm.com> --- arch/s390/kvm/dat.c | 7 ++++++- arch/s390/kvm/kvm-s390.c | 25 +++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 171b61959908..3f2d6e8902d7 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -850,6 +850,7 @@ static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_wal struct slot_priv *p = walk->priv; union crste dummy = { .val = p->token }; union pte new_pte, pte = READ_ONCE(*ptep); + union pgste pgste; new_pte = _PTE_TOK(dummy.tok.type, dummy.tok.par); @@ -857,7 +858,11 @@ static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_wal if (pte.val == new_pte.val) return 0; - dat_ptep_xchg(ptep, new_pte, gfn, walk->asce, false); + pgste = pgste_get_lock(ptep); + pgste = __dat_ptep_xchg(ptep, pgste, new_pte, gfn, walk->asce, false); + pgste.cmma_d = 0; + pgste_set_unlock(ptep, pgste); + return 0; } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index e5c5e9f61cb2..ba811f0673d1 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -5812,14 +5812,30 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, return 0; } +static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) +{ + union pgste pgste; + + pgste = pgste_get_lock(ptep); + if (pgste.cmma_d) { + pgste.cmma_d = 0; + atomic64_dec(walk->priv); + } + pgste_set_unlock(ptep, pgste); + return 0; +} + void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { - struct kvm_s390_mmu_cache *mc = NULL; + const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, }; + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; int rc = 0; + guard(mutex)(&kvm->slots_arch_lock); + if (change == KVM_MR_FLAGS_ONLY) return; @@ -5830,6 +5846,12 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, } scoped_guard(write_lock, &kvm->mmu_lock) { + if (kvm->arch.migration_mode && kvm->arch.use_cmma && old) { + _dat_walk_gfn_range(old->base_gfn, old->base_gfn + old->npages, + kvm->arch.gmap->asce, &ops, DAT_WALK_IGN_HOLES, + &kvm->arch.cmma_dirty_pages); + } + switch (change) { case KVM_MR_DELETE: rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); @@ -5851,7 +5873,6 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, out: if (rc) pr_warn("failed to commit memory region\n"); - kvm_s390_free_mmu_cache(mc); return; } From dab62d218754e00a22aff45d2b0116c5ee30cbd2 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:36 +0200 Subject: [PATCH 30/82] KVM: s390: ucontrol: Add missing locking around gmap_remove_child() gmap_remove_child() needs to be called while holding the children_lock of the parent gmap. This was not the case in the error handling path of kvm_arch_vcpu_create() for UCONTROL guests. Fix by adding the missing lock. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Reviewed-by: Steffen Eiden Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-10-imbrenda@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index ba811f0673d1..2741ca323ede 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3875,7 +3875,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) out_ucontrol_uninit: if (kvm_is_ucontrol(vcpu->kvm)) { + spin_lock(&vcpu->kvm->arch.gmap->children_lock); gmap_remove_child(vcpu->arch.gmap); + spin_unlock(&vcpu->kvm->arch.gmap->children_lock); vcpu->arch.gmap = gmap_put(vcpu->arch.gmap); } out_free_sie_block: From e4d678900a1ae66112812fe4d2aff16044ca37f2 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:37 +0200 Subject: [PATCH 31/82] KVM: s390: Free the mmu cache when kvm_arch_vcpu_create() fails The mmu cache is the first thing that is allocated in kvm_arch_vcpu_create(), but in case of failure it was not freed. Fix by freeing the mmu cache in case of failure. Refactor kvm_arch_vcpu_create() to use scope-based cleanup instead of gotos. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-11-imbrenda@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 2741ca323ede..9b8a35e7dc82 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3800,21 +3800,21 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) return 0; } +DEFINE_FREE(sie_page, struct sie_page *, if (_T) free_page((unsigned long)(_T))) + int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) { - struct sie_page *sie_page; + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + struct sie_page *sie_page __free(sie_page) = NULL; int rc; BUILD_BUG_ON(sizeof(struct sie_page) != 4096); - vcpu->arch.mc = kvm_s390_new_mmu_cache(); - if (!vcpu->arch.mc) + mc = kvm_s390_new_mmu_cache(); + if (!mc) return -ENOMEM; sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT); - if (!sie_page) { - kvm_s390_free_mmu_cache(vcpu->arch.mc); - vcpu->arch.mc = NULL; + if (!sie_page) return -ENOMEM; - } vcpu->arch.sie_block = &sie_page->sie_block; vcpu->arch.sie_block->itdba = virt_to_phys(&sie_page->itdb); @@ -3856,10 +3856,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS; if (kvm_is_ucontrol(vcpu->kvm)) { - rc = -ENOMEM; vcpu->arch.gmap = gmap_new_child(vcpu->kvm->arch.gmap, -1UL); if (!vcpu->arch.gmap) - goto out_free_sie_block; + return -ENOMEM; } VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%p, sie block at 0x%p", @@ -3867,22 +3866,19 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block); rc = kvm_s390_vcpu_setup(vcpu); - if (rc) - goto out_ucontrol_uninit; + if (rc) { + if (kvm_is_ucontrol(vcpu->kvm)) { + scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock) + gmap_remove_child(vcpu->arch.gmap); + vcpu->arch.gmap = gmap_put(vcpu->arch.gmap); + } + return rc; + } + vcpu->arch.mc = no_free_ptr(mc); + sie_page = NULL; kvm_s390_update_topology_change_report(vcpu->kvm, 1); return 0; - -out_ucontrol_uninit: - if (kvm_is_ucontrol(vcpu->kvm)) { - spin_lock(&vcpu->kvm->arch.gmap->children_lock); - gmap_remove_child(vcpu->arch.gmap); - spin_unlock(&vcpu->kvm->arch.gmap->children_lock); - vcpu->arch.gmap = gmap_put(vcpu->arch.gmap); - } -out_free_sie_block: - free_page((unsigned long)(vcpu->arch.sie_block)); - return rc; } int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) From 216c5289dd66d9cde6b42cbe68ee8dec5669678a Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:38 +0200 Subject: [PATCH 32/82] KVM: s390: Return -EINTR if a signal is pending while faulting-in If a fatal signal is pending while trying to fault-in a page, return -EINTR instead of -EAGAIN. Also fix unpack_one() to handle -EINTR properly. Fixes: e907ae530133 ("KVM: s390: Add helper functions for fault handling") Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-12-imbrenda@linux.ibm.com> --- arch/s390/kvm/faultin.c | 6 +++--- arch/s390/kvm/pv.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/s390/kvm/faultin.c b/arch/s390/kvm/faultin.c index fee80047bd94..3cc45f7f5b2d 100644 --- a/arch/s390/kvm/faultin.c +++ b/arch/s390/kvm/faultin.c @@ -91,9 +91,9 @@ int kvm_s390_faultin_gfn(struct kvm_vcpu *vcpu, struct kvm *kvm, struct guest_fa /* Access outside memory, addressing exception. */ if (is_noslot_pfn(f->pfn)) return PGM_ADDRESSING; - /* Signal pending: try again. */ - if (f->pfn == KVM_PFN_ERR_SIGPENDING) - return -EAGAIN; + /* Fatal signal pending: bail out. */ + if (is_sigpending_pfn(f->pfn)) + return -EINTR; /* Check if it's read-only memory; don't try to actually handle that case. */ if (f->pfn == KVM_PFN_ERR_RO_FAULT) return -EOPNOTSUPP; diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c index 1beacc841ca8..dc204b521052 100644 --- a/arch/s390/kvm/pv.c +++ b/arch/s390/kvm/pv.c @@ -809,7 +809,7 @@ static int unpack_one(struct kvm *kvm, unsigned long addr, u64 tweak, return -EAGAIN; } - if (ret && ret != -EAGAIN) + if (ret && ret != -EAGAIN && ret != -EINTR) KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx with rc %x rrc %x", uvcb.gaddr, *rc, *rrc); return ret; From 9187a9186d0ac7b260b07917aabc672a80d37c3c Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:39 +0200 Subject: [PATCH 33/82] KVM: s390: Fix ordering when adding to SCA When adding a new vCPU to the SCA area, the validity bit in the MCN was set before the pointer to the state description, potentially allowing for a race. Fix by setting the pointer before setting the bit. Fixes: 14542a0a54c5 ("KVM: S390: Remove sca_lock") Reviewed-by: Steffen Eiden Reviewed-by: Janosch Frank Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-13-imbrenda@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 9b8a35e7dc82..518a69c55e85 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3485,8 +3485,8 @@ static void sca_add_vcpu(struct kvm_vcpu *vcpu) if (!kvm_s390_use_sca_entries()) return; + WRITE_ONCE(sca->cpu[vcpu->vcpu_id].sda, virt_to_phys(vcpu->arch.sie_block)); set_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn); - sca->cpu[vcpu->vcpu_id].sda = virt_to_phys(vcpu->arch.sie_block); } static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id) From feadc5e84dcb53422a437556c35af9efd9826fd5 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Mon, 3 Aug 2026 14:40:40 +0200 Subject: [PATCH 34/82] KVM: s390: Fix cleanup in kvm_s390_pv_create_cpu() If creating a protected vCPU in kvm_s390_pv_create_cpu() fails, kvm_s390_pv_destroy_cpu() was called, which checks whether the vCPU has a PV handle and exits doing nothing otherwise. At that point, due to not having created the protected vCPU, the PV handle will not be set, and kvm_s390_pv_destroy_cpu() will do nothing, thus leaking the allocated memory. Fix by factoring out the code to free and reset a PV vCPU; call it from kvm_s390_pv_destroy_cpu() and kvm_s390_pv_create_cpu(). Opportunistically fix the return value of kvm_s390_pv_destroy_cpu() in case of errors: return -EIO instead if EIO. Fixes: d4074324b07a ("KVM: s390: pv: avoid double free of sida page") Reviewed-by: Steffen Eiden Reviewed-by: Janosch Frank Signed-off-by: Claudio Imbrenda Message-ID: <20260803124040.126471-14-imbrenda@linux.ibm.com> --- arch/s390/kvm/pv.c | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c index dc204b521052..b02e0159d3cd 100644 --- a/arch/s390/kvm/pv.c +++ b/arch/s390/kvm/pv.c @@ -244,6 +244,24 @@ static void kvm_s390_clear_pv_state(struct kvm *kvm) kvm->arch.pv.stor_var = NULL; } +static void kvm_s390_pv_dispose_cpu(struct kvm_vcpu *vcpu, bool free_stor_base) +{ + if (free_stor_base) + free_pages(vcpu->arch.pv.stor_base, get_order(uv_info.guest_cpu_stor_len)); + free_page((unsigned long)sida_addr(vcpu->arch.sie_block)); + vcpu->arch.sie_block->pv_handle_cpu = 0; + vcpu->arch.sie_block->pv_handle_config = 0; + memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv)); + vcpu->arch.sie_block->sdf = 0; + /* + * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0). + * Use the reset value of gbea to avoid leaking the kernel pointer of + * the just freed sida. + */ + vcpu->arch.sie_block->gbea = 1; + kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); +} + int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) { int cc; @@ -258,24 +276,9 @@ int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) WARN_ONCE(cc, "protvirt destroy cpu failed rc %x rrc %x", *rc, *rrc); /* Intended memory leak for something that should never happen. */ - if (!cc) - free_pages(vcpu->arch.pv.stor_base, - get_order(uv_info.guest_cpu_stor_len)); + kvm_s390_pv_dispose_cpu(vcpu, !cc); - free_page((unsigned long)sida_addr(vcpu->arch.sie_block)); - vcpu->arch.sie_block->pv_handle_cpu = 0; - vcpu->arch.sie_block->pv_handle_config = 0; - memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv)); - vcpu->arch.sie_block->sdf = 0; - /* - * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0). - * Use the reset value of gbea to avoid leaking the kernel pointer of - * the just freed sida. - */ - vcpu->arch.sie_block->gbea = 1; - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); - - return cc ? EIO : 0; + return cc ? -EIO : 0; } int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) @@ -319,9 +322,7 @@ int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc) uvcb.header.rrc); if (cc) { - u16 dummy; - - kvm_s390_pv_destroy_cpu(vcpu, &dummy, &dummy); + kvm_s390_pv_dispose_cpu(vcpu, true); return -EIO; } From c39c3d231f42ba50cc3a2b9161e075f4f1b7d9e0 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 6 Aug 2026 17:38:47 +0200 Subject: [PATCH 35/82] s390/vfio_ccw: Add __must_hold() attribute to vfio_ccw_sch_quiesce() Add __must_hold() attribute to vfio_ccw_sch_quiesce() in order to let clang's context analysis know that sch->lock must be held on function entry. This can also be easily verified when inspecting the function. Without this annotation this leads to a valid warning when context analysis is enabled: drivers/s390/cio/vfio_ccw_drv.c:55:9: warning: expecting spinlock 'sch->lock' to be held at start of each loop [-Wthread-safety-analysis] 55 | ret = cio_cancel_halt_clear(sch, &iretry); | ^ Signed-off-by: Heiko Carstens Acked-by: Christian Borntraeger Reviewed-by: Matthew Rosato [borntraeger@linux.ibm.com: fix spurious ;] Signed-off-by: Christian Borntraeger --- drivers/s390/cio/vfio_ccw_drv.c | 1 + drivers/s390/cio/vfio_ccw_private.h | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c index 1a095085bc72..ef4c69d70afb 100644 --- a/drivers/s390/cio/vfio_ccw_drv.c +++ b/drivers/s390/cio/vfio_ccw_drv.c @@ -35,6 +35,7 @@ debug_info_t *vfio_ccw_debug_trace_id; * Helpers */ int vfio_ccw_sch_quiesce(struct subchannel *sch) + __must_hold(&sch->lock) { struct vfio_ccw_parent *parent = dev_get_drvdata(&sch->dev); struct vfio_ccw_private *private = dev_get_drvdata(&parent->dev); diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h index 0501d4bbcdbd..8db29519dbfd 100644 --- a/drivers/s390/cio/vfio_ccw_private.h +++ b/drivers/s390/cio/vfio_ccw_private.h @@ -127,7 +127,8 @@ struct vfio_ccw_private { struct work_struct crw_work; } __aligned(8); -int vfio_ccw_sch_quiesce(struct subchannel *sch); +int vfio_ccw_sch_quiesce(struct subchannel *sch) + __must_hold(&sch->lock); void vfio_ccw_sch_io_todo(struct work_struct *work); void vfio_ccw_crw_todo(struct work_struct *work); From 7008546c7ba4a6b3bfacddeaea7c3b446c0b057c Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 6 Aug 2026 17:38:48 +0200 Subject: [PATCH 36/82] s390/cio: Enable CONTEXT_ANALYSIS All cio code passes clang's compile time context analysis. Therefore enable CONTEXT_ANALYSIS. Signed-off-by: Heiko Carstens Acked-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- drivers/s390/cio/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/s390/cio/Makefile b/drivers/s390/cio/Makefile index 3bd1c245183f..611a39103df9 100644 --- a/drivers/s390/cio/Makefile +++ b/drivers/s390/cio/Makefile @@ -3,6 +3,8 @@ # Makefile for the S/390 common i/o drivers # +CONTEXT_ANALYSIS := y + # The following is required for define_trace.h to find ./trace.h CFLAGS_trace.o := -I$(src) CFLAGS_vfio_ccw_trace.o := -I$(src) From 8dbc99a1c317b5cd7191688a4bcf13ce19fc3f59 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 6 Aug 2026 17:38:49 +0200 Subject: [PATCH 37/82] s390/virtio: Enable CONTEXT_ANALYSIS All virtio code passes clang's compile time context analysis. Therefore enable CONTEXT_ANALYSIS. Signed-off-by: Heiko Carstens Acked-by: Matthew Rosato Acked-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- drivers/s390/virtio/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/s390/virtio/Makefile b/drivers/s390/virtio/Makefile index 2dc4d9aab634..cab140fe0b29 100644 --- a/drivers/s390/virtio/Makefile +++ b/drivers/s390/virtio/Makefile @@ -3,4 +3,6 @@ # # Copyright IBM Corp. 2008 +CONTEXT_ANALYSIS := y + obj-$(CONFIG_S390_GUEST) += virtio_ccw.o From b5e93a1496a86bb0dee7261cc999ae1df0daded8 Mon Sep 17 00:00:00 2001 From: Christoph Schlameuss Date: Fri, 7 Aug 2026 11:57:42 +0200 Subject: [PATCH 38/82] KVM: s390: Remove double 64bscao feature check sclp.has_64bscao is already verified in the guard clause a few lines above this. So we cannot reach this code if it is not true. Reviewed-by: Hendrik Brueckner Reviewed-by: Eric Farman Reviewed-by: Janosch Frank Signed-off-by: Christoph Schlameuss Reviewed-by: Claudio Imbrenda Reviewed-by: Janosch Frank Signed-off-by: Christian Borntraeger --- arch/s390/kvm/kvm-s390.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 23c817595e28..4b1ef955c7e8 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -455,8 +455,7 @@ static void __init kvm_s390_cpu_feat_init(void) !test_facility(3) || !nested) return; allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2); - if (sclp.has_64bscao) - allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO); + allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO); if (sclp.has_siif) allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF); if (sclp.has_gpere) From 2ac76c04d3095c1b6b5ad0b730b858ac6cc9fbe9 Mon Sep 17 00:00:00 2001 From: Christoph Schlameuss Date: Fri, 7 Aug 2026 11:57:43 +0200 Subject: [PATCH 39/82] KVM: s390: vsie: Remove duplicate assertion Remove useless BUILD_BUG_ON() checking the size of struct vsie_page. This is already covered by a static_assert at the struct vsie_page definition. Fixes: e38c884df921 ("KVM: s390: Switch to new gmap") Signed-off-by: Christoph Schlameuss [borntraeger@linux.ibm.com: improve commit message] Signed-off-by: Christian Borntraeger --- arch/s390/kvm/vsie.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index eea24562e7db..3e956c7faa4b 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -1565,7 +1565,6 @@ int kvm_s390_handle_vsie(struct kvm_vcpu *vcpu) if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); - BUILD_BUG_ON(sizeof(struct vsie_page) != PAGE_SIZE); scb_addr = kvm_s390_get_base_disp_s(vcpu, NULL); /* 512 byte alignment */ From 38926a93f7792da96681bf40051c3fe8095435f0 Mon Sep 17 00:00:00 2001 From: Christoph Schlameuss Date: Fri, 7 Aug 2026 11:57:44 +0200 Subject: [PATCH 40/82] KVM: s390: vsie: Convert shift to phys_to_pfn() Make the code slightly more readable by using phys_to_pfn instead of an open coded shift. Signed-off-by: Christoph Schlameuss Reviewed-by: Christian Borntraeger [borntraeger@linux.ibm.com: improve commit message] Signed-off-by: Christian Borntraeger --- arch/s390/kvm/vsie.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 3e956c7faa4b..937f9c99bc00 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -701,7 +701,7 @@ static int pin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t *hpa) /* Unpins a page previously pinned via pin_guest_page, marking it as dirty. */ static void unpin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t hpa) { - kvm_release_page_dirty(pfn_to_page(hpa >> PAGE_SHIFT)); + kvm_release_page_dirty(pfn_to_page(phys_to_pfn(hpa))); /* mark the page always as dirty for migration */ mark_page_dirty(kvm, gpa_to_gfn(gpa)); } From 7cdbd4c7436d3c744002825f6001bf90c81c2242 Mon Sep 17 00:00:00 2001 From: Christoph Schlameuss Date: Fri, 7 Aug 2026 11:57:45 +0200 Subject: [PATCH 41/82] KVM: s390: vsie: Assert mcck_info offset in vsie_page Ensure that the backup info for machine check is the same offset as that in struct sie_page! With the assertion in place we do not need the comment anymore. Signed-off-by: Christoph Schlameuss Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/vsie.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 937f9c99bc00..7c3b5e11b301 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -33,10 +33,7 @@ enum vsie_page_flags { struct vsie_page { struct kvm_s390_sie_block scb_s; /* 0x0000 */ - /* - * the backup info for machine check. ensure it's at - * the same offset as that in struct sie_page! - */ + /* backup info for machine check */ struct mcck_volatile_info mcck_info; /* 0x0200 */ /* * The pinned original scb. Be aware that other VCPUs can modify @@ -71,6 +68,7 @@ struct vsie_page { }; static_assert(sizeof(struct vsie_page) == PAGE_SIZE); +static_assert(offsetof(struct vsie_page, mcck_info) == offsetof(struct sie_page, mcck_info)); /* trigger a validity icpt for the given scb */ static int set_validity_icpt(struct kvm_s390_sie_block *scb, From 6a3fe9ecc255e780e7f93dbaf4bfb94d7222c4b1 Mon Sep 17 00:00:00 2001 From: Christoph Schlameuss Date: Fri, 7 Aug 2026 11:57:46 +0200 Subject: [PATCH 42/82] KVM: s390: vsie: Assert crycb alignment in vsie_page The crypto control block address is required to have double word alignment. Add a static_assert to enforce correct alignment. Signed-off-by: Christoph Schlameuss Reviewed-by: Christian Borntraeger [borntraeger@linux.ibm.com: improve commit message] Signed-off-by: Christian Borntraeger --- arch/s390/kvm/vsie.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 7c3b5e11b301..c9392460edca 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -69,6 +69,7 @@ struct vsie_page { static_assert(sizeof(struct vsie_page) == PAGE_SIZE); static_assert(offsetof(struct vsie_page, mcck_info) == offsetof(struct sie_page, mcck_info)); +static_assert(IS_ALIGNED(offsetof(struct vsie_page, crycb), 8)); /* trigger a validity icpt for the given scb */ static int set_validity_icpt(struct kvm_s390_sie_block *scb, From 3105276cef9dcbc81a9b5302e542eaca723c9493 Mon Sep 17 00:00:00 2001 From: Christoph Schlameuss Date: Fri, 7 Aug 2026 11:57:47 +0200 Subject: [PATCH 43/82] KVM: s390: vsie: Create constant SCB_ALIGNMENT_SHIFT Create a simple constant for the SCB alignment shift. Signed-off-by: Christoph Schlameuss Signed-off-by: Christian Borntraeger --- arch/s390/include/asm/kvm_host_types.h | 2 ++ arch/s390/kvm/vsie.c | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/s390/include/asm/kvm_host_types.h b/arch/s390/include/asm/kvm_host_types.h index 3f50942bdfe6..ac82dd09fce5 100644 --- a/arch/s390/include/asm/kvm_host_types.h +++ b/arch/s390/include/asm/kvm_host_types.h @@ -9,6 +9,8 @@ #define KVM_S390_BSCA_CPU_SLOTS 64 #define KVM_S390_ESCA_CPU_SLOTS 248 +#define SCB_ALIGNMENT_SHIFT 9 + #define SIGP_CTRL_C 0x80 #define SIGP_CTRL_SCN_MASK 0x3f diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index c9392460edca..eb60a359d0aa 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -1485,7 +1485,7 @@ static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr) int nr_vcpus; rcu_read_lock(); - vsie_page = radix_tree_lookup(&kvm->arch.vsie.addr_to_page, addr >> 9); + vsie_page = radix_tree_lookup(&kvm->arch.vsie.addr_to_page, addr >> SCB_ALIGNMENT_SHIFT); rcu_read_unlock(); if (vsie_page) { if (try_get_vsie_page(vsie_page)) { @@ -1526,13 +1526,14 @@ static struct vsie_page *get_vsie_page(struct kvm *kvm, unsigned long addr) } if (vsie_page->scb_gpa != ULONG_MAX) radix_tree_delete(&kvm->arch.vsie.addr_to_page, - vsie_page->scb_gpa >> 9); + vsie_page->scb_gpa >> SCB_ALIGNMENT_SHIFT); } /* Mark it as invalid until it resides in the tree. */ vsie_page->scb_gpa = ULONG_MAX; /* Double use of the same address or allocation failure. */ - if (radix_tree_insert(&kvm->arch.vsie.addr_to_page, addr >> 9, vsie_page)) { + if (radix_tree_insert(&kvm->arch.vsie.addr_to_page, addr >> SCB_ALIGNMENT_SHIFT, + vsie_page)) { put_vsie_page(vsie_page); mutex_unlock(&kvm->arch.vsie.mutex); return NULL; @@ -1630,7 +1631,7 @@ void kvm_s390_vsie_destroy(struct kvm *kvm) /* free the radix tree entry */ if (vsie_page->scb_gpa != ULONG_MAX) radix_tree_delete(&kvm->arch.vsie.addr_to_page, - vsie_page->scb_gpa >> 9); + vsie_page->scb_gpa >> SCB_ALIGNMENT_SHIFT); free_page((unsigned long)vsie_page); } kvm->arch.vsie.page_count = 0; From b98014e98e0842b498506d9b3060bf1707ed0739 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Wed, 5 Aug 2026 13:04:48 +0200 Subject: [PATCH 44/82] KVM: s390: Remove user triggerable WARN_ON pin_map_page() fails legitimately whenever the userspace mapping behind the adapter route has gone away, e.g. when the VMM unmaps that memory. As this can happen without a kernel programming error, remove the WARN_ON. Fixes: 1e95e3bc6b05 ("KVM: s390: Enable adapter_indicators_set to use mapped pages") Cc: Douglas Freimuth Cc: Matthew Rosato Signed-off-by: Christian Borntraeger Reviewed-by: Matthew Rosato Reviewed-by: Douglas Freimuth Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260805110455.7200-2-borntraeger@linux.ibm.com> --- arch/s390/kvm/interrupt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 8f24bcd1a6d3..23f0e0821474 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -3013,7 +3013,7 @@ static int adapter_indicators_set(struct kvm *kvm, if (!summary_info) { spin_unlock_irqrestore(&adapter->maps_lock, flags); summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0); - if (WARN_ON_ONCE(!summary_page)) + if (!summary_page) return -1; idx = srcu_read_lock(&kvm->srcu); map = page_address(summary_page); From 4e2c7f7cbc27418f9a290399b986c1b85ff93b90 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Wed, 5 Aug 2026 13:04:49 +0200 Subject: [PATCH 45/82] KVM: s390: Zero initialize data structures for inject_pfault_token __kvm_inject_pfault_token() only sets .type and .u.ext.ext_params2 of the on-stack struct kvm_s390_irq but the full ext substructure is copied into the cpu local variable on inject. ext_params and pad contain stale stack values. Interrupt delivery only uses ext_params2, so nothing leaks to the guest, but a host user can use the migration ioctls to get to the data. Fix by zero-initializing the irq struct. Do the same for the inti data structure. Fixes: 383d0b050106 ("KVM: s390: handle pending local interrupts via bitmap") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Matthew Rosato Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260805110455.7200-3-borntraeger@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 518a69c55e85..611dcb534210 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4474,8 +4474,8 @@ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clo static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, unsigned long token) { - struct kvm_s390_interrupt inti; - struct kvm_s390_irq irq; + struct kvm_s390_interrupt inti = {}; + struct kvm_s390_irq irq = {}; struct kvm_s390_interrupt_info *inti_mem = NULL; int ret = 0; From b239410c7653ff6781d4cf1d63cfc52a1bb71788 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Wed, 5 Aug 2026 13:04:50 +0200 Subject: [PATCH 46/82] KVM: s390: Zero initialize irq in reinject_machine_check kvm_s390_reinject_machine_check() fills cr14, mcic, ext_damage_code and failing_storage_address of the on-stack struct kvm_s390_irq, but struct kvm_s390_mchk_info also has a pad word and a 16 byte fixed_logout array. struct mcck_volatile_info carries no logout data, so there is nothing to copy there and both stay whatever was on the stack. __inject_mchk() then memcpy()s fixed_logout into the vcpu local interrupt state unconditionally. This will reach the guest during deliver and userspace during migration. Reflecting zeroes is the correct behaviour here, as KVM has no logout data for a reinjected machine check. This needs a host machine check while the cpu is in SIE so not trivial to trigger. Fixes: 4d62fcc0b692 ("KVM: s390: Inject machine check into the guest") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Matthew Rosato Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260805110455.7200-4-borntraeger@linux.ibm.com> --- arch/s390/kvm/interrupt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 23f0e0821474..009d6a845d59 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -3109,7 +3109,7 @@ void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, struct mcck_volatile_info *mcck_info) { struct kvm_s390_interrupt_info inti; - struct kvm_s390_irq irq; + struct kvm_s390_irq irq = {}; struct kvm_s390_mchk_info *mchk; union mci mci; __u64 cr14 = 0; /* upper bits are not used */ From 121ea1de927c8b9bfdf53c31cad27b86d5de0293 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Wed, 5 Aug 2026 13:04:51 +0200 Subject: [PATCH 47/82] KVM: s390: Fix memory leak in guest debug handling bp_data is freed only for the error case by kfree(bp_data). Every successful KVM_SET_GUEST_DEBUG will leak bp_data. Fixes: 27291e2165b6 ("KVM: s390: hardware support for guest debugging") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Matthew Rosato Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260805110455.7200-5-borntraeger@linux.ibm.com> --- arch/s390/kvm/guestdbg.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/guestdbg.c index 69835e1d4f20..4c02dbebb2eb 100644 --- a/arch/s390/kvm/guestdbg.c +++ b/arch/s390/kvm/guestdbg.c @@ -267,6 +267,7 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, vcpu->arch.guestdbg.hw_bp_info = bp_info; vcpu->arch.guestdbg.nr_hw_wp = nr_wp; vcpu->arch.guestdbg.hw_wp_info = wp_info; + kfree(bp_data); return 0; error: kfree(bp_data); From aa9c8e8baf1e765fa65b93212522c636f25d846f Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Wed, 5 Aug 2026 13:04:52 +0200 Subject: [PATCH 48/82] KVM: s390: Fix old_data leak in guest debug error path __import_wp_info() allocates a per-watchpoint old_data buffer to back up the original guest memory contents. If a later watchpoint of the same KVM_SET_GUEST_DEBUG request fails to import, kvm_s390_import_bp_data() jumps to the error label, which frees the wp_info array but not the old_data buffers of the entries that were imported successfully. Up to MAX_BP_COUNT - 1 buffers of up to MAX_WP_SIZE bytes are leaked per failed request, and the request can be repeated. Create error handling for cleaning up all created old_data memory areas. Fixes: 27291e2165b6 ("KVM: s390: hardware support for guest debugging") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Matthew Rosato Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260805110455.7200-6-borntraeger@linux.ibm.com> --- arch/s390/kvm/guestdbg.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/guestdbg.c index 4c02dbebb2eb..f7c94d54efbe 100644 --- a/arch/s390/kvm/guestdbg.c +++ b/arch/s390/kvm/guestdbg.c @@ -252,7 +252,7 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, ret = __import_wp_info(vcpu, &bp_data[i], &wp_info[nr_wp]); if (ret) - goto error; + goto error_wp; nr_wp++; break; case KVM_HW_BP: @@ -269,6 +269,10 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, vcpu->arch.guestdbg.hw_wp_info = wp_info; kfree(bp_data); return 0; + +error_wp: + while (nr_wp--) + kfree(wp_info[nr_wp].old_data); error: kfree(bp_data); kfree(wp_info); From a4e482def8533ebace517d9f67f1465841b1f982 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Wed, 5 Aug 2026 13:04:53 +0200 Subject: [PATCH 49/82] KVM: s390: Take srcu when importing watchpoint data __import_wp_info() backs up the original guest memory contents of a watchpoint with read_guest_abs(), which is kvm_read_guest() and therefore resolves the memslot via __kvm_memslots(). That requires kvm->srcu (or kvm->slots_lock) to be held, otherwise a concurrent memslot update can free the memslots array under us once its SRCU grace period has elapsed. As this is not fast path, following lock ordering (mutex first, then srcu) take the big hammer and hold the srcu for the full import. Fixes: 27291e2165b6 ("KVM: s390: hardware support for guest debugging") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260805110455.7200-7-borntraeger@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 611dcb534210..0a197e055cad 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4248,8 +4248,10 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, /* enforce guest PER */ kvm_s390_set_cpuflags(vcpu, CPUSTAT_P); - if (dbg->control & KVM_GUESTDBG_USE_HW_BP) - rc = kvm_s390_import_bp_data(vcpu, dbg); + if (dbg->control & KVM_GUESTDBG_USE_HW_BP) { + scoped_guard(srcu, &vcpu->kvm->srcu) + rc = kvm_s390_import_bp_data(vcpu, dbg); + } } else { kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P); vcpu->arch.guestdbg.last_bp = 0; From e7f698b09d4a7c36b299acf680fc50fe868e2bcd Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Wed, 5 Aug 2026 13:04:54 +0200 Subject: [PATCH 50/82] KVM: s390: Free guest debug data on vcpu destroy kvm_s390_clear_bp_data() is only called from kvm_arch_vcpu_ioctl_set_guest_debug(), i.e. when user space changes or disables debugging. A vCPU that is destroyed while hardware breakpoints are still armed - the normal case when the VMM just exits or crashes - leaks hw_bp_info, hw_wp_info and all old_data buffers, since generic KVM frees the vCPU right after kvm_arch_vcpu_destroy(). That is bounded by MAX_BP_COUNT entries, so roughly 8 KiB per vCPU, but it is unbounded over VM lifetimes. The allocations are GFP_KERNEL_ACCOUNT, so the charge also outlives the exiting process and pins dying memcgs. Fix by clearing the debug data on vCPU destruction. Calling it unconditionally is fine: struct kvm_vcpu is zero allocated, so for a vCPU that never enabled debugging the counters are 0 and the pointers NULL. Fixes: 27291e2165b6 ("KVM: s390: hardware support for guest debugging") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Matthew Rosato Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260805110455.7200-8-borntraeger@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 0a197e055cad..095c9ba0aa5c 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3410,6 +3410,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); kvm_s390_clear_local_irqs(vcpu); kvm_clear_async_pf_completion_queue(vcpu); + kvm_s390_clear_bp_data(vcpu); if (!kvm_is_ucontrol(vcpu->kvm)) sca_del_vcpu(vcpu); kvm_s390_update_topology_change_report(vcpu->kvm, 1); From 4c07680a467e2f7697245bcd11691bffb2a6f0ed Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Wed, 5 Aug 2026 13:04:55 +0200 Subject: [PATCH 51/82] KVM: s390: Fix length check __import_wp_info() struct kvm_hw_breakpoint::len is a __u64 that is fully controlled by user space. This is then assigned to wp_info->len, which is an int. The bounds check is done on the truncated value while the allocation uses the untruncated one: wp_info->len = bp_data->len; [...] if (wp_info->len < 0 || wp_info->len > MAX_WP_SIZE) return -EINVAL; wp_info->old_data = kmalloc(bp_data->len, GFP_KERNEL_ACCOUNT); Use the validated value for the allocation as intended. Without this fix userspace can trigger >4GB allocations which will fail and result in a WARN due to MAX_PAGE_ORDER. Fixes: 27291e2165b6 ("KVM: s390: hardware support for guest debugging") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260805110455.7200-9-borntraeger@linux.ibm.com> --- arch/s390/kvm/guestdbg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/guestdbg.c index f7c94d54efbe..9a6149e310bb 100644 --- a/arch/s390/kvm/guestdbg.c +++ b/arch/s390/kvm/guestdbg.c @@ -184,7 +184,7 @@ static int __import_wp_info(struct kvm_vcpu *vcpu, if (wp_info->len < 0 || wp_info->len > MAX_WP_SIZE) return -EINVAL; - wp_info->old_data = kmalloc(bp_data->len, GFP_KERNEL_ACCOUNT); + wp_info->old_data = kmalloc(wp_info->len, GFP_KERNEL_ACCOUNT); if (!wp_info->old_data) return -ENOMEM; /* try to backup the original value */ From 4400270ec0348d05dc0439d8f0130853ce7f9e20 Mon Sep 17 00:00:00 2001 From: Anthony Krowiak Date: Thu, 6 Aug 2026 13:34:35 -0400 Subject: [PATCH 52/82] s390/vfio-ap: fix stale pqap_hook pointer on error in vfio_ap_mdev_set_kvm() In vfio_ap_mdev_set_kvm(), kvm->arch.crypto.pqap_hook is set to &matrix_mdev->pqap_hook before the update locks are acquired and the mdev list is checked for a conflicting assignment. If another mdev is already attached to the same KVM instance, the function returns -EPERM without restoring the hook pointer, leaving kvm->arch.crypto.pqap_hook pointing at the failing matrix_mdev instead of the mdev that legitimately owns the KVM. Since matrix_mdev->kvm is never set on this error path, vfio_ap_mdev_unset_kvm() will not clean up the hook when matrix_mdev is later closed. If matrix_mdev is subsequently freed, any PQAP instruction executed by the guest will dereference the stale pointer through pqap_hook_rwsem, resulting in a use-after-free. Since kvm->arch.crypto.pqap_hook is only set in the vfio_ap_mdev_set_kvm() function and is cleared in the vfio_ap_mdev_unset_kvm() function, a check for 'kvm->arch.crypto.pqap_hook != NULL' is all that is needed to determine whether it belongs to another mdev. This will alleviate the need to iterate the matrix_dev->mdev_list list to see if the kvm object is assigned to another mdev.This was introduced in v3 to alleviate the need to take the mdevs_lock while iterating the list; however, this did not prevent a potential race condition. The pqap_hook_rwsem(write) is now performed inside get_update_locks_for_kvm(), which is updated to acquire pqap_hook_rwsem(write) between kvm->lock and mdevs_lock. This ordering is consistent with the PQAP intercept path, which acquires pqap_hook_rwsem in read mode while srcu is held under vcpu->mutex, establishing the dependency: kvm->lock -> vcpu->mutex -> srcu -> pqap_hook_rwsem(read). The pqap_hook_rwsem is now released inside the release_update_locks_for_kvm(), which is updated to release pqap_hook_rwsem(write) between mdevs_lock and kvm->lock. Additionally, kvm_put_kvm() in vfio_ap_mdev_unset_kvm() is moved after release_update_locks_for_kvm(). Previously it was called while kvm->lock was held; if it were ever the last reference, kvm_destroy_vm() would run under kvm->lock, which would deadlock. Fixes: 86956e70761b3 ("s390/vfio-ap: replace open coded locks for VFIO_GROUP_NOTIFY_SET_KVM notification") Cc: stable@vger.kernel.org Co-developed-by: Matthew Rosato Signed-off-by: Matthew Rosato Signed-off-by: Anthony Krowiak Acked-by: Christian Borntraeger Signed-off-by: Claudio Imbrenda Message-ID: <20260806173435.105044-1-akrowiak@linux.ibm.com> --- drivers/s390/crypto/vfio_ap_ops.c | 45 ++++++++++++++----------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 44b3a1dcc1b3..99a0efd999ef 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -48,15 +48,19 @@ static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q); * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM * guest's APCB. * 2. kvm->lock: required to update a guest's APCB - * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev + * 3. kvm->arch.crypto.pqap_hook_rwsem: required to update pqap_hook and + * serialize against PQAP intercepts + * 4. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev * - * Note: If @kvm is NULL, the KVM lock will not be taken. + * Note: If @kvm is NULL, the KVM lock and pqap_hook_rwsem will not be taken. */ static inline void get_update_locks_for_kvm(struct kvm *kvm) { mutex_lock(&matrix_dev->guests_lock); - if (kvm) + if (kvm) { mutex_lock(&kvm->lock); + down_write(&kvm->arch.crypto.pqap_hook_rwsem); + } mutex_lock(&matrix_dev->mdevs_lock); } @@ -68,16 +72,19 @@ static inline void get_update_locks_for_kvm(struct kvm *kvm) * * The proper unlocking order is: * 1. matrix_dev->mdevs_lock - * 2. kvm->lock - * 3. matrix_dev->guests_lock + * 2. kvm->arch.crypto.pqap_hook_rwsem + * 3. kvm->lock + * 4. matrix_dev->guests_lock * - * Note: If @kvm is NULL, the KVM lock will not be released. + * Note: If @kvm is NULL, the KVM lock and pqap_hook_rwsem will not be released. */ static inline void release_update_locks_for_kvm(struct kvm *kvm) { mutex_unlock(&matrix_dev->mdevs_lock); - if (kvm) + if (kvm) { + up_write(&kvm->arch.crypto.pqap_hook_rwsem); mutex_unlock(&kvm->lock); + } mutex_unlock(&matrix_dev->guests_lock); } @@ -1821,26 +1828,17 @@ static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, struct kvm *kvm) { - struct ap_matrix_mdev *m; - if (kvm->arch.crypto.crycbd) { - down_write(&kvm->arch.crypto.pqap_hook_rwsem); - kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; - up_write(&kvm->arch.crypto.pqap_hook_rwsem); - get_update_locks_for_kvm(kvm); - - list_for_each_entry(m, &matrix_dev->mdev_list, node) { - if (m != matrix_mdev && m->kvm == kvm) { - release_update_locks_for_kvm(kvm); - return -EPERM; - } + if (kvm->arch.crypto.pqap_hook) { + release_update_locks_for_kvm(kvm); + return -EPERM; } + kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; kvm_get_kvm(kvm); matrix_mdev->kvm = kvm; vfio_ap_mdev_update_guest_apcb(matrix_mdev); - release_update_locks_for_kvm(kvm); } @@ -1883,18 +1881,15 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) struct kvm *kvm = matrix_mdev->kvm; if (kvm && kvm->arch.crypto.crycbd) { - down_write(&kvm->arch.crypto.pqap_hook_rwsem); - kvm->arch.crypto.pqap_hook = NULL; - up_write(&kvm->arch.crypto.pqap_hook_rwsem); - get_update_locks_for_kvm(kvm); + kvm->arch.crypto.pqap_hook = NULL; kvm_arch_crypto_clear_masks(kvm); vfio_ap_mdev_reset_queues(matrix_mdev); - kvm_put_kvm(kvm); matrix_mdev->kvm = NULL; release_update_locks_for_kvm(kvm); + kvm_put_kvm(kvm); } } From 546dde823a36d7283dcf46127c2f3d093443860f Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Thu, 6 Aug 2026 16:58:35 +0200 Subject: [PATCH 53/82] KVM: s390: Fix memory corruption by not reinjecting CK machine checks Channel-subsystem damage machine checks are for the host channel subsystem. The guest channel subsystem is emulated in the userspace VMM. There is no point in forwarding such machine checks into the guest. This also simplifies the machine check reinjection and avoids kfree of a stack variable as reported by sashiko. There might be still machine checks that have the ck bit set with another bit (like instruction damage), mask out the CK bit in s390_backup_mcck_info(), like the CP and ED bits already are. Fixes: 4d62fcc0b692 ("KVM: s390: Inject machine check into the guest") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Acked-by: Heiko Carstens Acked-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260806145835.31818-1-borntraeger@linux.ibm.com> --- arch/s390/include/asm/nmi.h | 3 +++ arch/s390/kernel/nmi.c | 5 +---- arch/s390/kvm/interrupt.c | 24 ++++++++---------------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/arch/s390/include/asm/nmi.h b/arch/s390/include/asm/nmi.h index 6454c1531854..7919b2b9ac9c 100644 --- a/arch/s390/include/asm/nmi.h +++ b/arch/s390/include/asm/nmi.h @@ -22,6 +22,7 @@ #define MCCK_CODE_SYSTEM_DAMAGE BIT(63) #define MCCK_CODE_EXT_DAMAGE BIT(63 - 5) #define MCCK_CODE_CP BIT(63 - 9) +#define MCCK_CODE_CK BIT(63 - 11) #define MCCK_CODE_STG_ERROR BIT(63 - 16) #define MCCK_CODE_STG_KEY_ERROR BIT(63 - 18) #define MCCK_CODE_STG_DEGRAD BIT(63 - 19) @@ -33,6 +34,8 @@ #define MCCK_CODE_FC_VALID BIT(63 - 43) #define MCCK_CODE_CPU_TIMER_VALID BIT(63 - 46) +#define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE | MCCK_CODE_CK) + #ifndef __ASSEMBLER__ union mci { diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index e17a59d4d5a4..17297a8b63d9 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -344,8 +344,7 @@ static void notrace s390_backup_mcck_info(struct pt_regs *regs) sie_page = container_of(sie_block, struct sie_page, sie_block); mcck_backup = &sie_page->mcck_info; - mcck_backup->mcic = get_lowcore()->mcck_interruption_code & - ~(MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE); + mcck_backup->mcic = get_lowcore()->mcck_interruption_code & ~MCCK_CODE_NO_GUEST; mcck_backup->ext_damage_code = get_lowcore()->external_damage_code; mcck_backup->failing_storage_address = get_lowcore()->failing_storage_address; } @@ -357,8 +356,6 @@ NOKPROBE_SYMBOL(s390_backup_mcck_info); #define ED_STP_ISLAND 6 /* External damage STP island check */ #define ED_STP_SYNC 7 /* External damage STP sync check */ -#define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE) - /* * machine check handler. */ diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 009d6a845d59..0d9fbe75dc66 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -3108,9 +3108,7 @@ static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, struct mcck_volatile_info *mcck_info) { - struct kvm_s390_interrupt_info inti; struct kvm_s390_irq irq = {}; - struct kvm_s390_mchk_info *mchk; union mci mci; __u64 cr14 = 0; /* upper bits are not used */ int rc; @@ -3129,20 +3127,14 @@ void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, if (mci.w) cr14 |= CR14_WARNING_SUBMASK; - mchk = mci.ck ? &inti.mchk : &irq.u.mchk; - mchk->cr14 = cr14; - mchk->mcic = mcck_info->mcic; - mchk->ext_damage_code = mcck_info->ext_damage_code; - mchk->failing_storage_address = mcck_info->failing_storage_address; - if (mci.ck) { - /* Inject the floating machine check */ - inti.type = KVM_S390_MCHK; - rc = __inject_vm(vcpu->kvm, &inti); - } else { - /* Inject the machine check to specified vcpu */ - irq.type = KVM_S390_MCHK; - rc = kvm_s390_inject_vcpu(vcpu, &irq); - } + irq.u.mchk.cr14 = cr14; + irq.u.mchk.mcic = mcck_info->mcic; + irq.u.mchk.ext_damage_code = mcck_info->ext_damage_code; + irq.u.mchk.failing_storage_address = mcck_info->failing_storage_address; + + /* Inject the machine check to specified vcpu */ + irq.type = KVM_S390_MCHK; + rc = kvm_s390_inject_vcpu(vcpu, &irq); WARN_ON_ONCE(rc); } From da07a751efa4583385f9f0f47113549fe8871242 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Tue, 11 Aug 2026 17:37:35 +0200 Subject: [PATCH 54/82] KVM: s390: keyop: use mmu_lock to read gmap->asce Every other dat_* consumer in this file (kvm_s390_get_skeys, set_skeys, get_cmma_bits, set_cmma_bits, MEM_CLR_CMMA, kvm_s390_fixup_prefix, kvm_test_age_gfn, kvm_age_gfn) reads kvm->arch.gmap->asce *inside* the mmu_lock read-side. keyop is the only outlier. gmap->asce is mutated under write_lock(mmu_lock) by gmap_set_limit() and keyop might use a stale asce value for walking as KVM_S390_KEYOP and KVM_S390_VM_MEM_LIMIT_SIZE can run concurrently. This can result in memory corruption. Fixes: 0ee4ddc1647b ("KVM: s390: Storage key manipulation IOCTL") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260811153738.206885-2-borntraeger@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 095c9ba0aa5c..a3e55c613243 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -562,11 +562,12 @@ static void __kvm_s390_exit(void) static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op, unsigned long addr, union skey skey) { - union asce asce = kvm->arch.gmap->asce; gfn_t gfn = gpa_to_gfn(addr); + union asce asce; int r; guard(read_lock)(&kvm->mmu_lock); + asce = kvm->arch.gmap->asce; switch (op) { case KVM_S390_KEYOP_SSKE: From 34d5b5b646c91cfb9338d7a12c955a70ffb8c66b Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Tue, 11 Aug 2026 17:37:36 +0200 Subject: [PATCH 55/82] KVM: s390: vsie: zero stale crypto bits When shadowing crypto access bits from a format0 apcb (crycb 0 or 1), the bits 64..255 are unchanged from whatever is in the vsie page in the crycb and thus in the apcb. This gives a nested guest potential access to a device no longer available. Zero out the remaining bits. Fixes: 6b79de4b056e ("KVM: s390: vsie: allow guest FORMAT-1 CRYCB on host FORMAT-2") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260811153738.206885-3-borntraeger@linux.ibm.com> --- arch/s390/kvm/vsie.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index eea24562e7db..0a7d8dfab6b7 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -173,6 +173,7 @@ static int setup_apcb10(struct kvm_vcpu *vcpu, struct kvm_s390_apcb1 *apcb_s, sizeof(struct kvm_s390_apcb0))) return -EFAULT; + memset(apcb_s, 0, sizeof(*apcb_s)); apcb_s->apm[0] = apcb_h->apm[0] & tmp.apm[0]; apcb_s->aqm[0] = apcb_h->aqm[0] & tmp.aqm[0] & 0xffff000000000000UL; apcb_s->adm[0] = apcb_h->adm[0] & tmp.adm[0] & 0xffff000000000000UL; From 1e3c8e7b3465fb8a49d3623d2d0f449c0b5b48f3 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Tue, 11 Aug 2026 17:37:37 +0200 Subject: [PATCH 56/82] KVM: s390: pv: Fix rc/rrc offset for PVM_DUMP The rc/rrc value is copied to the cmd location of the cmd in the kvm_pv_cmd structure. Fix the offset. Fixes: 8aba09588d2a ("KVM: s390: Add CPU dump functionality") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260811153738.206885-4-borntraeger@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index a3e55c613243..ed4190128812 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -5736,7 +5736,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = kvm_s390_handle_pv_vcpu_dump(vcpu, &cmd); /* Always copy over UV rc / rrc data */ - if (copy_to_user((__u8 __user *)argp, &cmd.rc, + if (copy_to_user(argp + offsetof(struct kvm_pv_cmd, rc), &cmd.rc, sizeof(cmd.rc) + sizeof(cmd.rrc))) r = -EFAULT; break; From c44d36d8e6501c4934412d9014e5e02da9efdb8f Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Tue, 11 Aug 2026 17:37:38 +0200 Subject: [PATCH 57/82] KVM: s390: Restore sigset on error path kvm_sigset_activate() installs vcpu->sigset via sigprocmask() and stashes the caller's mask in current->real_blocked; only kvm_sigset_deactivate() restores it. For KVM_RUN on a STOPPED vcpu the error path will not restore the userspace mask. Re-arrange the error handling to also restore the signal mask. Fixes: 6352e4d2dd9a3 ("KVM: s390: implement KVM_(S|G)ET_MP_STATE for user space state control") Cc: stable@vger.kernel.org Signed-off-by: Christian Borntraeger Reviewed-by: Claudio Imbrenda Signed-off-by: Claudio Imbrenda Message-ID: <20260811153738.206885-5-borntraeger@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index ed4190128812..17aa38710170 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -5073,7 +5073,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) pr_err_ratelimited("can't run stopped vcpu %d\n", vcpu->vcpu_id); rc = -EINVAL; - goto out; + goto out_sigset; } kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR); @@ -5103,9 +5103,11 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) store_regs(vcpu); kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR); + vcpu->stat.exit_userspace++; + +out_sigset: kvm_sigset_deactivate(vcpu); - vcpu->stat.exit_userspace++; out: vcpu_put(vcpu); return rc; From 88e22ffd1e46b95e40a6afabe486deb1d31a3ae1 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 12 Aug 2026 12:44:28 +0200 Subject: [PATCH 58/82] KVM: s390: Properly handle NULL pointer in dat_cond_set_storage_key() Some callers pass NULL as oldkey. Calling page_cond_set_storage_key() will cause that NULL pointer to get dereferenced. Fix by checking for NULL and assigning the pointer to a dummy local variable to avoid crashes. Fixes: 8e03e8316eb2 ("KVM: s390: KVM page table management functions: storage keys") Reviewed-by: Christian Borntraeger Reviewed-by: Christoph Schlameuss Signed-off-by: Claudio Imbrenda Message-ID: <20260812104436.109741-2-imbrenda@linux.ibm.com> --- arch/s390/kvm/dat.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 3f2d6e8902d7..165c704fcf29 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -722,9 +722,12 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf if (rc) return rc; - if (!ptep) + if (!ptep) { + if (!oldkey) + oldkey = &prev; return page_cond_set_storage_key(large_crste_to_phys(*crstep, gfn), skey, oldkey, nq, mr, mc); + } old = pgste_get_lock(ptep); pgste = old; From 062e44a9319f14b6f66a5e8accb98cfa873c7b0d Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 12 Aug 2026 12:44:29 +0200 Subject: [PATCH 59/82] KVM: s390: Use srcu in kvm_arch_vcpu_unlocked_ioctl() kvm_arch_vcpu_unlocked_ioctl() is called without further locks held, but kvm_s390_inject_vcpu(), which is called from there, needs either the kvm->srcu or the slots lock. Fix by taking the kvm->srcu in kvm_arch_vcpu_unlocked_ioctl(). Fixes: ba5c1e9b6cee ("KVM: s390: interrupt subsystem, cpu timer, waitpsw") Reviewed-by: Christian Borntraeger Reviewed-by: Christoph Schlameuss Signed-off-by: Claudio Imbrenda Message-ID: <20260812104436.109741-3-imbrenda@linux.ibm.com> --- arch/s390/kvm/kvm-s390.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 17aa38710170..91b975835457 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -5456,7 +5456,8 @@ long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, if (copy_from_user(&s390irq, argp, sizeof(s390irq))) return -EFAULT; - rc = kvm_s390_inject_vcpu(vcpu, &s390irq); + scoped_guard(srcu, &vcpu->kvm->srcu) + rc = kvm_s390_inject_vcpu(vcpu, &s390irq); break; } case KVM_S390_INTERRUPT: { @@ -5469,7 +5470,8 @@ long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, return -EFAULT; if (s390int_to_s390irq(&s390int, &s390irq)) return -EINVAL; - rc = kvm_s390_inject_vcpu(vcpu, &s390irq); + scoped_guard(srcu, &vcpu->kvm->srcu) + rc = kvm_s390_inject_vcpu(vcpu, &s390irq); break; } default: From ae50d472225895e0d2a2c176aedf907ec1342c07 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 12 Aug 2026 12:44:30 +0200 Subject: [PATCH 60/82] KVM: s390: Fix get_all_floating_irqs() When attempting to report all pending floating interrupt to userspace, the GISA IPM bits are atomically tested and cleared, and the corresponding interrupt description is written in the output buffer. If the output buffer is too small, an error is returned to userspace, but the GISA IPM bits are now lost. Moreover, the contract of KVM_DEV_FLIC_GET_ALL_IRQS, which is the only path to get_all_floating_irqs(), states that: > All interrupts remain pending, i.e. are not deleted from the list of > currently pending interrupts. Fix by non-destructively testing for the GISA IPM bits. Fixes: 24160af6cb28 ("KVM: s390: add GISA interrupts to FLIC ioctl interface") Reviewed-by: Christian Borntraeger Signed-off-by: Claudio Imbrenda Message-ID: <20260812104436.109741-4-imbrenda@linux.ibm.com> --- arch/s390/kvm/interrupt.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 0d9fbe75dc66..ca2f521092be 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -273,6 +273,11 @@ static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa); } +static inline int gisa_test_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) +{ + return test_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *)gisa); +} + static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu) { unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs | @@ -2242,7 +2247,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) ret = -ENOMEM; goto out_nolock; } - if (gisa_tac_ipm_gisc(gi->origin, i)) { + if (gisa_test_ipm_gisc(gi->origin, i)) { irq = (struct kvm_s390_irq *) &buf[n]; irq->type = KVM_S390_INT_IO(1, 0, 0, 0); irq->u.io.io_int_word = isc_to_int_word(i); From b5a941112fe46e4c30c93c0500efe641aef99840 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 12 Aug 2026 12:44:32 +0200 Subject: [PATCH 61/82] KVM: s390: Fix pgste_get_trylock_multiple() In case of failure, pgste_get_trylock_multiple() will attempt to unlock the locked PGSTEs based on whether the PCL is set. In some circumstances this can lead to unlocking PGSTEs that were locked by other threads. Fix by unlocking the amount of PGSTEs that were actually locked, ignoring the PCL bit in the array. Fixes: 94fd9b16cc67 ("KVM: s390: KVM page table management functions: lifecycle management") Reviewed-by: Christian Borntraeger Reviewed-by: Christoph Schlameuss Signed-off-by: Claudio Imbrenda Message-ID: <20260812104436.109741-6-imbrenda@linux.ibm.com> --- arch/s390/kvm/dat.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index 165c704fcf29..f4dd6f783417 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -923,11 +923,8 @@ static void pgste_set_unlock_multiple(union pte *first, int n, union pgste *pgst { int i; - for (i = 0; i < n; i++) { - if (!pgstes[i].pcl) - break; + for (i = 0; i < n; i++) pgste_set_unlock(first + i, pgstes[i]); - } } static bool pgste_get_trylock_multiple(union pte *first, int n, union pgste *pgstes) @@ -940,7 +937,7 @@ static bool pgste_get_trylock_multiple(union pte *first, int n, union pgste *pgs } if (i == n) return true; - pgste_set_unlock_multiple(first, n, pgstes); + pgste_set_unlock_multiple(first, i, pgstes); return false; } From d343407b728a80b74be3c24b59f15e60289ea527 Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 12 Aug 2026 12:44:33 +0200 Subject: [PATCH 62/82] KVM: s390: Fix IRQ injection with SIGP Stop and Store Status When __inject_sigp_stop() is called for a Stop and Store Status operation, if the vCPU is running, the interrupt is marked as pending and the status is stored by the thread performing the KVM_RUN IOCTL. If the vCPU is already stopped, the status is stored immediately. Storing the status means writing into userspace, which might fault, and __inject_sigp_stop() is called from do_inject_vcpu() which in turn is always called holding a spinlock, which is obviously an issue. Fix this by returning -EWOULDBLOCK from __inject_sigp_stop(), and adding a bool flag to indicate whether a store status is needed. The callers of do_inject_vcpu() are modified to pass the pointer to the bool flag; whenever a Store Status operation is needed, the callers can now perform it outside the spinlock. Opportunistically refactor kvm_s390_set_irq_state() to use scoped_guard() and __free(). Fixes: 6cddd432e3da ("KVM: s390: handle stop irqs without action_bits") Signed-off-by: Claudio Imbrenda [ Added Fixes tag while picking -- Claudio ] Message-ID: <20260812104436.109741-7-imbrenda@linux.ibm.com> --- arch/s390/kvm/interrupt.c | 72 +++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index ca2f521092be..da740a378a8c 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -1555,23 +1555,21 @@ static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) } #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS) -static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) +static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq, bool *storestatus) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; struct kvm_s390_stop_info *stop = &li->irq.stop; - int rc = 0; vcpu->stat.inject_stop_signal++; trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0); if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS) return -EINVAL; - if (is_vcpu_stopped(vcpu)) { - if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS) - rc = kvm_s390_store_status_unloaded(vcpu, - KVM_S390_STORE_STATUS_NOADDR); - return rc; + if (!(irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)) + return 0; + *storestatus = true; + return -EWOULDBLOCK; } if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs)) @@ -2107,7 +2105,7 @@ void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu) spin_unlock(&li->lock); } -static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) +static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq, bool *storestatus) { int rc; @@ -2119,7 +2117,7 @@ static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) rc = __inject_set_prefix(vcpu, irq); break; case KVM_S390_SIGP_STOP: - rc = __inject_sigp_stop(vcpu, irq); + rc = __inject_sigp_stop(vcpu, irq, storestatus); break; case KVM_S390_RESTART: rc = __inject_sigp_restart(vcpu); @@ -2155,11 +2153,16 @@ static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; + bool storestatus = false; int rc; spin_lock(&li->lock); - rc = do_inject_vcpu(vcpu, irq); + rc = do_inject_vcpu(vcpu, irq, &storestatus); spin_unlock(&li->lock); + + if (rc == -EWOULDBLOCK && storestatus) + rc = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); + if (!rc) kvm_s390_vcpu_wakeup(vcpu); return rc; @@ -3194,7 +3197,8 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len) { struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; - struct kvm_s390_irq *buf; + struct kvm_s390_irq *buf __free(kvfree) = NULL; + bool tmp, storestatus = false; int r = 0; int n; @@ -3202,32 +3206,34 @@ int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len if (!buf) return -ENOMEM; - if (copy_from_user((void *) buf, irqstate, len)) { - r = -EFAULT; - goto out_free; + if (copy_from_user((void *)buf, irqstate, len)) + return -EFAULT; + + scoped_guard(spinlock, &li->lock) { + /* + * Don't allow setting the interrupt state + * when there are already interrupts pending + */ + if (li->pending_irqs) + return -EBUSY; + + for (n = 0; n < len / sizeof(*buf); n++) { + tmp = false; + r = do_inject_vcpu(vcpu, &buf[n], &tmp); + if (r == -EWOULDBLOCK && tmp) { + storestatus = true; + r = 0; + } + if (r) + break; + } } - /* - * Don't allow setting the interrupt state - * when there are already interrupts pending - */ - spin_lock(&li->lock); - if (li->pending_irqs) { - r = -EBUSY; - goto out_unlock; + if (storestatus) { + n = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); + return r ? r : n; } - for (n = 0; n < len / sizeof(*buf); n++) { - r = do_inject_vcpu(vcpu, &buf[n]); - if (r) - break; - } - -out_unlock: - spin_unlock(&li->lock); -out_free: - vfree(buf); - return r; } From 6b1130e1b4715f5aab4768111c26bd718a14686a Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 12 Aug 2026 12:44:34 +0200 Subject: [PATCH 63/82] KVM: s390: Fix kvm_s390_clear_pv_state() kvm_s390_clear_pv_state() needs to also clear the dumping flag, to allow the protected VM to be started again (as non-protected, with all protected state safely destroyed) after a forced reboot while a protected dump was ongoing and not completed. Fixes: e40df9efd68a ("KVM: s390: pv: clear the state without memset") Reviewed-by: Christian Borntraeger Signed-off-by: Claudio Imbrenda Message-ID: <20260812104436.109741-8-imbrenda@linux.ibm.com> --- arch/s390/kvm/pv.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c index b02e0159d3cd..98a9a57f71b9 100644 --- a/arch/s390/kvm/pv.c +++ b/arch/s390/kvm/pv.c @@ -242,6 +242,10 @@ static void kvm_s390_clear_pv_state(struct kvm *kvm) kvm->arch.pv.guest_len = 0; kvm->arch.pv.stor_base = 0; kvm->arch.pv.stor_var = NULL; + if (kvm->arch.pv.dumping) { + kvm_s390_vcpu_unblock_all(kvm); + kvm->arch.pv.dumping = false; + } } static void kvm_s390_pv_dispose_cpu(struct kvm_vcpu *vcpu, bool free_stor_base) From 30300ec2780efa6cb9ea0887d1b94506af9c4eea Mon Sep 17 00:00:00 2001 From: Claudio Imbrenda Date: Wed, 12 Aug 2026 12:44:35 +0200 Subject: [PATCH 64/82] KVM: s390: Fix potential tiny kernel stack leak In some circumstances, one bit of kernel stack could have been leaked from dat_cond_set_storage_key(). Fix by clearing prev before use. Fixes: 8e03e8316eb2 ("KVM: s390: KVM page table management functions: storage keys") Reviewed-by: Christian Borntraeger Signed-off-by: Claudio Imbrenda Message-ID: <20260812104436.109741-9-imbrenda@linux.ibm.com> --- arch/s390/kvm/dat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c index f4dd6f783417..f2ea013cb33e 100644 --- a/arch/s390/kvm/dat.c +++ b/arch/s390/kvm/dat.c @@ -737,6 +737,7 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf pgste.fp = skey.fp; pgste.gc = skey.c; pgste.gr = skey.r; + prev.skey = 0; if (!ptep->h.i) { rc = page_cond_set_storage_key(pte_origin(*ptep), skey, &prev, nq, mr, mc); From 6ece2811aa3fd8a83abb819f38b2c6f1c332f3c1 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 3 Aug 2026 13:01:36 +0200 Subject: [PATCH 65/82] KVM: s390: Extract gmap tracing to a separate header Move the kvm_s390_major_guest_pfault trace event from trace.h to a new trace_gmap.h header. This separates gmap-specific tracing from general KVM/s390 tracing, preparing for code sharing between multiple KVM implementations. The trace event definition is updated to use local defines for parameters so that they can be replaced later with ease for when another KVM implementation uses these traces. No functional change. Signed-off-by: Steffen Eiden Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/faultin.c | 3 ++- arch/s390/kvm/trace.h | 14 ------------ arch/s390/kvm/trace_gmap.h | 47 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 15 deletions(-) create mode 100644 arch/s390/kvm/trace_gmap.h diff --git a/arch/s390/kvm/faultin.c b/arch/s390/kvm/faultin.c index 3cc45f7f5b2d..1dc79807012c 100644 --- a/arch/s390/kvm/faultin.c +++ b/arch/s390/kvm/faultin.c @@ -9,10 +9,11 @@ #include #include "gmap.h" -#include "trace.h" #include "faultin.h" bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); +#define CREATE_TRACE_POINTS +#include "trace_gmap.h" /* * kvm_s390_faultin_gfn() - handle a dat fault. diff --git a/arch/s390/kvm/trace.h b/arch/s390/kvm/trace.h index dd2685c7df5b..3d2fffb1555a 100644 --- a/arch/s390/kvm/trace.h +++ b/arch/s390/kvm/trace.h @@ -45,20 +45,6 @@ TRACE_EVENT(kvm_s390_skey_related_inst, VCPU_TP_PRINTK("%s", "storage key related instruction") ); -TRACE_EVENT(kvm_s390_major_guest_pfault, - TP_PROTO(VCPU_PROTO_COMMON), - TP_ARGS(VCPU_ARGS_COMMON), - - TP_STRUCT__entry( - VCPU_FIELD_COMMON - ), - - TP_fast_assign( - VCPU_ASSIGN_COMMON - ), - VCPU_TP_PRINTK("%s", "major fault, maybe applicable for pfault") - ); - TRACE_EVENT(kvm_s390_pfault_init, TP_PROTO(VCPU_PROTO_COMMON, long pfault_token), TP_ARGS(VCPU_ARGS_COMMON, pfault_token), diff --git a/arch/s390/kvm/trace_gmap.h b/arch/s390/kvm/trace_gmap.h new file mode 100644 index 000000000000..a3cad705021b --- /dev/null +++ b/arch/s390/kvm/trace_gmap.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(GMAP_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define GMAP_TRACE_KVM_H + +#include + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace_gmap + +#define __KVM_FIELDS \ + __field(unsigned long, pswmask) \ + __field(unsigned long, pswaddr) +#define __KVM_ASSIGN ({\ + __entry->pswmask = vcpu->arch.sie_block->gpsw.mask; \ + __entry->pswaddr = vcpu->arch.sie_block->gpsw.addr; \ + }) +#define __KVM_PRINT \ + __entry->pswmask,\ + __entry->pswaddr + +TRACE_EVENT(kvm_s390_major_guest_pfault, + TP_PROTO(struct kvm_vcpu *vcpu), + TP_ARGS(vcpu), + + TP_STRUCT__entry( + __field(int, id) + __KVM_FIELDS + ), + + TP_fast_assign( + __entry->id = vcpu->vcpu_id; + __KVM_ASSIGN + ), + TP_printk("%02d[%016lx-%016lx]: major fault, maybe applicable for pfault", + __entry->id, + __KVM_PRINT + ) + ); + +#endif /* GMAP_TRACE_KVM_H */ + +/* This part must be outside protection */ +#include From 8019bc6830fd3f99dc0e00b3cc342ab19da38196 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 3 Aug 2026 13:03:08 +0200 Subject: [PATCH 66/82] KVM: s390: Prepare include guards for a new location Update include guard names in dat.h, faultin.h, and gmap.h to ARCH_KVM_GMAP_* to reflect their upcoming relocation to a shared gmap directory. No functional change. Signed-off-by: Steffen Eiden Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/dat.h | 6 +++--- arch/s390/kvm/faultin.h | 6 +++--- arch/s390/kvm/gmap.h | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h index 141ee7b9f019..87108f51228e 100644 --- a/arch/s390/kvm/dat.h +++ b/arch/s390/kvm/dat.h @@ -6,8 +6,8 @@ * Author(s): Claudio Imbrenda */ -#ifndef __KVM_S390_DAT_H -#define __KVM_S390_DAT_H +#ifndef ARCH_KVM_GMAP_DAT_H +#define ARCH_KVM_GMAP_DAT_H #include #include @@ -975,4 +975,4 @@ static inline bool crste_is_ucas(union crste crste) return is_pmd(crste) && crste.h.i && crste.h.fc0.tl == 1 && crste.h.fc == 0; } -#endif /* __KVM_S390_DAT_H */ +#endif /* ARCH_KVM_GMAP_DAT_H */ diff --git a/arch/s390/kvm/faultin.h b/arch/s390/kvm/faultin.h index f86176d2769c..f343b6fb6f16 100644 --- a/arch/s390/kvm/faultin.h +++ b/arch/s390/kvm/faultin.h @@ -6,8 +6,8 @@ * Author(s): Claudio Imbrenda */ -#ifndef __KVM_S390_FAULTIN_H -#define __KVM_S390_FAULTIN_H +#ifndef ARCH_KVM_GMAP_FAULTIN_H +#define ARCH_KVM_GMAP_FAULTIN_H #include @@ -89,4 +89,4 @@ static inline int kvm_s390_get_guest_pages(struct kvm *kvm, struct guest_fault * #define kvm_s390_array_needs_retry_safe(kvm, seq, array) \ kvm_s390_multiple_faults_need_retry(kvm, seq, array, ARRAY_SIZE(array), false) -#endif /* __KVM_S390_FAULTIN_H */ +#endif /* ARCH_KVM_GMAP_FAULTIN_H */ diff --git a/arch/s390/kvm/gmap.h b/arch/s390/kvm/gmap.h index 39938d363ec9..c54c35e47d6d 100644 --- a/arch/s390/kvm/gmap.h +++ b/arch/s390/kvm/gmap.h @@ -7,8 +7,8 @@ * Claudio Imbrenda */ -#ifndef ARCH_KVM_S390_GMAP_H -#define ARCH_KVM_S390_GMAP_H +#ifndef ARCH_KVM_GMAP_GMAP_H +#define ARCH_KVM_GMAP_GMAP_H #include "dat.h" @@ -330,4 +330,4 @@ static inline bool gmap_is_shadow_valid(struct gmap *sg, union asce asce, int ed return sg->guest_asce.val == asce.val && sg->edat_level == edat_level; } -#endif /* ARCH_KVM_S390_GMAP_H */ +#endif /* ARCH_KVM_GMAP_GMAP_H */ From 64e9b55437ee81ea7d5be6a37ef3bd7b2ef764b9 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 3 Aug 2026 13:05:08 +0200 Subject: [PATCH 67/82] KVM: s390: Rename kvm-s390.{c,h} to s390.{c,h} Rename kvm-s390.c to s390.c and kvm-s390.h to s390.h for consistency with the new directory structure. Update all include statements and simplify Makefile ccflags from explicit paths to -I$(src). No functional change. Signed-off-by: Steffen Eiden Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/Makefile | 4 ++-- arch/s390/kvm/diag.c | 2 +- arch/s390/kvm/gaccess.c | 2 +- arch/s390/kvm/gaccess.h | 2 +- arch/s390/kvm/gmap.c | 2 +- arch/s390/kvm/guestdbg.c | 2 +- arch/s390/kvm/intercept.c | 2 +- arch/s390/kvm/interrupt.c | 2 +- arch/s390/kvm/pci.c | 2 +- arch/s390/kvm/priv.c | 2 +- arch/s390/kvm/pv.c | 2 +- arch/s390/kvm/{kvm-s390.c => s390.c} | 2 +- arch/s390/kvm/{kvm-s390.h => s390.h} | 2 +- arch/s390/kvm/sigp.c | 2 +- arch/s390/kvm/vsie.c | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) rename arch/s390/kvm/{kvm-s390.c => s390.c} (99%) rename arch/s390/kvm/{kvm-s390.h => s390.h} (99%) diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile index dac9d53b23d8..961de97b8e80 100644 --- a/arch/s390/kvm/Makefile +++ b/arch/s390/kvm/Makefile @@ -5,9 +5,9 @@ include $(srctree)/virt/kvm/Makefile.kvm -ccflags-y := -Ivirt/kvm -Iarch/s390/kvm +ccflags-y := -I$(src) -kvm-y += kvm-s390.o intercept.o interrupt.o priv.o sigp.o +kvm-y += s390.o intercept.o interrupt.o priv.o sigp.o kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o kvm-y += dat.o gmap.o faultin.o diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c index 031ab6e5d6c4..09033c8ff1bb 100644 --- a/arch/s390/kvm/diag.c +++ b/arch/s390/kvm/diag.c @@ -12,7 +12,7 @@ #include #include #include -#include "kvm-s390.h" +#include "s390.h" #include "trace.h" #include "trace-s390.h" #include "gaccess.h" diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 36102b2727fb..2a4b2525329a 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -17,7 +17,7 @@ #include #include #include -#include "kvm-s390.h" +#include "s390.h" #include "dat.h" #include "gmap.h" #include "gaccess.h" diff --git a/arch/s390/kvm/gaccess.h b/arch/s390/kvm/gaccess.h index b5385cec60f4..ef922b3b4990 100644 --- a/arch/s390/kvm/gaccess.h +++ b/arch/s390/kvm/gaccess.h @@ -14,7 +14,7 @@ #include #include #include -#include "kvm-s390.h" +#include "s390.h" /** * kvm_s390_real_to_abs - convert guest real address to guest absolute address diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c index 8abb4f55b306..9ae55a1d6f09 100644 --- a/arch/s390/kvm/gmap.c +++ b/arch/s390/kvm/gmap.c @@ -21,7 +21,7 @@ #include "dat.h" #include "gmap.h" -#include "kvm-s390.h" +#include "s390.h" #include "faultin.h" static inline bool kvm_s390_is_in_sie(struct kvm_vcpu *vcpu) diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/guestdbg.c index 9a6149e310bb..1bf7e91b7e61 100644 --- a/arch/s390/kvm/guestdbg.c +++ b/arch/s390/kvm/guestdbg.c @@ -8,7 +8,7 @@ */ #include #include -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" /* diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c index 1980df61ef30..ca1205dfac8b 100644 --- a/arch/s390/kvm/intercept.c +++ b/arch/s390/kvm/intercept.c @@ -17,7 +17,7 @@ #include #include -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" #include "trace.h" #include "trace-s390.h" diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index da740a378a8c..0381ae981703 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -29,7 +29,7 @@ #include #include #include -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" #include "trace-s390.h" #include "pci.h" diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 50f5ec79600e..82892e1e03d9 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -14,7 +14,7 @@ #include #include #include "pci.h" -#include "kvm-s390.h" +#include "s390.h" struct zpci_aift *aift; diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index b1ba24c346ef..b3cb2c2c3aa7 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -26,7 +26,7 @@ #include #include #include "gaccess.h" -#include "kvm-s390.h" +#include "s390.h" #include "trace.h" #include "gmap.h" diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/pv.c index 98a9a57f71b9..e7489080c1c5 100644 --- a/arch/s390/kvm/pv.c +++ b/arch/s390/kvm/pv.c @@ -18,7 +18,7 @@ #include #include #include -#include "kvm-s390.h" +#include "s390.h" #include "dat.h" #include "gaccess.h" #include "gmap.h" diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/s390.c similarity index 99% rename from arch/s390/kvm/kvm-s390.c rename to arch/s390/kvm/s390.c index 619b0957f812..bb8ef40e3ab5 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/s390.c @@ -50,7 +50,7 @@ #include #include #include -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" #include "gmap.h" #include "faultin.h" diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/s390.h similarity index 99% rename from arch/s390/kvm/kvm-s390.h rename to arch/s390/kvm/s390.h index 6d2842fb71a3..e144ca2e6d5e 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/s390.h @@ -452,7 +452,7 @@ void kvm_s390_vsie_destroy(struct kvm *kvm); int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu); int kvm_s390_handle_sigp_pei(struct kvm_vcpu *vcpu); -/* implemented in kvm-s390.c */ +/* implemented in s390.c */ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod); int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr); int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr); diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c index 55c34cb35428..131b3371ef4f 100644 --- a/arch/s390/kvm/sigp.c +++ b/arch/s390/kvm/sigp.c @@ -14,7 +14,7 @@ #include #include #include "gaccess.h" -#include "kvm-s390.h" +#include "s390.h" #include "trace.h" static int __sigp_sense(struct kvm_vcpu *vcpu, struct kvm_vcpu *dst_vcpu, diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index f55e4e9551e3..2bdd516b4d5c 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -23,7 +23,7 @@ #include #include #include -#include "kvm-s390.h" +#include "s390.h" #include "gaccess.h" #include "gmap.h" From 3da0913c6945c4ff116c9949df75d3ef1f8f8fa6 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 3 Aug 2026 13:07:00 +0200 Subject: [PATCH 68/82] KVM: s390: Move kvm_host definitions to kvm_host_s390 Rename kvm_host.h to kvm_host_s390.h and kvm_host_types.h to kvm_host_s390_types.h to distinguish s390-specific KVM definitions from the generic kvm_host.h that will be used for shared definitions. No functional change. Signed-off-by: Steffen Eiden Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/include/asm/kvm_host.h | 780 +------------------ arch/s390/include/asm/kvm_host_s390.h | 784 ++++++++++++++++++++ arch/s390/include/asm/kvm_host_s390_types.h | 349 +++++++++ arch/s390/include/asm/kvm_host_types.h | 349 +-------- 4 files changed, 1139 insertions(+), 1123 deletions(-) create mode 100644 arch/s390/include/asm/kvm_host_s390.h create mode 100644 arch/s390/include/asm/kvm_host_s390_types.h diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index b4182ca4435f..e76ceee11ef5 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -1,784 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * definition for kernel virtual machines on s390 - * - * Copyright IBM Corp. 2008, 2018 - * - * Author(s): Carsten Otte - */ - #ifndef ASM_KVM_HOST_H #define ASM_KVM_HOST_H -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include -#define KVM_HAVE_MMU_RWLOCK -#define KVM_MAX_VCPUS 255 - -#define KVM_INTERNAL_MEM_SLOTS 1 - -/* - * These seem to be used for allocating ->chip in the routing table, which we - * don't use. 1 is as small as we can get to reduce the needed memory. If we - * need to look at ->chip later on, we'll need to revisit this. - */ -#define KVM_NR_IRQCHIPS 1 -#define KVM_IRQCHIP_NUM_PINS 1 -#define KVM_HALT_POLL_NS_DEFAULT 50000 - -/* s390-specific vcpu->requests bit members */ -#define KVM_REQ_ENABLE_IBS KVM_ARCH_REQ(0) -#define KVM_REQ_DISABLE_IBS KVM_ARCH_REQ(1) -#define KVM_REQ_ICPT_OPEREXC KVM_ARCH_REQ(2) -#define KVM_REQ_START_MIGRATION KVM_ARCH_REQ(3) -#define KVM_REQ_STOP_MIGRATION KVM_ARCH_REQ(4) -#define KVM_REQ_VSIE_RESTART KVM_ARCH_REQ(5) -#define KVM_REQ_REFRESH_GUEST_PREFIX \ - KVM_ARCH_REQ_FLAGS(6, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) - -struct kvm_vcpu_stat { - struct kvm_vcpu_stat_generic generic; - u64 exit_userspace; - u64 exit_null; - u64 exit_external_request; - u64 exit_io_request; - u64 exit_external_interrupt; - u64 exit_stop_request; - u64 exit_validity; - u64 exit_instruction; - u64 exit_pei; - u64 halt_no_poll_steal; - u64 instruction_lctl; - u64 instruction_lctlg; - u64 instruction_stctl; - u64 instruction_stctg; - u64 exit_program_interruption; - u64 exit_instr_and_program; - u64 exit_operation_exception; - u64 deliver_ckc; - u64 deliver_cputm; - u64 deliver_external_call; - u64 deliver_emergency_signal; - u64 deliver_service_signal; - u64 deliver_virtio; - u64 deliver_stop_signal; - u64 deliver_prefix_signal; - u64 deliver_restart_signal; - u64 deliver_program; - u64 deliver_io; - u64 deliver_machine_check; - u64 exit_wait_state; - u64 inject_ckc; - u64 inject_cputm; - u64 inject_external_call; - u64 inject_emergency_signal; - u64 inject_mchk; - u64 inject_pfault_init; - u64 inject_program; - u64 inject_restart; - u64 inject_set_prefix; - u64 inject_stop_signal; - u64 instruction_epsw; - u64 instruction_gs; - u64 instruction_io_other; - u64 instruction_lpsw; - u64 instruction_lpswe; - u64 instruction_lpswey; - u64 instruction_pfmf; - u64 instruction_ptff; - u64 instruction_sck; - u64 instruction_sckpf; - u64 instruction_stidp; - u64 instruction_spx; - u64 instruction_stpx; - u64 instruction_stap; - u64 instruction_iske; - u64 instruction_ri; - u64 instruction_rrbe; - u64 instruction_sske; - u64 instruction_ipte_interlock; - u64 instruction_stsi; - u64 instruction_stfl; - u64 instruction_tb; - u64 instruction_tpi; - u64 instruction_tprot; - u64 instruction_tsch; - u64 instruction_sie; - u64 instruction_essa; - u64 instruction_sthyi; - u64 instruction_sigp_sense; - u64 instruction_sigp_sense_running; - u64 instruction_sigp_external_call; - u64 instruction_sigp_emergency; - u64 instruction_sigp_cond_emergency; - u64 instruction_sigp_start; - u64 instruction_sigp_stop; - u64 instruction_sigp_stop_store_status; - u64 instruction_sigp_store_status; - u64 instruction_sigp_store_adtl_status; - u64 instruction_sigp_arch; - u64 instruction_sigp_prefix; - u64 instruction_sigp_restart; - u64 instruction_sigp_init_cpu_reset; - u64 instruction_sigp_cpu_reset; - u64 instruction_sigp_unknown; - u64 instruction_diagnose_10; - u64 instruction_diagnose_44; - u64 instruction_diagnose_9c; - u64 diag_9c_ignored; - u64 diag_9c_forward; - u64 instruction_diagnose_258; - u64 instruction_diagnose_308; - u64 instruction_diagnose_500; - u64 instruction_diagnose_other; - u64 pfault_sync; - u64 signal_exits; -}; - -#define PGM_OPERATION 0x01 -#define PGM_PRIVILEGED_OP 0x02 -#define PGM_EXECUTE 0x03 -#define PGM_PROTECTION 0x04 -#define PGM_ADDRESSING 0x05 -#define PGM_SPECIFICATION 0x06 -#define PGM_DATA 0x07 -#define PGM_FIXED_POINT_OVERFLOW 0x08 -#define PGM_FIXED_POINT_DIVIDE 0x09 -#define PGM_DECIMAL_OVERFLOW 0x0a -#define PGM_DECIMAL_DIVIDE 0x0b -#define PGM_HFP_EXPONENT_OVERFLOW 0x0c -#define PGM_HFP_EXPONENT_UNDERFLOW 0x0d -#define PGM_HFP_SIGNIFICANCE 0x0e -#define PGM_HFP_DIVIDE 0x0f -#define PGM_SEGMENT_TRANSLATION 0x10 -#define PGM_PAGE_TRANSLATION 0x11 -#define PGM_TRANSLATION_SPEC 0x12 -#define PGM_SPECIAL_OPERATION 0x13 -#define PGM_OPERAND 0x15 -#define PGM_TRACE_TABEL 0x16 -#define PGM_VECTOR_PROCESSING 0x1b -#define PGM_SPACE_SWITCH 0x1c -#define PGM_HFP_SQUARE_ROOT 0x1d -#define PGM_PC_TRANSLATION_SPEC 0x1f -#define PGM_AFX_TRANSLATION 0x20 -#define PGM_ASX_TRANSLATION 0x21 -#define PGM_LX_TRANSLATION 0x22 -#define PGM_EX_TRANSLATION 0x23 -#define PGM_PRIMARY_AUTHORITY 0x24 -#define PGM_SECONDARY_AUTHORITY 0x25 -#define PGM_LFX_TRANSLATION 0x26 -#define PGM_LSX_TRANSLATION 0x27 -#define PGM_ALET_SPECIFICATION 0x28 -#define PGM_ALEN_TRANSLATION 0x29 -#define PGM_ALE_SEQUENCE 0x2a -#define PGM_ASTE_VALIDITY 0x2b -#define PGM_ASTE_SEQUENCE 0x2c -#define PGM_EXTENDED_AUTHORITY 0x2d -#define PGM_LSTE_SEQUENCE 0x2e -#define PGM_ASTE_INSTANCE 0x2f -#define PGM_STACK_FULL 0x30 -#define PGM_STACK_EMPTY 0x31 -#define PGM_STACK_SPECIFICATION 0x32 -#define PGM_STACK_TYPE 0x33 -#define PGM_STACK_OPERATION 0x34 -#define PGM_ASCE_TYPE 0x38 -#define PGM_REGION_FIRST_TRANS 0x39 -#define PGM_REGION_SECOND_TRANS 0x3a -#define PGM_REGION_THIRD_TRANS 0x3b -#define PGM_SECURE_STORAGE_ACCESS 0x3d -#define PGM_NON_SECURE_STORAGE_ACCESS 0x3e -#define PGM_SECURE_STORAGE_VIOLATION 0x3f -#define PGM_MONITOR 0x40 -#define PGM_PER 0x80 -#define PGM_CRYPTO_OPERATION 0x119 - -/* irq types in ascend order of priorities */ -enum irq_types { - IRQ_PEND_SET_PREFIX = 0, - IRQ_PEND_RESTART, - IRQ_PEND_SIGP_STOP, - IRQ_PEND_IO_ISC_7, - IRQ_PEND_IO_ISC_6, - IRQ_PEND_IO_ISC_5, - IRQ_PEND_IO_ISC_4, - IRQ_PEND_IO_ISC_3, - IRQ_PEND_IO_ISC_2, - IRQ_PEND_IO_ISC_1, - IRQ_PEND_IO_ISC_0, - IRQ_PEND_VIRTIO, - IRQ_PEND_PFAULT_DONE, - IRQ_PEND_PFAULT_INIT, - IRQ_PEND_EXT_HOST, - IRQ_PEND_EXT_SERVICE, - IRQ_PEND_EXT_SERVICE_EV, - IRQ_PEND_EXT_TIMING, - IRQ_PEND_EXT_CPU_TIMER, - IRQ_PEND_EXT_CLOCK_COMP, - IRQ_PEND_EXT_EXTERNAL, - IRQ_PEND_EXT_EMERGENCY, - IRQ_PEND_EXT_MALFUNC, - IRQ_PEND_EXT_IRQ_KEY, - IRQ_PEND_MCHK_REP, - IRQ_PEND_PROG, - IRQ_PEND_SVC, - IRQ_PEND_MCHK_EX, - IRQ_PEND_COUNT -}; - -/* We have 2M for virtio device descriptor pages. Smallest amount of - * memory per page is 24 bytes (1 queue), so (2048*1024) / 24 = 87381 - */ -#define KVM_S390_MAX_VIRTIO_IRQS 87381 - -/* - * Repressible (non-floating) machine check interrupts - * subclass bits in MCIC - */ -#define MCHK_EXTD_BIT 58 -#define MCHK_DEGR_BIT 56 -#define MCHK_WARN_BIT 55 -#define MCHK_REP_MASK ((1UL << MCHK_DEGR_BIT) | \ - (1UL << MCHK_EXTD_BIT) | \ - (1UL << MCHK_WARN_BIT)) - -/* Exigent machine check interrupts subclass bits in MCIC */ -#define MCHK_SD_BIT 63 -#define MCHK_PD_BIT 62 -#define MCHK_EX_MASK ((1UL << MCHK_SD_BIT) | (1UL << MCHK_PD_BIT)) - -#define IRQ_PEND_EXT_MASK ((1UL << IRQ_PEND_EXT_IRQ_KEY) | \ - (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ - (1UL << IRQ_PEND_EXT_CPU_TIMER) | \ - (1UL << IRQ_PEND_EXT_MALFUNC) | \ - (1UL << IRQ_PEND_EXT_EMERGENCY) | \ - (1UL << IRQ_PEND_EXT_EXTERNAL) | \ - (1UL << IRQ_PEND_EXT_TIMING) | \ - (1UL << IRQ_PEND_EXT_HOST) | \ - (1UL << IRQ_PEND_EXT_SERVICE) | \ - (1UL << IRQ_PEND_EXT_SERVICE_EV) | \ - (1UL << IRQ_PEND_VIRTIO) | \ - (1UL << IRQ_PEND_PFAULT_INIT) | \ - (1UL << IRQ_PEND_PFAULT_DONE)) - -#define IRQ_PEND_IO_MASK ((1UL << IRQ_PEND_IO_ISC_0) | \ - (1UL << IRQ_PEND_IO_ISC_1) | \ - (1UL << IRQ_PEND_IO_ISC_2) | \ - (1UL << IRQ_PEND_IO_ISC_3) | \ - (1UL << IRQ_PEND_IO_ISC_4) | \ - (1UL << IRQ_PEND_IO_ISC_5) | \ - (1UL << IRQ_PEND_IO_ISC_6) | \ - (1UL << IRQ_PEND_IO_ISC_7)) - -#define IRQ_PEND_MCHK_MASK ((1UL << IRQ_PEND_MCHK_REP) | \ - (1UL << IRQ_PEND_MCHK_EX)) - -#define IRQ_PEND_EXT_II_MASK ((1UL << IRQ_PEND_EXT_CPU_TIMER) | \ - (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ - (1UL << IRQ_PEND_EXT_EMERGENCY) | \ - (1UL << IRQ_PEND_EXT_EXTERNAL) | \ - (1UL << IRQ_PEND_EXT_SERVICE) | \ - (1UL << IRQ_PEND_EXT_SERVICE_EV)) - -struct kvm_s390_interrupt_info { - struct list_head list; - u64 type; - union { - struct kvm_s390_io_info io; - struct kvm_s390_ext_info ext; - struct kvm_s390_pgm_info pgm; - struct kvm_s390_emerg_info emerg; - struct kvm_s390_extcall_info extcall; - struct kvm_s390_prefix_info prefix; - struct kvm_s390_stop_info stop; - struct kvm_s390_mchk_info mchk; - }; -}; - -struct kvm_s390_irq_payload { - struct kvm_s390_io_info io; - struct kvm_s390_ext_info ext; - struct kvm_s390_pgm_info pgm; - struct kvm_s390_emerg_info emerg; - struct kvm_s390_extcall_info extcall; - struct kvm_s390_prefix_info prefix; - struct kvm_s390_stop_info stop; - struct kvm_s390_mchk_info mchk; -}; - -struct kvm_s390_local_interrupt { - spinlock_t lock; - DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS); - struct kvm_s390_irq_payload irq; - unsigned long pending_irqs; -}; - -#define FIRQ_LIST_IO_ISC_0 0 -#define FIRQ_LIST_IO_ISC_1 1 -#define FIRQ_LIST_IO_ISC_2 2 -#define FIRQ_LIST_IO_ISC_3 3 -#define FIRQ_LIST_IO_ISC_4 4 -#define FIRQ_LIST_IO_ISC_5 5 -#define FIRQ_LIST_IO_ISC_6 6 -#define FIRQ_LIST_IO_ISC_7 7 -#define FIRQ_LIST_PFAULT 8 -#define FIRQ_LIST_VIRTIO 9 -#define FIRQ_LIST_COUNT 10 -#define FIRQ_CNTR_IO 0 -#define FIRQ_CNTR_SERVICE 1 -#define FIRQ_CNTR_VIRTIO 2 -#define FIRQ_CNTR_PFAULT 3 -#define FIRQ_MAX_COUNT 4 - -/* mask the AIS mode for a given ISC */ -#define AIS_MODE_MASK(isc) (0x80 >> isc) - -#define KVM_S390_AIS_MODE_ALL 0 -#define KVM_S390_AIS_MODE_SINGLE 1 - -struct kvm_s390_float_interrupt { - unsigned long pending_irqs; - unsigned long masked_irqs; - spinlock_t lock; - struct list_head lists[FIRQ_LIST_COUNT]; - int counters[FIRQ_MAX_COUNT]; - struct kvm_s390_mchk_info mchk; - struct kvm_s390_ext_info srv_signal; - int last_sleep_cpu; - spinlock_t ais_lock; - u8 simm; - u8 nimm; -}; - -struct kvm_hw_wp_info_arch { - unsigned long addr; - unsigned long phys_addr; - int len; - char *old_data; -}; - -struct kvm_hw_bp_info_arch { - unsigned long addr; - int len; -}; - -/* - * Only the upper 16 bits of kvm_guest_debug->control are arch specific. - * Further KVM_GUESTDBG flags which an be used from userspace can be found in - * arch/s390/include/uapi/asm/kvm.h - */ -#define KVM_GUESTDBG_EXIT_PENDING 0x10000000 - -#define guestdbg_enabled(vcpu) \ - (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) -#define guestdbg_sstep_enabled(vcpu) \ - (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) -#define guestdbg_hw_bp_enabled(vcpu) \ - (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) -#define guestdbg_exit_pending(vcpu) (guestdbg_enabled(vcpu) && \ - (vcpu->guest_debug & KVM_GUESTDBG_EXIT_PENDING)) - -#define KVM_GUESTDBG_VALID_MASK \ - (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP |\ - KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_EXIT_PENDING) - -struct kvm_guestdbg_info_arch { - unsigned long cr0; - unsigned long cr9; - unsigned long cr10; - unsigned long cr11; - struct kvm_hw_bp_info_arch *hw_bp_info; - struct kvm_hw_wp_info_arch *hw_wp_info; - int nr_hw_bp; - int nr_hw_wp; - unsigned long last_bp; -}; - -struct kvm_s390_pv_vcpu { - u64 handle; - unsigned long stor_base; -}; - -struct kvm_vcpu_arch { - struct kvm_s390_sie_block *sie_block; - /* if vsie is active, currently executed shadow sie control block */ - struct kvm_s390_sie_block *vsie_block; - unsigned int host_acrs[NUM_ACRS]; - struct gs_cb *host_gscb; - struct kvm_s390_local_interrupt local_int; - struct hrtimer ckc_timer; - struct kvm_s390_pgm_info pgm; - struct gmap *gmap; - struct kvm_guestdbg_info_arch guestdbg; - unsigned long pfault_token; - unsigned long pfault_select; - unsigned long pfault_compare; - bool cputm_enabled; - /* - * The seqcount protects updates to cputm_start and sie_block.cputm, - * this way we can have non-blocking reads with consistent values. - * Only the owning VCPU thread (vcpu->cpu) is allowed to change these - * values and to start/stop/enable/disable cpu timer accounting. - */ - seqcount_t cputm_seqcount; - __u64 cputm_start; - bool gs_enabled; - bool skey_enabled; - /* Indicator if the access registers have been loaded from guest */ - bool acrs_loaded; - bool initialized; - struct kvm_s390_pv_vcpu pv; - union diag318_info diag318_info; - struct kvm_s390_mmu_cache *mc; -}; - -struct kvm_vm_stat { - struct kvm_vm_stat_generic generic; - u64 inject_io; - u64 io_390_adapter_map; - u64 io_390_adapter_unmap; - u64 io_390_inatomic; - u64 io_flic_inject_airq; - u64 io_set_adapter_int; - u64 io_390_inatomic_no_inject; - u64 inject_float_mchk; - u64 inject_pfault_done; - u64 inject_service_signal; - u64 inject_virtio; - u64 aen_forward; - u64 gmap_shadow_create; - u64 gmap_shadow_reuse; - u64 gmap_shadow_r1_entry; - u64 gmap_shadow_r2_entry; - u64 gmap_shadow_r3_entry; - u64 gmap_shadow_sg_entry; - u64 gmap_shadow_pg_entry; -}; - -struct kvm_arch_memory_slot { -}; - -struct s390_map_info { - struct list_head list; - __u64 guest_addr; - __u64 addr; - struct page *page; - /* - * True if the page is long-term pinned. False if long-term pinning - * failed and this entry exists only to preserve MAP/UNMAP symmetry. - */ - bool pinned; -}; - -struct s390_io_adapter { - unsigned int id; - int isc; - bool maskable; - bool masked; - bool swap; - bool suppressible; - spinlock_t maps_lock; - struct list_head maps; - unsigned int nr_maps; -}; - -#define MAX_S390_IO_ADAPTERS ((MAX_ISC + 1) * 8) -#define MAX_S390_ADAPTER_MAPS 256 - -/* maximum size of facilities and facility mask is 2k bytes */ -#define S390_ARCH_FAC_LIST_SIZE_BYTE (1<<11) -#define S390_ARCH_FAC_LIST_SIZE_U64 \ - (S390_ARCH_FAC_LIST_SIZE_BYTE / sizeof(u64)) -#define S390_ARCH_FAC_MASK_SIZE_BYTE S390_ARCH_FAC_LIST_SIZE_BYTE -#define S390_ARCH_FAC_MASK_SIZE_U64 \ - (S390_ARCH_FAC_MASK_SIZE_BYTE / sizeof(u64)) - -struct kvm_s390_cpu_model { - /* facility mask supported by kvm & hosting machine */ - __u64 fac_mask[S390_ARCH_FAC_MASK_SIZE_U64]; - struct kvm_s390_vm_cpu_subfunc subfuncs; - /* facility list requested by guest (in dma page) */ - __u64 *fac_list; - u64 cpuid; - unsigned short ibc; - /* subset of available UV-features for pv-guests enabled by user space */ - struct kvm_s390_vm_cpu_uv_feat uv_feat_guest; -}; - -#define S390_ARCH_FAC_FORMAT_2 2 -struct kvm_s390_flcb2 { - union { - struct { - u8 reserved0[7]; - u8 length; - }; - u64 header_val; - }; - u64 facilities[S390_ARCH_FAC_LIST_SIZE_U64]; -}; - -typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); - -struct kvm_s390_crypto { - struct kvm_s390_crypto_cb *crycb; - struct rw_semaphore pqap_hook_rwsem; - crypto_hook *pqap_hook; - __u32 crycbd; - __u8 aes_kw; - __u8 dea_kw; - __u8 apie; -}; - -#define APCB0_MASK_SIZE 1 -struct kvm_s390_apcb0 { - __u64 apm[APCB0_MASK_SIZE]; /* 0x0000 */ - __u64 aqm[APCB0_MASK_SIZE]; /* 0x0008 */ - __u64 adm[APCB0_MASK_SIZE]; /* 0x0010 */ - __u64 reserved18; /* 0x0018 */ -}; - -#define APCB1_MASK_SIZE 4 -struct kvm_s390_apcb1 { - __u64 apm[APCB1_MASK_SIZE]; /* 0x0000 */ - __u64 aqm[APCB1_MASK_SIZE]; /* 0x0020 */ - __u64 adm[APCB1_MASK_SIZE]; /* 0x0040 */ - __u64 reserved60[4]; /* 0x0060 */ -}; - -struct kvm_s390_crypto_cb { - struct kvm_s390_apcb0 apcb0; /* 0x0000 */ - __u8 reserved20[0x0048 - 0x0020]; /* 0x0020 */ - __u8 dea_wrapping_key_mask[24]; /* 0x0048 */ - __u8 aes_wrapping_key_mask[32]; /* 0x0060 */ - struct kvm_s390_apcb1 apcb1; /* 0x0080 */ -}; - -struct kvm_s390_gisa { - union { - struct { /* common to all formats */ - u32 next_alert; - u8 ipm; - u8 reserved01[2]; - u8 iam; - }; - struct { /* format 0 */ - u32 next_alert; - u8 ipm; - u8 reserved01; - u8 : 6; - u8 g : 1; - u8 c : 1; - u8 iam; - u8 reserved02[4]; - u32 airq_count; - } g0; - struct { /* format 1 */ - u32 next_alert; - u8 ipm; - u8 simm; - u8 nimm; - u8 iam; - u8 aism[8]; - u8 : 6; - u8 g : 1; - u8 c : 1; - u8 reserved03[11]; - u32 airq_count; - } g1; - struct { - u64 word[4]; - } u64; - }; -}; - -struct kvm_s390_gib { - u32 alert_list_origin; - u32 reserved01; - u8:5; - u8 nisc:3; - u8 reserved03[3]; - u32 reserved04[5]; -}; - -/* - * sie_page2 has to be allocated as DMA because fac_list, crycb and - * gisa need 31bit addresses in the sie control block. - */ -struct sie_page2 { - __u64 fac_list[S390_ARCH_FAC_LIST_SIZE_U64]; /* 0x0000 */ - struct kvm_s390_crypto_cb crycb; /* 0x0800 */ - struct kvm_s390_gisa gisa; /* 0x0900 */ - struct kvm *kvm; /* 0x0920 */ - u8 reserved928[0x1000 - 0x928]; /* 0x0928 */ -}; - -struct vsie_page; - -struct kvm_s390_vsie { - struct mutex mutex; - struct radix_tree_root addr_to_page; - int page_count; - int next; - struct vsie_page *pages[KVM_MAX_VCPUS]; -}; - -struct kvm_s390_gisa_iam { - u8 mask; - spinlock_t ref_lock; - u32 ref_count[MAX_ISC + 1]; -}; - -struct kvm_s390_gisa_interrupt { - struct kvm_s390_gisa *origin; - struct kvm_s390_gisa_iam alert; - struct hrtimer timer; - u64 expires; - DECLARE_BITMAP(kicked_mask, KVM_MAX_VCPUS); -}; - -struct kvm_s390_pv { - u64 handle; - u64 guest_len; - unsigned long stor_base; - void *stor_var; - bool dumping; - void *set_aside; - struct list_head need_cleanup; - struct mmu_notifier mmu_notifier; - /* Protects against concurrent import-like operations */ - struct mutex import_lock; -}; - -struct kvm_s390_mmu_cache; - -struct kvm_arch { - struct esca_block *sca; - debug_info_t *dbf; - struct kvm_s390_float_interrupt float_int; - struct kvm_device *flic; - struct gmap *gmap; - unsigned long mem_limit; - int css_support; - int use_irqchip; - int use_cmma; - int use_pfmfi; - int use_skf; - int use_zpci_interp; - int user_cpu_state_ctrl; - int user_sigp; - int user_stsi; - int user_instr0; - int user_operexec; - int allow_vsie_esamode; - struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS]; - wait_queue_head_t ipte_wq; - int ipte_lock_count; - struct mutex ipte_mutex; - spinlock_t start_stop_lock; - struct sie_page2 *sie_page2; - struct kvm_s390_cpu_model model; - struct kvm_s390_crypto crypto; - struct kvm_s390_vsie vsie; - u8 epdx; - u64 epoch; - int migration_mode; - atomic64_t cmma_dirty_pages; - /* subset of available cpu features enabled by user space */ - DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); - /* indexed by vcpu_idx */ - DECLARE_BITMAP(idle_mask, KVM_MAX_VCPUS); - struct kvm_s390_gisa_interrupt gisa_int; - struct kvm_s390_pv pv; - struct list_head kzdev_list; - spinlock_t kzdev_list_lock; - struct kvm_s390_mmu_cache *mc; -}; - -#define KVM_HVA_ERR_BAD (-1UL) -#define KVM_HVA_ERR_RO_BAD (-2UL) - -static inline bool kvm_is_error_hva(unsigned long addr) -{ - return IS_ERR_VALUE(addr); -} - -#define ASYNC_PF_PER_VCPU 64 -struct kvm_arch_async_pf { - unsigned long pfault_token; -}; - -bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); - -void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, - struct kvm_async_pf *work); - -bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, - struct kvm_async_pf *work); - -void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, - struct kvm_async_pf *work); - -static inline void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) {} - -void kvm_arch_crypto_clear_masks(struct kvm *kvm); -void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm, - unsigned long *aqm, unsigned long *adm); - -#define SIE64_RETURN_NORMAL 0 -#define SIE64_RETURN_MCCK 1 - -int __sie64a(phys_addr_t sie_block_phys, struct kvm_s390_sie_block *sie_block, u64 *rsa, - unsigned long gasce); - -static inline int sie64a(struct kvm_s390_sie_block *sie_block, u64 *rsa, unsigned long gasce) -{ - return __sie64a(virt_to_phys(sie_block), sie_block, rsa, gasce); -} - -extern char sie_exit; - -bool kvm_s390_pv_is_protected(struct kvm *kvm); -bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu); - -extern int kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb, - u64 *gprs, unsigned long gasce); - -extern int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc); -extern int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc); - -bool kvm_s390_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa); - -static inline void kvm_arch_free_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) {} -static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} -static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} -static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) {} -static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} -static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} - -#define __KVM_HAVE_ARCH_VM_FREE -void kvm_arch_free_vm(struct kvm *kvm); - -struct zpci_kvm_hook { - int (*kvm_register)(void *opaque, struct kvm *kvm); - void (*kvm_unregister)(void *opaque); -}; - -extern struct zpci_kvm_hook zpci_kvm_hook; - -#endif +#endif /* ASM_KVM_HOST_H */ diff --git a/arch/s390/include/asm/kvm_host_s390.h b/arch/s390/include/asm/kvm_host_s390.h new file mode 100644 index 000000000000..0c4e061d7437 --- /dev/null +++ b/arch/s390/include/asm/kvm_host_s390.h @@ -0,0 +1,784 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * definition for kernel virtual machines on s390 + * + * Copyright IBM Corp. 2008, 2018 + * + * Author(s): Carsten Otte + */ + + +#ifndef ASM_KVM_HOST_S390_H +#define ASM_KVM_HOST_S390_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define KVM_HAVE_MMU_RWLOCK +#define KVM_MAX_VCPUS 255 + +#define KVM_INTERNAL_MEM_SLOTS 1 + +/* + * These seem to be used for allocating ->chip in the routing table, which we + * don't use. 1 is as small as we can get to reduce the needed memory. If we + * need to look at ->chip later on, we'll need to revisit this. + */ +#define KVM_NR_IRQCHIPS 1 +#define KVM_IRQCHIP_NUM_PINS 1 +#define KVM_HALT_POLL_NS_DEFAULT 50000 + +/* s390-specific vcpu->requests bit members */ +#define KVM_REQ_ENABLE_IBS KVM_ARCH_REQ(0) +#define KVM_REQ_DISABLE_IBS KVM_ARCH_REQ(1) +#define KVM_REQ_ICPT_OPEREXC KVM_ARCH_REQ(2) +#define KVM_REQ_START_MIGRATION KVM_ARCH_REQ(3) +#define KVM_REQ_STOP_MIGRATION KVM_ARCH_REQ(4) +#define KVM_REQ_VSIE_RESTART KVM_ARCH_REQ(5) +#define KVM_REQ_REFRESH_GUEST_PREFIX \ + KVM_ARCH_REQ_FLAGS(6, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) + +struct kvm_vcpu_stat { + struct kvm_vcpu_stat_generic generic; + u64 exit_userspace; + u64 exit_null; + u64 exit_external_request; + u64 exit_io_request; + u64 exit_external_interrupt; + u64 exit_stop_request; + u64 exit_validity; + u64 exit_instruction; + u64 exit_pei; + u64 halt_no_poll_steal; + u64 instruction_lctl; + u64 instruction_lctlg; + u64 instruction_stctl; + u64 instruction_stctg; + u64 exit_program_interruption; + u64 exit_instr_and_program; + u64 exit_operation_exception; + u64 deliver_ckc; + u64 deliver_cputm; + u64 deliver_external_call; + u64 deliver_emergency_signal; + u64 deliver_service_signal; + u64 deliver_virtio; + u64 deliver_stop_signal; + u64 deliver_prefix_signal; + u64 deliver_restart_signal; + u64 deliver_program; + u64 deliver_io; + u64 deliver_machine_check; + u64 exit_wait_state; + u64 inject_ckc; + u64 inject_cputm; + u64 inject_external_call; + u64 inject_emergency_signal; + u64 inject_mchk; + u64 inject_pfault_init; + u64 inject_program; + u64 inject_restart; + u64 inject_set_prefix; + u64 inject_stop_signal; + u64 instruction_epsw; + u64 instruction_gs; + u64 instruction_io_other; + u64 instruction_lpsw; + u64 instruction_lpswe; + u64 instruction_lpswey; + u64 instruction_pfmf; + u64 instruction_ptff; + u64 instruction_sck; + u64 instruction_sckpf; + u64 instruction_stidp; + u64 instruction_spx; + u64 instruction_stpx; + u64 instruction_stap; + u64 instruction_iske; + u64 instruction_ri; + u64 instruction_rrbe; + u64 instruction_sske; + u64 instruction_ipte_interlock; + u64 instruction_stsi; + u64 instruction_stfl; + u64 instruction_tb; + u64 instruction_tpi; + u64 instruction_tprot; + u64 instruction_tsch; + u64 instruction_sie; + u64 instruction_essa; + u64 instruction_sthyi; + u64 instruction_sigp_sense; + u64 instruction_sigp_sense_running; + u64 instruction_sigp_external_call; + u64 instruction_sigp_emergency; + u64 instruction_sigp_cond_emergency; + u64 instruction_sigp_start; + u64 instruction_sigp_stop; + u64 instruction_sigp_stop_store_status; + u64 instruction_sigp_store_status; + u64 instruction_sigp_store_adtl_status; + u64 instruction_sigp_arch; + u64 instruction_sigp_prefix; + u64 instruction_sigp_restart; + u64 instruction_sigp_init_cpu_reset; + u64 instruction_sigp_cpu_reset; + u64 instruction_sigp_unknown; + u64 instruction_diagnose_10; + u64 instruction_diagnose_44; + u64 instruction_diagnose_9c; + u64 diag_9c_ignored; + u64 diag_9c_forward; + u64 instruction_diagnose_258; + u64 instruction_diagnose_308; + u64 instruction_diagnose_500; + u64 instruction_diagnose_other; + u64 pfault_sync; + u64 signal_exits; +}; + +#define PGM_OPERATION 0x01 +#define PGM_PRIVILEGED_OP 0x02 +#define PGM_EXECUTE 0x03 +#define PGM_PROTECTION 0x04 +#define PGM_ADDRESSING 0x05 +#define PGM_SPECIFICATION 0x06 +#define PGM_DATA 0x07 +#define PGM_FIXED_POINT_OVERFLOW 0x08 +#define PGM_FIXED_POINT_DIVIDE 0x09 +#define PGM_DECIMAL_OVERFLOW 0x0a +#define PGM_DECIMAL_DIVIDE 0x0b +#define PGM_HFP_EXPONENT_OVERFLOW 0x0c +#define PGM_HFP_EXPONENT_UNDERFLOW 0x0d +#define PGM_HFP_SIGNIFICANCE 0x0e +#define PGM_HFP_DIVIDE 0x0f +#define PGM_SEGMENT_TRANSLATION 0x10 +#define PGM_PAGE_TRANSLATION 0x11 +#define PGM_TRANSLATION_SPEC 0x12 +#define PGM_SPECIAL_OPERATION 0x13 +#define PGM_OPERAND 0x15 +#define PGM_TRACE_TABEL 0x16 +#define PGM_VECTOR_PROCESSING 0x1b +#define PGM_SPACE_SWITCH 0x1c +#define PGM_HFP_SQUARE_ROOT 0x1d +#define PGM_PC_TRANSLATION_SPEC 0x1f +#define PGM_AFX_TRANSLATION 0x20 +#define PGM_ASX_TRANSLATION 0x21 +#define PGM_LX_TRANSLATION 0x22 +#define PGM_EX_TRANSLATION 0x23 +#define PGM_PRIMARY_AUTHORITY 0x24 +#define PGM_SECONDARY_AUTHORITY 0x25 +#define PGM_LFX_TRANSLATION 0x26 +#define PGM_LSX_TRANSLATION 0x27 +#define PGM_ALET_SPECIFICATION 0x28 +#define PGM_ALEN_TRANSLATION 0x29 +#define PGM_ALE_SEQUENCE 0x2a +#define PGM_ASTE_VALIDITY 0x2b +#define PGM_ASTE_SEQUENCE 0x2c +#define PGM_EXTENDED_AUTHORITY 0x2d +#define PGM_LSTE_SEQUENCE 0x2e +#define PGM_ASTE_INSTANCE 0x2f +#define PGM_STACK_FULL 0x30 +#define PGM_STACK_EMPTY 0x31 +#define PGM_STACK_SPECIFICATION 0x32 +#define PGM_STACK_TYPE 0x33 +#define PGM_STACK_OPERATION 0x34 +#define PGM_ASCE_TYPE 0x38 +#define PGM_REGION_FIRST_TRANS 0x39 +#define PGM_REGION_SECOND_TRANS 0x3a +#define PGM_REGION_THIRD_TRANS 0x3b +#define PGM_SECURE_STORAGE_ACCESS 0x3d +#define PGM_NON_SECURE_STORAGE_ACCESS 0x3e +#define PGM_SECURE_STORAGE_VIOLATION 0x3f +#define PGM_MONITOR 0x40 +#define PGM_PER 0x80 +#define PGM_CRYPTO_OPERATION 0x119 + +/* irq types in ascend order of priorities */ +enum irq_types { + IRQ_PEND_SET_PREFIX = 0, + IRQ_PEND_RESTART, + IRQ_PEND_SIGP_STOP, + IRQ_PEND_IO_ISC_7, + IRQ_PEND_IO_ISC_6, + IRQ_PEND_IO_ISC_5, + IRQ_PEND_IO_ISC_4, + IRQ_PEND_IO_ISC_3, + IRQ_PEND_IO_ISC_2, + IRQ_PEND_IO_ISC_1, + IRQ_PEND_IO_ISC_0, + IRQ_PEND_VIRTIO, + IRQ_PEND_PFAULT_DONE, + IRQ_PEND_PFAULT_INIT, + IRQ_PEND_EXT_HOST, + IRQ_PEND_EXT_SERVICE, + IRQ_PEND_EXT_SERVICE_EV, + IRQ_PEND_EXT_TIMING, + IRQ_PEND_EXT_CPU_TIMER, + IRQ_PEND_EXT_CLOCK_COMP, + IRQ_PEND_EXT_EXTERNAL, + IRQ_PEND_EXT_EMERGENCY, + IRQ_PEND_EXT_MALFUNC, + IRQ_PEND_EXT_IRQ_KEY, + IRQ_PEND_MCHK_REP, + IRQ_PEND_PROG, + IRQ_PEND_SVC, + IRQ_PEND_MCHK_EX, + IRQ_PEND_COUNT +}; + +/* We have 2M for virtio device descriptor pages. Smallest amount of + * memory per page is 24 bytes (1 queue), so (2048*1024) / 24 = 87381 + */ +#define KVM_S390_MAX_VIRTIO_IRQS 87381 + +/* + * Repressible (non-floating) machine check interrupts + * subclass bits in MCIC + */ +#define MCHK_EXTD_BIT 58 +#define MCHK_DEGR_BIT 56 +#define MCHK_WARN_BIT 55 +#define MCHK_REP_MASK ((1UL << MCHK_DEGR_BIT) | \ + (1UL << MCHK_EXTD_BIT) | \ + (1UL << MCHK_WARN_BIT)) + +/* Exigent machine check interrupts subclass bits in MCIC */ +#define MCHK_SD_BIT 63 +#define MCHK_PD_BIT 62 +#define MCHK_EX_MASK ((1UL << MCHK_SD_BIT) | (1UL << MCHK_PD_BIT)) + +#define IRQ_PEND_EXT_MASK ((1UL << IRQ_PEND_EXT_IRQ_KEY) | \ + (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ + (1UL << IRQ_PEND_EXT_CPU_TIMER) | \ + (1UL << IRQ_PEND_EXT_MALFUNC) | \ + (1UL << IRQ_PEND_EXT_EMERGENCY) | \ + (1UL << IRQ_PEND_EXT_EXTERNAL) | \ + (1UL << IRQ_PEND_EXT_TIMING) | \ + (1UL << IRQ_PEND_EXT_HOST) | \ + (1UL << IRQ_PEND_EXT_SERVICE) | \ + (1UL << IRQ_PEND_EXT_SERVICE_EV) | \ + (1UL << IRQ_PEND_VIRTIO) | \ + (1UL << IRQ_PEND_PFAULT_INIT) | \ + (1UL << IRQ_PEND_PFAULT_DONE)) + +#define IRQ_PEND_IO_MASK ((1UL << IRQ_PEND_IO_ISC_0) | \ + (1UL << IRQ_PEND_IO_ISC_1) | \ + (1UL << IRQ_PEND_IO_ISC_2) | \ + (1UL << IRQ_PEND_IO_ISC_3) | \ + (1UL << IRQ_PEND_IO_ISC_4) | \ + (1UL << IRQ_PEND_IO_ISC_5) | \ + (1UL << IRQ_PEND_IO_ISC_6) | \ + (1UL << IRQ_PEND_IO_ISC_7)) + +#define IRQ_PEND_MCHK_MASK ((1UL << IRQ_PEND_MCHK_REP) | \ + (1UL << IRQ_PEND_MCHK_EX)) + +#define IRQ_PEND_EXT_II_MASK ((1UL << IRQ_PEND_EXT_CPU_TIMER) | \ + (1UL << IRQ_PEND_EXT_CLOCK_COMP) | \ + (1UL << IRQ_PEND_EXT_EMERGENCY) | \ + (1UL << IRQ_PEND_EXT_EXTERNAL) | \ + (1UL << IRQ_PEND_EXT_SERVICE) | \ + (1UL << IRQ_PEND_EXT_SERVICE_EV)) + +struct kvm_s390_interrupt_info { + struct list_head list; + u64 type; + union { + struct kvm_s390_io_info io; + struct kvm_s390_ext_info ext; + struct kvm_s390_pgm_info pgm; + struct kvm_s390_emerg_info emerg; + struct kvm_s390_extcall_info extcall; + struct kvm_s390_prefix_info prefix; + struct kvm_s390_stop_info stop; + struct kvm_s390_mchk_info mchk; + }; +}; + +struct kvm_s390_irq_payload { + struct kvm_s390_io_info io; + struct kvm_s390_ext_info ext; + struct kvm_s390_pgm_info pgm; + struct kvm_s390_emerg_info emerg; + struct kvm_s390_extcall_info extcall; + struct kvm_s390_prefix_info prefix; + struct kvm_s390_stop_info stop; + struct kvm_s390_mchk_info mchk; +}; + +struct kvm_s390_local_interrupt { + spinlock_t lock; + DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS); + struct kvm_s390_irq_payload irq; + unsigned long pending_irqs; +}; + +#define FIRQ_LIST_IO_ISC_0 0 +#define FIRQ_LIST_IO_ISC_1 1 +#define FIRQ_LIST_IO_ISC_2 2 +#define FIRQ_LIST_IO_ISC_3 3 +#define FIRQ_LIST_IO_ISC_4 4 +#define FIRQ_LIST_IO_ISC_5 5 +#define FIRQ_LIST_IO_ISC_6 6 +#define FIRQ_LIST_IO_ISC_7 7 +#define FIRQ_LIST_PFAULT 8 +#define FIRQ_LIST_VIRTIO 9 +#define FIRQ_LIST_COUNT 10 +#define FIRQ_CNTR_IO 0 +#define FIRQ_CNTR_SERVICE 1 +#define FIRQ_CNTR_VIRTIO 2 +#define FIRQ_CNTR_PFAULT 3 +#define FIRQ_MAX_COUNT 4 + +/* mask the AIS mode for a given ISC */ +#define AIS_MODE_MASK(isc) (0x80 >> isc) + +#define KVM_S390_AIS_MODE_ALL 0 +#define KVM_S390_AIS_MODE_SINGLE 1 + +struct kvm_s390_float_interrupt { + unsigned long pending_irqs; + unsigned long masked_irqs; + spinlock_t lock; + struct list_head lists[FIRQ_LIST_COUNT]; + int counters[FIRQ_MAX_COUNT]; + struct kvm_s390_mchk_info mchk; + struct kvm_s390_ext_info srv_signal; + int last_sleep_cpu; + spinlock_t ais_lock; + u8 simm; + u8 nimm; +}; + +struct kvm_hw_wp_info_arch { + unsigned long addr; + unsigned long phys_addr; + int len; + char *old_data; +}; + +struct kvm_hw_bp_info_arch { + unsigned long addr; + int len; +}; + +/* + * Only the upper 16 bits of kvm_guest_debug->control are arch specific. + * Further KVM_GUESTDBG flags which an be used from userspace can be found in + * arch/s390/include/uapi/asm/kvm.h + */ +#define KVM_GUESTDBG_EXIT_PENDING 0x10000000 + +#define guestdbg_enabled(vcpu) \ + (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) +#define guestdbg_sstep_enabled(vcpu) \ + (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) +#define guestdbg_hw_bp_enabled(vcpu) \ + (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) +#define guestdbg_exit_pending(vcpu) (guestdbg_enabled(vcpu) && \ + (vcpu->guest_debug & KVM_GUESTDBG_EXIT_PENDING)) + +#define KVM_GUESTDBG_VALID_MASK \ + (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP |\ + KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_EXIT_PENDING) + +struct kvm_guestdbg_info_arch { + unsigned long cr0; + unsigned long cr9; + unsigned long cr10; + unsigned long cr11; + struct kvm_hw_bp_info_arch *hw_bp_info; + struct kvm_hw_wp_info_arch *hw_wp_info; + int nr_hw_bp; + int nr_hw_wp; + unsigned long last_bp; +}; + +struct kvm_s390_pv_vcpu { + u64 handle; + unsigned long stor_base; +}; + +struct kvm_vcpu_arch { + struct kvm_s390_sie_block *sie_block; + /* if vsie is active, currently executed shadow sie control block */ + struct kvm_s390_sie_block *vsie_block; + unsigned int host_acrs[NUM_ACRS]; + struct gs_cb *host_gscb; + struct kvm_s390_local_interrupt local_int; + struct hrtimer ckc_timer; + struct kvm_s390_pgm_info pgm; + struct gmap *gmap; + struct kvm_guestdbg_info_arch guestdbg; + unsigned long pfault_token; + unsigned long pfault_select; + unsigned long pfault_compare; + bool cputm_enabled; + /* + * The seqcount protects updates to cputm_start and sie_block.cputm, + * this way we can have non-blocking reads with consistent values. + * Only the owning VCPU thread (vcpu->cpu) is allowed to change these + * values and to start/stop/enable/disable cpu timer accounting. + */ + seqcount_t cputm_seqcount; + __u64 cputm_start; + bool gs_enabled; + bool skey_enabled; + /* Indicator if the access registers have been loaded from guest */ + bool acrs_loaded; + bool initialized; + struct kvm_s390_pv_vcpu pv; + union diag318_info diag318_info; + struct kvm_s390_mmu_cache *mc; +}; + +struct kvm_vm_stat { + struct kvm_vm_stat_generic generic; + u64 inject_io; + u64 io_390_adapter_map; + u64 io_390_adapter_unmap; + u64 io_390_inatomic; + u64 io_flic_inject_airq; + u64 io_set_adapter_int; + u64 io_390_inatomic_no_inject; + u64 inject_float_mchk; + u64 inject_pfault_done; + u64 inject_service_signal; + u64 inject_virtio; + u64 aen_forward; + u64 gmap_shadow_create; + u64 gmap_shadow_reuse; + u64 gmap_shadow_r1_entry; + u64 gmap_shadow_r2_entry; + u64 gmap_shadow_r3_entry; + u64 gmap_shadow_sg_entry; + u64 gmap_shadow_pg_entry; +}; + +struct kvm_arch_memory_slot { +}; + +struct s390_map_info { + struct list_head list; + __u64 guest_addr; + __u64 addr; + struct page *page; + /* + * True if the page is long-term pinned. False if long-term pinning + * failed and this entry exists only to preserve MAP/UNMAP symmetry. + */ + bool pinned; +}; + +struct s390_io_adapter { + unsigned int id; + int isc; + bool maskable; + bool masked; + bool swap; + bool suppressible; + spinlock_t maps_lock; + struct list_head maps; + unsigned int nr_maps; +}; + +#define MAX_S390_IO_ADAPTERS ((MAX_ISC + 1) * 8) +#define MAX_S390_ADAPTER_MAPS 256 + +/* maximum size of facilities and facility mask is 2k bytes */ +#define S390_ARCH_FAC_LIST_SIZE_BYTE (1<<11) +#define S390_ARCH_FAC_LIST_SIZE_U64 \ + (S390_ARCH_FAC_LIST_SIZE_BYTE / sizeof(u64)) +#define S390_ARCH_FAC_MASK_SIZE_BYTE S390_ARCH_FAC_LIST_SIZE_BYTE +#define S390_ARCH_FAC_MASK_SIZE_U64 \ + (S390_ARCH_FAC_MASK_SIZE_BYTE / sizeof(u64)) + +struct kvm_s390_cpu_model { + /* facility mask supported by kvm & hosting machine */ + __u64 fac_mask[S390_ARCH_FAC_MASK_SIZE_U64]; + struct kvm_s390_vm_cpu_subfunc subfuncs; + /* facility list requested by guest (in dma page) */ + __u64 *fac_list; + u64 cpuid; + unsigned short ibc; + /* subset of available UV-features for pv-guests enabled by user space */ + struct kvm_s390_vm_cpu_uv_feat uv_feat_guest; +}; + +#define S390_ARCH_FAC_FORMAT_2 2 +struct kvm_s390_flcb2 { + union { + struct { + u8 reserved0[7]; + u8 length; + }; + u64 header_val; + }; + u64 facilities[S390_ARCH_FAC_LIST_SIZE_U64]; +}; + +typedef int (*crypto_hook)(struct kvm_vcpu *vcpu); + +struct kvm_s390_crypto { + struct kvm_s390_crypto_cb *crycb; + struct rw_semaphore pqap_hook_rwsem; + crypto_hook *pqap_hook; + __u32 crycbd; + __u8 aes_kw; + __u8 dea_kw; + __u8 apie; +}; + +#define APCB0_MASK_SIZE 1 +struct kvm_s390_apcb0 { + __u64 apm[APCB0_MASK_SIZE]; /* 0x0000 */ + __u64 aqm[APCB0_MASK_SIZE]; /* 0x0008 */ + __u64 adm[APCB0_MASK_SIZE]; /* 0x0010 */ + __u64 reserved18; /* 0x0018 */ +}; + +#define APCB1_MASK_SIZE 4 +struct kvm_s390_apcb1 { + __u64 apm[APCB1_MASK_SIZE]; /* 0x0000 */ + __u64 aqm[APCB1_MASK_SIZE]; /* 0x0020 */ + __u64 adm[APCB1_MASK_SIZE]; /* 0x0040 */ + __u64 reserved60[4]; /* 0x0060 */ +}; + +struct kvm_s390_crypto_cb { + struct kvm_s390_apcb0 apcb0; /* 0x0000 */ + __u8 reserved20[0x0048 - 0x0020]; /* 0x0020 */ + __u8 dea_wrapping_key_mask[24]; /* 0x0048 */ + __u8 aes_wrapping_key_mask[32]; /* 0x0060 */ + struct kvm_s390_apcb1 apcb1; /* 0x0080 */ +}; + +struct kvm_s390_gisa { + union { + struct { /* common to all formats */ + u32 next_alert; + u8 ipm; + u8 reserved01[2]; + u8 iam; + }; + struct { /* format 0 */ + u32 next_alert; + u8 ipm; + u8 reserved01; + u8 : 6; + u8 g : 1; + u8 c : 1; + u8 iam; + u8 reserved02[4]; + u32 airq_count; + } g0; + struct { /* format 1 */ + u32 next_alert; + u8 ipm; + u8 simm; + u8 nimm; + u8 iam; + u8 aism[8]; + u8 : 6; + u8 g : 1; + u8 c : 1; + u8 reserved03[11]; + u32 airq_count; + } g1; + struct { + u64 word[4]; + } u64; + }; +}; + +struct kvm_s390_gib { + u32 alert_list_origin; + u32 reserved01; + u8:5; + u8 nisc:3; + u8 reserved03[3]; + u32 reserved04[5]; +}; + +/* + * sie_page2 has to be allocated as DMA because fac_list, crycb and + * gisa need 31bit addresses in the sie control block. + */ +struct sie_page2 { + __u64 fac_list[S390_ARCH_FAC_LIST_SIZE_U64]; /* 0x0000 */ + struct kvm_s390_crypto_cb crycb; /* 0x0800 */ + struct kvm_s390_gisa gisa; /* 0x0900 */ + struct kvm *kvm; /* 0x0920 */ + u8 reserved928[0x1000 - 0x928]; /* 0x0928 */ +}; + +struct vsie_page; + +struct kvm_s390_vsie { + struct mutex mutex; + struct radix_tree_root addr_to_page; + int page_count; + int next; + struct vsie_page *pages[KVM_MAX_VCPUS]; +}; + +struct kvm_s390_gisa_iam { + u8 mask; + spinlock_t ref_lock; + u32 ref_count[MAX_ISC + 1]; +}; + +struct kvm_s390_gisa_interrupt { + struct kvm_s390_gisa *origin; + struct kvm_s390_gisa_iam alert; + struct hrtimer timer; + u64 expires; + DECLARE_BITMAP(kicked_mask, KVM_MAX_VCPUS); +}; + +struct kvm_s390_pv { + u64 handle; + u64 guest_len; + unsigned long stor_base; + void *stor_var; + bool dumping; + void *set_aside; + struct list_head need_cleanup; + struct mmu_notifier mmu_notifier; + /* Protects against concurrent import-like operations */ + struct mutex import_lock; +}; + +struct kvm_s390_mmu_cache; + +struct kvm_arch { + struct esca_block *sca; + debug_info_t *dbf; + struct kvm_s390_float_interrupt float_int; + struct kvm_device *flic; + struct gmap *gmap; + unsigned long mem_limit; + int css_support; + int use_irqchip; + int use_cmma; + int use_pfmfi; + int use_skf; + int use_zpci_interp; + int user_cpu_state_ctrl; + int user_sigp; + int user_stsi; + int user_instr0; + int user_operexec; + int allow_vsie_esamode; + struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS]; + wait_queue_head_t ipte_wq; + int ipte_lock_count; + struct mutex ipte_mutex; + spinlock_t start_stop_lock; + struct sie_page2 *sie_page2; + struct kvm_s390_cpu_model model; + struct kvm_s390_crypto crypto; + struct kvm_s390_vsie vsie; + u8 epdx; + u64 epoch; + int migration_mode; + atomic64_t cmma_dirty_pages; + /* subset of available cpu features enabled by user space */ + DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); + /* indexed by vcpu_idx */ + DECLARE_BITMAP(idle_mask, KVM_MAX_VCPUS); + struct kvm_s390_gisa_interrupt gisa_int; + struct kvm_s390_pv pv; + struct list_head kzdev_list; + spinlock_t kzdev_list_lock; + struct kvm_s390_mmu_cache *mc; +}; + +#define KVM_HVA_ERR_BAD (-1UL) +#define KVM_HVA_ERR_RO_BAD (-2UL) + +static inline bool kvm_is_error_hva(unsigned long addr) +{ + return IS_ERR_VALUE(addr); +} + +#define ASYNC_PF_PER_VCPU 64 +struct kvm_arch_async_pf { + unsigned long pfault_token; +}; + +bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); + +void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); + +bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); + +void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work); + +static inline void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) {} + +void kvm_arch_crypto_clear_masks(struct kvm *kvm); +void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm, + unsigned long *aqm, unsigned long *adm); + +#define SIE64_RETURN_NORMAL 0 +#define SIE64_RETURN_MCCK 1 + +int __sie64a(phys_addr_t sie_block_phys, struct kvm_s390_sie_block *sie_block, u64 *rsa, + unsigned long gasce); + +static inline int sie64a(struct kvm_s390_sie_block *sie_block, u64 *rsa, unsigned long gasce) +{ + return __sie64a(virt_to_phys(sie_block), sie_block, rsa, gasce); +} + +extern char sie_exit; + +bool kvm_s390_pv_is_protected(struct kvm *kvm); +bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu); + +extern int kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb, + u64 *gprs, unsigned long gasce); + +extern int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc); +extern int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc); + +bool kvm_s390_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa); + +static inline void kvm_arch_free_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} +static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} +static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) {} +static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} + +#define __KVM_HAVE_ARCH_VM_FREE +void kvm_arch_free_vm(struct kvm *kvm); + +struct zpci_kvm_hook { + int (*kvm_register)(void *opaque, struct kvm *kvm); + void (*kvm_unregister)(void *opaque); +}; + +extern struct zpci_kvm_hook zpci_kvm_hook; + +#endif /* ASM_KVM_HOST_S390_H */ diff --git a/arch/s390/include/asm/kvm_host_s390_types.h b/arch/s390/include/asm/kvm_host_s390_types.h new file mode 100644 index 000000000000..9e348a530421 --- /dev/null +++ b/arch/s390/include/asm/kvm_host_s390_types.h @@ -0,0 +1,349 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_KVM_HOST_S390_TYPES_H +#define _ASM_KVM_HOST_S390_TYPES_H + +#include +#include + +#define KVM_S390_BSCA_CPU_SLOTS 64 +#define KVM_S390_ESCA_CPU_SLOTS 248 + +#define SCB_ALIGNMENT_SHIFT 9 + +#define SIGP_CTRL_C 0x80 +#define SIGP_CTRL_SCN_MASK 0x3f + +union bsca_sigp_ctrl { + __u8 value; + struct { + __u8 c : 1; + __u8 r : 1; + __u8 scn : 6; + }; +}; + +union esca_sigp_ctrl { + __u16 value; + struct { + __u8 c : 1; + __u8 reserved: 7; + __u8 scn; + }; +}; + +struct esca_entry { + union esca_sigp_ctrl sigp_ctrl; + __u16 reserved1[3]; + __u64 sda; + __u64 reserved2[6]; +}; + +struct bsca_entry { + __u8 reserved0; + union bsca_sigp_ctrl sigp_ctrl; + __u16 reserved[3]; + __u64 sda; + __u64 reserved2[2]; +}; + +union ipte_control { + unsigned long val; + struct { + unsigned long k : 1; + unsigned long kh : 31; + unsigned long kg : 32; + }; +}; + +/* + * Utility is defined as two bytes but having it four bytes wide + * generates more efficient code. Since the following bytes are + * reserved this makes no functional difference. + */ +union sca_utility { + __u32 val; + struct { + __u32 mtcr : 1; + __u32 : 31; + }; +}; + +struct bsca_block { + union ipte_control ipte_control; + __u64 reserved[5]; + __u64 mcn; + union sca_utility utility; + __u8 reserved2[4]; + struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS]; +}; + +struct esca_block { + union ipte_control ipte_control; + __u64 reserved1[6]; + union sca_utility utility; + __u8 reserved2[4]; + __u64 mcn[4]; + __u64 reserved3[20]; + struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS]; +}; + +/* + * This struct is used to store some machine check info from lowcore + * for machine checks that happen while the guest is running. + * This info in host's lowcore might be overwritten by a second machine + * check from host when host is in the machine check's high-level handling. + * The size is 24 bytes. + */ +struct mcck_volatile_info { + __u64 mcic; + __u64 failing_storage_address; + __u32 ext_damage_code; + __u32 reserved; +}; + +#define CR0_INITIAL_MASK (CR0_UNUSED_56 | CR0_INTERRUPT_KEY_SUBMASK | \ + CR0_MEASUREMENT_ALERT_SUBMASK) +#define CR14_INITIAL_MASK (CR14_UNUSED_32 | CR14_UNUSED_33 | \ + CR14_EXTERNAL_DAMAGE_SUBMASK) + +#define SIDAD_SIZE_MASK 0xff +#define sida_addr(sie_block) phys_to_virt((sie_block)->sidad & PAGE_MASK) +#define sida_size(sie_block) \ + ((((sie_block)->sidad & SIDAD_SIZE_MASK) + 1) * PAGE_SIZE) + +#define CPUSTAT_STOPPED 0x80000000 +#define CPUSTAT_WAIT 0x10000000 +#define CPUSTAT_ECALL_PEND 0x08000000 +#define CPUSTAT_STOP_INT 0x04000000 +#define CPUSTAT_IO_INT 0x02000000 +#define CPUSTAT_EXT_INT 0x01000000 +#define CPUSTAT_RUNNING 0x00800000 +#define CPUSTAT_RETAINED 0x00400000 +#define CPUSTAT_TIMING_SUB 0x00020000 +#define CPUSTAT_SIE_SUB 0x00010000 +#define CPUSTAT_RRF 0x00008000 +#define CPUSTAT_SLSV 0x00004000 +#define CPUSTAT_SLSR 0x00002000 +#define CPUSTAT_ZARCH 0x00000800 +#define CPUSTAT_MCDS 0x00000100 +#define CPUSTAT_KSS 0x00000200 +#define CPUSTAT_SM 0x00000080 +#define CPUSTAT_IBS 0x00000040 +#define CPUSTAT_GED2 0x00000010 +#define CPUSTAT_G 0x00000008 +#define CPUSTAT_GED 0x00000004 +#define CPUSTAT_J 0x00000002 +#define CPUSTAT_P 0x00000001 + +struct kvm_s390_sie_block { + atomic_t cpuflags; /* 0x0000 */ + __u32 : 1; /* 0x0004 */ + __u32 prefix : 19; + __u32 ibc : 12; + __u8 reserved08[4]; /* 0x0008 */ +#define PROG_IN_SIE (1<<0) + __u32 prog0c; /* 0x000c */ + union { + __u8 reserved10[16]; /* 0x0010 */ + struct { + __u64 pv_handle_cpu; + __u64 pv_handle_config; + }; + }; +#define PROG_BLOCK_SIE (1<<0) +#define PROG_REQUEST (1<<1) + atomic_t prog20; /* 0x0020 */ + __u8 reserved24[4]; /* 0x0024 */ + __u64 cputm; /* 0x0028 */ + __u64 ckc; /* 0x0030 */ + __u64 epoch; /* 0x0038 */ + __u32 svcc; /* 0x0040 */ +#define LCTL_CR0 0x8000 +#define LCTL_CR6 0x0200 +#define LCTL_CR9 0x0040 +#define LCTL_CR10 0x0020 +#define LCTL_CR11 0x0010 +#define LCTL_CR14 0x0002 + __u16 lctl; /* 0x0044 */ + __s16 icpua; /* 0x0046 */ +#define ICTL_OPEREXC 0x80000000 +#define ICTL_PINT 0x20000000 +#define ICTL_LPSW 0x00400000 +#define ICTL_STCTL 0x00040000 +#define ICTL_ISKE 0x00004000 +#define ICTL_SSKE 0x00002000 +#define ICTL_RRBE 0x00001000 +#define ICTL_TPROT 0x00000200 + __u32 ictl; /* 0x0048 */ +#define ECA_CEI 0x80000000 +#define ECA_IB 0x40000000 +#define ECA_SIGPI 0x10000000 +#define ECA_MVPGI 0x01000000 +#define ECA_AIV 0x00200000 +#define ECA_VX 0x00020000 +#define ECA_PROTEXCI 0x00002000 +#define ECA_APIE 0x00000008 +#define ECA_SII 0x00000001 + __u32 eca; /* 0x004c */ +#define ICPT_INST 0x04 +#define ICPT_PROGI 0x08 +#define ICPT_INSTPROGI 0x0C +#define ICPT_EXTREQ 0x10 +#define ICPT_EXTINT 0x14 +#define ICPT_IOREQ 0x18 +#define ICPT_WAIT 0x1c +#define ICPT_VALIDITY 0x20 +#define ICPT_STOP 0x28 +#define ICPT_OPEREXC 0x2C +#define ICPT_PARTEXEC 0x38 +#define ICPT_IOINST 0x40 +#define ICPT_KSS 0x5c +#define ICPT_MCHKREQ 0x60 +#define ICPT_INT_ENABLE 0x64 +#define ICPT_PV_INSTR 0x68 +#define ICPT_PV_NOTIFY 0x6c +#define ICPT_PV_PREF 0x70 + __u8 icptcode; /* 0x0050 */ + __u8 icptstatus; /* 0x0051 */ + __u16 ihcpu; /* 0x0052 */ + __u8 reserved54; /* 0x0054 */ +#define IICTL_CODE_NONE 0x00 +#define IICTL_CODE_MCHK 0x01 +#define IICTL_CODE_EXT 0x02 +#define IICTL_CODE_IO 0x03 +#define IICTL_CODE_RESTART 0x04 +#define IICTL_CODE_SPECIFICATION 0x10 +#define IICTL_CODE_OPERAND 0x11 + __u8 iictl; /* 0x0055 */ + __u16 ipa; /* 0x0056 */ + __u32 ipb; /* 0x0058 */ + __u32 scaoh; /* 0x005c */ +#define FPF_BPBC 0x20 + __u8 fpf; /* 0x0060 */ +#define ECB_GS 0x40 +#define ECB_TE 0x10 +#define ECB_SPECI 0x08 +#define ECB_SRSI 0x04 +#define ECB_HOSTPROTINT 0x02 +#define ECB_PTF 0x01 + __u8 ecb; /* 0x0061 */ +#define ECB2_CMMA 0x80 +#define ECB2_IEP 0x20 +#define ECB2_PFMFI 0x08 +#define ECB2_ESCA 0x04 +#define ECB2_ZPCI_LSI 0x02 + __u8 ecb2; /* 0x0062 */ +#define ECB3_AISI 0x20 +#define ECB3_AISII 0x10 +#define ECB3_DEA 0x08 +#define ECB3_AES 0x04 +#define ECB3_RI 0x01 + __u8 ecb3; /* 0x0063 */ +#define ESCA_SCAOL_MASK ~0x3fU + __u32 scaol; /* 0x0064 */ + __u8 sdf; /* 0x0068 */ + __u8 epdx; /* 0x0069 */ + __u8 cpnc; /* 0x006a */ + __u8 reserved6b; /* 0x006b */ + __u32 todpr; /* 0x006c */ +#define GISA_FORMAT1 0x00000001 + __u32 gd; /* 0x0070 */ + __u8 reserved74[12]; /* 0x0074 */ + __u64 mso; /* 0x0080 */ + __u64 msl; /* 0x0088 */ + psw_t gpsw; /* 0x0090 */ + __u64 gg14; /* 0x00a0 */ + __u64 gg15; /* 0x00a8 */ + __u8 reservedb0[8]; /* 0x00b0 */ +#define HPID_KVM 0x4 +#define HPID_VSIE 0x5 + __u8 hpid; /* 0x00b8 */ + __u8 reservedb9[7]; /* 0x00b9 */ + union { + struct { + __u32 eiparams; /* 0x00c0 */ + __u16 extcpuaddr; /* 0x00c4 */ + __u16 eic; /* 0x00c6 */ + }; + __u64 mcic; /* 0x00c0 */ + } __packed; + __u32 reservedc8; /* 0x00c8 */ + union { + struct { + __u16 pgmilc; /* 0x00cc */ + __u16 iprcc; /* 0x00ce */ + }; + __u32 edc; /* 0x00cc */ + } __packed; + union { + struct { + __u32 dxc; /* 0x00d0 */ + __u16 mcn; /* 0x00d4 */ + __u8 perc; /* 0x00d6 */ + __u8 peratmid; /* 0x00d7 */ + }; + __u64 faddr; /* 0x00d0 */ + } __packed; + __u64 peraddr; /* 0x00d8 */ + __u8 eai; /* 0x00e0 */ + __u8 peraid; /* 0x00e1 */ + __u8 oai; /* 0x00e2 */ + __u8 armid; /* 0x00e3 */ + __u8 reservede4[4]; /* 0x00e4 */ + union { + __u64 tecmc; /* 0x00e8 */ + struct { + __u16 subchannel_id; /* 0x00e8 */ + __u16 subchannel_nr; /* 0x00ea */ + __u32 io_int_parm; /* 0x00ec */ + __u32 io_int_word; /* 0x00f0 */ + }; + } __packed; + __u8 reservedf4[8]; /* 0x00f4 */ +#define CRYCB_FORMAT_MASK 0x00000003 +#define CRYCB_FORMAT0 0x00000000 +#define CRYCB_FORMAT1 0x00000001 +#define CRYCB_FORMAT2 0x00000003 + __u32 crycbd; /* 0x00fc */ + __u64 gcr[16]; /* 0x0100 */ + union { + __u64 gbea; /* 0x0180 */ + __u64 sidad; + }; + __u8 reserved188[8]; /* 0x0188 */ + __u64 sdnxo; /* 0x0190 */ + __u8 reserved198[8]; /* 0x0198 */ + __u32 fac; /* 0x01a0 */ + __u8 reserved1a4[20]; /* 0x01a4 */ + __u64 cbrlo; /* 0x01b8 */ + __u8 reserved1c0[8]; /* 0x01c0 */ +#define ECD_HOSTREGMGMT 0x20000000 +#define ECD_MEF 0x08000000 +#define ECD_ETOKENF 0x02000000 +#define ECD_ECC 0x00200000 +#define ECD_HMAC 0x00004000 + __u32 ecd; /* 0x01c8 */ + __u8 reserved1cc[18]; /* 0x01cc */ + __u64 pp; /* 0x01de */ + __u8 reserved1e6[2]; /* 0x01e6 */ + __u64 itdba; /* 0x01e8 */ + __u64 riccbd; /* 0x01f0 */ + __u64 gvrd; /* 0x01f8 */ +} __packed __aligned(512); + +struct kvm_s390_itdb { + __u8 data[256]; +}; + +struct sie_page { + struct kvm_s390_sie_block sie_block; + struct mcck_volatile_info mcck_info; /* 0x0200 */ + __u8 reserved218[360]; /* 0x0218 */ + __u64 pv_grregs[16]; /* 0x0380 */ + __u8 reserved400[512]; /* 0x0400 */ + struct kvm_s390_itdb itdb; /* 0x0600 */ + __u8 reserved700[2304]; /* 0x0700 */ +}; + +#endif /* _ASM_KVM_HOST_S390_TYPES_H */ diff --git a/arch/s390/include/asm/kvm_host_types.h b/arch/s390/include/asm/kvm_host_types.h index ac82dd09fce5..e5bdba07cab0 100644 --- a/arch/s390/include/asm/kvm_host_types.h +++ b/arch/s390/include/asm/kvm_host_types.h @@ -1,349 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_KVM_HOST_TYPES_H -#define _ASM_KVM_HOST_TYPES_H +#ifndef ASM_KVM_HOST_TYPES_H +#define ASM_KVM_HOST_TYPES_H -#include -#include +#include -#define KVM_S390_BSCA_CPU_SLOTS 64 -#define KVM_S390_ESCA_CPU_SLOTS 248 - -#define SCB_ALIGNMENT_SHIFT 9 - -#define SIGP_CTRL_C 0x80 -#define SIGP_CTRL_SCN_MASK 0x3f - -union bsca_sigp_ctrl { - __u8 value; - struct { - __u8 c : 1; - __u8 r : 1; - __u8 scn : 6; - }; -}; - -union esca_sigp_ctrl { - __u16 value; - struct { - __u8 c : 1; - __u8 reserved: 7; - __u8 scn; - }; -}; - -struct esca_entry { - union esca_sigp_ctrl sigp_ctrl; - __u16 reserved1[3]; - __u64 sda; - __u64 reserved2[6]; -}; - -struct bsca_entry { - __u8 reserved0; - union bsca_sigp_ctrl sigp_ctrl; - __u16 reserved[3]; - __u64 sda; - __u64 reserved2[2]; -}; - -union ipte_control { - unsigned long val; - struct { - unsigned long k : 1; - unsigned long kh : 31; - unsigned long kg : 32; - }; -}; - -/* - * Utility is defined as two bytes but having it four bytes wide - * generates more efficient code. Since the following bytes are - * reserved this makes no functional difference. - */ -union sca_utility { - __u32 val; - struct { - __u32 mtcr : 1; - __u32 : 31; - }; -}; - -struct bsca_block { - union ipte_control ipte_control; - __u64 reserved[5]; - __u64 mcn; - union sca_utility utility; - __u8 reserved2[4]; - struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS]; -}; - -struct esca_block { - union ipte_control ipte_control; - __u64 reserved1[6]; - union sca_utility utility; - __u8 reserved2[4]; - __u64 mcn[4]; - __u64 reserved3[20]; - struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS]; -}; - -/* - * This struct is used to store some machine check info from lowcore - * for machine checks that happen while the guest is running. - * This info in host's lowcore might be overwritten by a second machine - * check from host when host is in the machine check's high-level handling. - * The size is 24 bytes. - */ -struct mcck_volatile_info { - __u64 mcic; - __u64 failing_storage_address; - __u32 ext_damage_code; - __u32 reserved; -}; - -#define CR0_INITIAL_MASK (CR0_UNUSED_56 | CR0_INTERRUPT_KEY_SUBMASK | \ - CR0_MEASUREMENT_ALERT_SUBMASK) -#define CR14_INITIAL_MASK (CR14_UNUSED_32 | CR14_UNUSED_33 | \ - CR14_EXTERNAL_DAMAGE_SUBMASK) - -#define SIDAD_SIZE_MASK 0xff -#define sida_addr(sie_block) phys_to_virt((sie_block)->sidad & PAGE_MASK) -#define sida_size(sie_block) \ - ((((sie_block)->sidad & SIDAD_SIZE_MASK) + 1) * PAGE_SIZE) - -#define CPUSTAT_STOPPED 0x80000000 -#define CPUSTAT_WAIT 0x10000000 -#define CPUSTAT_ECALL_PEND 0x08000000 -#define CPUSTAT_STOP_INT 0x04000000 -#define CPUSTAT_IO_INT 0x02000000 -#define CPUSTAT_EXT_INT 0x01000000 -#define CPUSTAT_RUNNING 0x00800000 -#define CPUSTAT_RETAINED 0x00400000 -#define CPUSTAT_TIMING_SUB 0x00020000 -#define CPUSTAT_SIE_SUB 0x00010000 -#define CPUSTAT_RRF 0x00008000 -#define CPUSTAT_SLSV 0x00004000 -#define CPUSTAT_SLSR 0x00002000 -#define CPUSTAT_ZARCH 0x00000800 -#define CPUSTAT_MCDS 0x00000100 -#define CPUSTAT_KSS 0x00000200 -#define CPUSTAT_SM 0x00000080 -#define CPUSTAT_IBS 0x00000040 -#define CPUSTAT_GED2 0x00000010 -#define CPUSTAT_G 0x00000008 -#define CPUSTAT_GED 0x00000004 -#define CPUSTAT_J 0x00000002 -#define CPUSTAT_P 0x00000001 - -struct kvm_s390_sie_block { - atomic_t cpuflags; /* 0x0000 */ - __u32 : 1; /* 0x0004 */ - __u32 prefix : 19; - __u32 ibc : 12; - __u8 reserved08[4]; /* 0x0008 */ -#define PROG_IN_SIE (1<<0) - __u32 prog0c; /* 0x000c */ - union { - __u8 reserved10[16]; /* 0x0010 */ - struct { - __u64 pv_handle_cpu; - __u64 pv_handle_config; - }; - }; -#define PROG_BLOCK_SIE (1<<0) -#define PROG_REQUEST (1<<1) - atomic_t prog20; /* 0x0020 */ - __u8 reserved24[4]; /* 0x0024 */ - __u64 cputm; /* 0x0028 */ - __u64 ckc; /* 0x0030 */ - __u64 epoch; /* 0x0038 */ - __u32 svcc; /* 0x0040 */ -#define LCTL_CR0 0x8000 -#define LCTL_CR6 0x0200 -#define LCTL_CR9 0x0040 -#define LCTL_CR10 0x0020 -#define LCTL_CR11 0x0010 -#define LCTL_CR14 0x0002 - __u16 lctl; /* 0x0044 */ - __s16 icpua; /* 0x0046 */ -#define ICTL_OPEREXC 0x80000000 -#define ICTL_PINT 0x20000000 -#define ICTL_LPSW 0x00400000 -#define ICTL_STCTL 0x00040000 -#define ICTL_ISKE 0x00004000 -#define ICTL_SSKE 0x00002000 -#define ICTL_RRBE 0x00001000 -#define ICTL_TPROT 0x00000200 - __u32 ictl; /* 0x0048 */ -#define ECA_CEI 0x80000000 -#define ECA_IB 0x40000000 -#define ECA_SIGPI 0x10000000 -#define ECA_MVPGI 0x01000000 -#define ECA_AIV 0x00200000 -#define ECA_VX 0x00020000 -#define ECA_PROTEXCI 0x00002000 -#define ECA_APIE 0x00000008 -#define ECA_SII 0x00000001 - __u32 eca; /* 0x004c */ -#define ICPT_INST 0x04 -#define ICPT_PROGI 0x08 -#define ICPT_INSTPROGI 0x0C -#define ICPT_EXTREQ 0x10 -#define ICPT_EXTINT 0x14 -#define ICPT_IOREQ 0x18 -#define ICPT_WAIT 0x1c -#define ICPT_VALIDITY 0x20 -#define ICPT_STOP 0x28 -#define ICPT_OPEREXC 0x2C -#define ICPT_PARTEXEC 0x38 -#define ICPT_IOINST 0x40 -#define ICPT_KSS 0x5c -#define ICPT_MCHKREQ 0x60 -#define ICPT_INT_ENABLE 0x64 -#define ICPT_PV_INSTR 0x68 -#define ICPT_PV_NOTIFY 0x6c -#define ICPT_PV_PREF 0x70 - __u8 icptcode; /* 0x0050 */ - __u8 icptstatus; /* 0x0051 */ - __u16 ihcpu; /* 0x0052 */ - __u8 reserved54; /* 0x0054 */ -#define IICTL_CODE_NONE 0x00 -#define IICTL_CODE_MCHK 0x01 -#define IICTL_CODE_EXT 0x02 -#define IICTL_CODE_IO 0x03 -#define IICTL_CODE_RESTART 0x04 -#define IICTL_CODE_SPECIFICATION 0x10 -#define IICTL_CODE_OPERAND 0x11 - __u8 iictl; /* 0x0055 */ - __u16 ipa; /* 0x0056 */ - __u32 ipb; /* 0x0058 */ - __u32 scaoh; /* 0x005c */ -#define FPF_BPBC 0x20 - __u8 fpf; /* 0x0060 */ -#define ECB_GS 0x40 -#define ECB_TE 0x10 -#define ECB_SPECI 0x08 -#define ECB_SRSI 0x04 -#define ECB_HOSTPROTINT 0x02 -#define ECB_PTF 0x01 - __u8 ecb; /* 0x0061 */ -#define ECB2_CMMA 0x80 -#define ECB2_IEP 0x20 -#define ECB2_PFMFI 0x08 -#define ECB2_ESCA 0x04 -#define ECB2_ZPCI_LSI 0x02 - __u8 ecb2; /* 0x0062 */ -#define ECB3_AISI 0x20 -#define ECB3_AISII 0x10 -#define ECB3_DEA 0x08 -#define ECB3_AES 0x04 -#define ECB3_RI 0x01 - __u8 ecb3; /* 0x0063 */ -#define ESCA_SCAOL_MASK ~0x3fU - __u32 scaol; /* 0x0064 */ - __u8 sdf; /* 0x0068 */ - __u8 epdx; /* 0x0069 */ - __u8 cpnc; /* 0x006a */ - __u8 reserved6b; /* 0x006b */ - __u32 todpr; /* 0x006c */ -#define GISA_FORMAT1 0x00000001 - __u32 gd; /* 0x0070 */ - __u8 reserved74[12]; /* 0x0074 */ - __u64 mso; /* 0x0080 */ - __u64 msl; /* 0x0088 */ - psw_t gpsw; /* 0x0090 */ - __u64 gg14; /* 0x00a0 */ - __u64 gg15; /* 0x00a8 */ - __u8 reservedb0[8]; /* 0x00b0 */ -#define HPID_KVM 0x4 -#define HPID_VSIE 0x5 - __u8 hpid; /* 0x00b8 */ - __u8 reservedb9[7]; /* 0x00b9 */ - union { - struct { - __u32 eiparams; /* 0x00c0 */ - __u16 extcpuaddr; /* 0x00c4 */ - __u16 eic; /* 0x00c6 */ - }; - __u64 mcic; /* 0x00c0 */ - } __packed; - __u32 reservedc8; /* 0x00c8 */ - union { - struct { - __u16 pgmilc; /* 0x00cc */ - __u16 iprcc; /* 0x00ce */ - }; - __u32 edc; /* 0x00cc */ - } __packed; - union { - struct { - __u32 dxc; /* 0x00d0 */ - __u16 mcn; /* 0x00d4 */ - __u8 perc; /* 0x00d6 */ - __u8 peratmid; /* 0x00d7 */ - }; - __u64 faddr; /* 0x00d0 */ - } __packed; - __u64 peraddr; /* 0x00d8 */ - __u8 eai; /* 0x00e0 */ - __u8 peraid; /* 0x00e1 */ - __u8 oai; /* 0x00e2 */ - __u8 armid; /* 0x00e3 */ - __u8 reservede4[4]; /* 0x00e4 */ - union { - __u64 tecmc; /* 0x00e8 */ - struct { - __u16 subchannel_id; /* 0x00e8 */ - __u16 subchannel_nr; /* 0x00ea */ - __u32 io_int_parm; /* 0x00ec */ - __u32 io_int_word; /* 0x00f0 */ - }; - } __packed; - __u8 reservedf4[8]; /* 0x00f4 */ -#define CRYCB_FORMAT_MASK 0x00000003 -#define CRYCB_FORMAT0 0x00000000 -#define CRYCB_FORMAT1 0x00000001 -#define CRYCB_FORMAT2 0x00000003 - __u32 crycbd; /* 0x00fc */ - __u64 gcr[16]; /* 0x0100 */ - union { - __u64 gbea; /* 0x0180 */ - __u64 sidad; - }; - __u8 reserved188[8]; /* 0x0188 */ - __u64 sdnxo; /* 0x0190 */ - __u8 reserved198[8]; /* 0x0198 */ - __u32 fac; /* 0x01a0 */ - __u8 reserved1a4[20]; /* 0x01a4 */ - __u64 cbrlo; /* 0x01b8 */ - __u8 reserved1c0[8]; /* 0x01c0 */ -#define ECD_HOSTREGMGMT 0x20000000 -#define ECD_MEF 0x08000000 -#define ECD_ETOKENF 0x02000000 -#define ECD_ECC 0x00200000 -#define ECD_HMAC 0x00004000 - __u32 ecd; /* 0x01c8 */ - __u8 reserved1cc[18]; /* 0x01cc */ - __u64 pp; /* 0x01de */ - __u8 reserved1e6[2]; /* 0x01e6 */ - __u64 itdba; /* 0x01e8 */ - __u64 riccbd; /* 0x01f0 */ - __u64 gvrd; /* 0x01f8 */ -} __packed __aligned(512); - -struct kvm_s390_itdb { - __u8 data[256]; -}; - -struct sie_page { - struct kvm_s390_sie_block sie_block; - struct mcck_volatile_info mcck_info; /* 0x0200 */ - __u8 reserved218[360]; /* 0x0218 */ - __u64 pv_grregs[16]; /* 0x0380 */ - __u8 reserved400[512]; /* 0x0400 */ - struct kvm_s390_itdb itdb; /* 0x0600 */ - __u8 reserved700[2304]; /* 0x0700 */ -}; - -#endif /* _ASM_KVM_HOST_TYPES_H */ +#endif /* ASM_KVM_HOST_TYPES_H */ From 1b09c13c90ccd908e62f82b309fade358b41cd21 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Thu, 2 Jul 2026 10:29:28 +0200 Subject: [PATCH 69/82] KVM: s390: Move s390 kvm code into a subdirectory Move all the code required to run s390 KVM guests on s390 to a s390 subdirectory. Move gmap related code into a gmap directory to later share gmap code between KVM implementations. Update S390 VFIO-PCI MAINTAINERS filepath. No functional change. Signed-off-by: Steffen Eiden Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- MAINTAINERS | 2 +- arch/s390/kvm/Makefile | 11 +---------- arch/s390/kvm/gmap/Makefile | 5 +++++ arch/s390/kvm/{ => gmap}/dat.c | 0 arch/s390/kvm/{ => gmap}/dat.h | 0 arch/s390/kvm/{ => gmap}/faultin.c | 0 arch/s390/kvm/{ => gmap}/faultin.h | 0 arch/s390/kvm/{ => gmap}/gmap.c | 0 arch/s390/kvm/{ => gmap}/gmap.h | 0 arch/s390/kvm/{ => gmap}/trace_gmap.h | 2 +- arch/s390/kvm/s390/Makefile | 14 ++++++++++++++ arch/s390/kvm/{ => s390}/diag.c | 0 arch/s390/kvm/{ => s390}/gaccess.c | 0 arch/s390/kvm/{ => s390}/gaccess.h | 0 arch/s390/kvm/{ => s390}/guestdbg.c | 0 arch/s390/kvm/{ => s390}/intercept.c | 0 arch/s390/kvm/{ => s390}/interrupt.c | 0 arch/s390/kvm/{ => s390}/pci.c | 0 arch/s390/kvm/{ => s390}/pci.h | 0 arch/s390/kvm/{ => s390}/priv.c | 0 arch/s390/kvm/{ => s390}/pv.c | 0 arch/s390/kvm/{ => s390}/s390.c | 0 arch/s390/kvm/{ => s390}/s390.h | 0 arch/s390/kvm/{ => s390}/sigp.c | 0 arch/s390/kvm/{ => s390}/trace-s390.h | 0 arch/s390/kvm/{ => s390}/trace.h | 0 arch/s390/kvm/{ => s390}/vsie.c | 0 27 files changed, 22 insertions(+), 12 deletions(-) create mode 100644 arch/s390/kvm/gmap/Makefile rename arch/s390/kvm/{ => gmap}/dat.c (100%) rename arch/s390/kvm/{ => gmap}/dat.h (100%) rename arch/s390/kvm/{ => gmap}/faultin.c (100%) rename arch/s390/kvm/{ => gmap}/faultin.h (100%) rename arch/s390/kvm/{ => gmap}/gmap.c (100%) rename arch/s390/kvm/{ => gmap}/gmap.h (100%) rename arch/s390/kvm/{ => gmap}/trace_gmap.h (96%) create mode 100644 arch/s390/kvm/s390/Makefile rename arch/s390/kvm/{ => s390}/diag.c (100%) rename arch/s390/kvm/{ => s390}/gaccess.c (100%) rename arch/s390/kvm/{ => s390}/gaccess.h (100%) rename arch/s390/kvm/{ => s390}/guestdbg.c (100%) rename arch/s390/kvm/{ => s390}/intercept.c (100%) rename arch/s390/kvm/{ => s390}/interrupt.c (100%) rename arch/s390/kvm/{ => s390}/pci.c (100%) rename arch/s390/kvm/{ => s390}/pci.h (100%) rename arch/s390/kvm/{ => s390}/priv.c (100%) rename arch/s390/kvm/{ => s390}/pv.c (100%) rename arch/s390/kvm/{ => s390}/s390.c (100%) rename arch/s390/kvm/{ => s390}/s390.h (100%) rename arch/s390/kvm/{ => s390}/sigp.c (100%) rename arch/s390/kvm/{ => s390}/trace-s390.h (100%) rename arch/s390/kvm/{ => s390}/trace.h (100%) rename arch/s390/kvm/{ => s390}/vsie.c (100%) diff --git a/MAINTAINERS b/MAINTAINERS index 78f3f17f7b44..66fb3d50b580 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23915,7 +23915,7 @@ L: linux-s390@vger.kernel.org L: kvm@vger.kernel.org S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git -F: arch/s390/kvm/pci* +F: arch/s390/kvm/s390/pci* F: drivers/vfio/pci/vfio_pci_zdev.c F: include/uapi/linux/vfio_zdev.h diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile index 961de97b8e80..a4e3875b5bdd 100644 --- a/arch/s390/kvm/Makefile +++ b/arch/s390/kvm/Makefile @@ -3,13 +3,4 @@ # # Copyright IBM Corp. 2008 -include $(srctree)/virt/kvm/Makefile.kvm - -ccflags-y := -I$(src) - -kvm-y += s390.o intercept.o interrupt.o priv.o sigp.o -kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o -kvm-y += dat.o gmap.o faultin.o - -kvm-$(CONFIG_VFIO_PCI_ZDEV_KVM) += pci.o -obj-$(CONFIG_KVM) += kvm.o +obj-$(CONFIG_KVM) += s390/ diff --git a/arch/s390/kvm/gmap/Makefile b/arch/s390/kvm/gmap/Makefile new file mode 100644 index 000000000000..3a4f8df11622 --- /dev/null +++ b/arch/s390/kvm/gmap/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +GMAP ?= ../gmap + +gmap-y += $(GMAP)/dat.o $(GMAP)/gmap.o $(GMAP)/faultin.o diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/gmap/dat.c similarity index 100% rename from arch/s390/kvm/dat.c rename to arch/s390/kvm/gmap/dat.c diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/gmap/dat.h similarity index 100% rename from arch/s390/kvm/dat.h rename to arch/s390/kvm/gmap/dat.h diff --git a/arch/s390/kvm/faultin.c b/arch/s390/kvm/gmap/faultin.c similarity index 100% rename from arch/s390/kvm/faultin.c rename to arch/s390/kvm/gmap/faultin.c diff --git a/arch/s390/kvm/faultin.h b/arch/s390/kvm/gmap/faultin.h similarity index 100% rename from arch/s390/kvm/faultin.h rename to arch/s390/kvm/gmap/faultin.h diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap/gmap.c similarity index 100% rename from arch/s390/kvm/gmap.c rename to arch/s390/kvm/gmap/gmap.c diff --git a/arch/s390/kvm/gmap.h b/arch/s390/kvm/gmap/gmap.h similarity index 100% rename from arch/s390/kvm/gmap.h rename to arch/s390/kvm/gmap/gmap.h diff --git a/arch/s390/kvm/trace_gmap.h b/arch/s390/kvm/gmap/trace_gmap.h similarity index 96% rename from arch/s390/kvm/trace_gmap.h rename to arch/s390/kvm/gmap/trace_gmap.h index a3cad705021b..15ec88bdce55 100644 --- a/arch/s390/kvm/trace_gmap.h +++ b/arch/s390/kvm/gmap/trace_gmap.h @@ -7,7 +7,7 @@ #undef TRACE_SYSTEM #define TRACE_SYSTEM kvm #undef TRACE_INCLUDE_PATH -#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_PATH ../gmap #undef TRACE_INCLUDE_FILE #define TRACE_INCLUDE_FILE trace_gmap diff --git a/arch/s390/kvm/s390/Makefile b/arch/s390/kvm/s390/Makefile new file mode 100644 index 000000000000..762a63826423 --- /dev/null +++ b/arch/s390/kvm/s390/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 + +KVM := ../../../../virt/kvm +include $(srctree)/virt/kvm/Makefile.kvm +include $(srctree)/arch/s390/kvm/gmap/Makefile + +ccflags-y := -I$(src) -I$(srctree)/arch/s390/kvm/gmap + +kvm-y += s390.o intercept.o interrupt.o priv.o sigp.o +kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o +kvm-y += $(gmap-y) + +kvm-$(CONFIG_VFIO_PCI_ZDEV_KVM) += pci.o +obj-$(CONFIG_KVM) += kvm.o diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/s390/diag.c similarity index 100% rename from arch/s390/kvm/diag.c rename to arch/s390/kvm/s390/diag.c diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/s390/gaccess.c similarity index 100% rename from arch/s390/kvm/gaccess.c rename to arch/s390/kvm/s390/gaccess.c diff --git a/arch/s390/kvm/gaccess.h b/arch/s390/kvm/s390/gaccess.h similarity index 100% rename from arch/s390/kvm/gaccess.h rename to arch/s390/kvm/s390/gaccess.h diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/s390/guestdbg.c similarity index 100% rename from arch/s390/kvm/guestdbg.c rename to arch/s390/kvm/s390/guestdbg.c diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/s390/intercept.c similarity index 100% rename from arch/s390/kvm/intercept.c rename to arch/s390/kvm/s390/intercept.c diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/s390/interrupt.c similarity index 100% rename from arch/s390/kvm/interrupt.c rename to arch/s390/kvm/s390/interrupt.c diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/s390/pci.c similarity index 100% rename from arch/s390/kvm/pci.c rename to arch/s390/kvm/s390/pci.c diff --git a/arch/s390/kvm/pci.h b/arch/s390/kvm/s390/pci.h similarity index 100% rename from arch/s390/kvm/pci.h rename to arch/s390/kvm/s390/pci.h diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/s390/priv.c similarity index 100% rename from arch/s390/kvm/priv.c rename to arch/s390/kvm/s390/priv.c diff --git a/arch/s390/kvm/pv.c b/arch/s390/kvm/s390/pv.c similarity index 100% rename from arch/s390/kvm/pv.c rename to arch/s390/kvm/s390/pv.c diff --git a/arch/s390/kvm/s390.c b/arch/s390/kvm/s390/s390.c similarity index 100% rename from arch/s390/kvm/s390.c rename to arch/s390/kvm/s390/s390.c diff --git a/arch/s390/kvm/s390.h b/arch/s390/kvm/s390/s390.h similarity index 100% rename from arch/s390/kvm/s390.h rename to arch/s390/kvm/s390/s390.h diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/s390/sigp.c similarity index 100% rename from arch/s390/kvm/sigp.c rename to arch/s390/kvm/s390/sigp.c diff --git a/arch/s390/kvm/trace-s390.h b/arch/s390/kvm/s390/trace-s390.h similarity index 100% rename from arch/s390/kvm/trace-s390.h rename to arch/s390/kvm/s390/trace-s390.h diff --git a/arch/s390/kvm/trace.h b/arch/s390/kvm/s390/trace.h similarity index 100% rename from arch/s390/kvm/trace.h rename to arch/s390/kvm/s390/trace.h diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/s390/vsie.c similarity index 100% rename from arch/s390/kvm/vsie.c rename to arch/s390/kvm/s390/vsie.c From 9881bf55c58921b6742338cb487afca86da425e2 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Fri, 10 Jul 2026 14:40:10 +0200 Subject: [PATCH 70/82] KVM: s390: Move PGM code definitions to asm/kvm_host.h Move PGM code definitions from kvm_host_s390.h (back) to the generic kvm_host.h. These definitions are needed by multiple KVM implementations and should be in a shared location. No functional change. Signed-off-by: Steffen Eiden Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/include/asm/kvm_host.h | 57 +++++++++++++++++++++++++++ arch/s390/include/asm/kvm_host_s390.h | 57 --------------------------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index e76ceee11ef5..147325978225 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -5,4 +5,61 @@ #include +#define PGM_OPERATION 0x01 +#define PGM_PRIVILEGED_OP 0x02 +#define PGM_EXECUTE 0x03 +#define PGM_PROTECTION 0x04 +#define PGM_ADDRESSING 0x05 +#define PGM_SPECIFICATION 0x06 +#define PGM_DATA 0x07 +#define PGM_FIXED_POINT_OVERFLOW 0x08 +#define PGM_FIXED_POINT_DIVIDE 0x09 +#define PGM_DECIMAL_OVERFLOW 0x0a +#define PGM_DECIMAL_DIVIDE 0x0b +#define PGM_HFP_EXPONENT_OVERFLOW 0x0c +#define PGM_HFP_EXPONENT_UNDERFLOW 0x0d +#define PGM_HFP_SIGNIFICANCE 0x0e +#define PGM_HFP_DIVIDE 0x0f +#define PGM_SEGMENT_TRANSLATION 0x10 +#define PGM_PAGE_TRANSLATION 0x11 +#define PGM_TRANSLATION_SPEC 0x12 +#define PGM_SPECIAL_OPERATION 0x13 +#define PGM_OPERAND 0x15 +#define PGM_TRACE_TABEL 0x16 +#define PGM_VECTOR_PROCESSING 0x1b +#define PGM_SPACE_SWITCH 0x1c +#define PGM_HFP_SQUARE_ROOT 0x1d +#define PGM_PC_TRANSLATION_SPEC 0x1f +#define PGM_AFX_TRANSLATION 0x20 +#define PGM_ASX_TRANSLATION 0x21 +#define PGM_LX_TRANSLATION 0x22 +#define PGM_EX_TRANSLATION 0x23 +#define PGM_PRIMARY_AUTHORITY 0x24 +#define PGM_SECONDARY_AUTHORITY 0x25 +#define PGM_LFX_TRANSLATION 0x26 +#define PGM_LSX_TRANSLATION 0x27 +#define PGM_ALET_SPECIFICATION 0x28 +#define PGM_ALEN_TRANSLATION 0x29 +#define PGM_ALE_SEQUENCE 0x2a +#define PGM_ASTE_VALIDITY 0x2b +#define PGM_ASTE_SEQUENCE 0x2c +#define PGM_EXTENDED_AUTHORITY 0x2d +#define PGM_LSTE_SEQUENCE 0x2e +#define PGM_ASTE_INSTANCE 0x2f +#define PGM_STACK_FULL 0x30 +#define PGM_STACK_EMPTY 0x31 +#define PGM_STACK_SPECIFICATION 0x32 +#define PGM_STACK_TYPE 0x33 +#define PGM_STACK_OPERATION 0x34 +#define PGM_ASCE_TYPE 0x38 +#define PGM_REGION_FIRST_TRANS 0x39 +#define PGM_REGION_SECOND_TRANS 0x3a +#define PGM_REGION_THIRD_TRANS 0x3b +#define PGM_SECURE_STORAGE_ACCESS 0x3d +#define PGM_NON_SECURE_STORAGE_ACCESS 0x3e +#define PGM_SECURE_STORAGE_VIOLATION 0x3f +#define PGM_MONITOR 0x40 +#define PGM_PER 0x80 +#define PGM_CRYPTO_OPERATION 0x119 + #endif /* ASM_KVM_HOST_H */ diff --git a/arch/s390/include/asm/kvm_host_s390.h b/arch/s390/include/asm/kvm_host_s390.h index 0c4e061d7437..22e8d7c4a755 100644 --- a/arch/s390/include/asm/kvm_host_s390.h +++ b/arch/s390/include/asm/kvm_host_s390.h @@ -150,63 +150,6 @@ struct kvm_vcpu_stat { u64 signal_exits; }; -#define PGM_OPERATION 0x01 -#define PGM_PRIVILEGED_OP 0x02 -#define PGM_EXECUTE 0x03 -#define PGM_PROTECTION 0x04 -#define PGM_ADDRESSING 0x05 -#define PGM_SPECIFICATION 0x06 -#define PGM_DATA 0x07 -#define PGM_FIXED_POINT_OVERFLOW 0x08 -#define PGM_FIXED_POINT_DIVIDE 0x09 -#define PGM_DECIMAL_OVERFLOW 0x0a -#define PGM_DECIMAL_DIVIDE 0x0b -#define PGM_HFP_EXPONENT_OVERFLOW 0x0c -#define PGM_HFP_EXPONENT_UNDERFLOW 0x0d -#define PGM_HFP_SIGNIFICANCE 0x0e -#define PGM_HFP_DIVIDE 0x0f -#define PGM_SEGMENT_TRANSLATION 0x10 -#define PGM_PAGE_TRANSLATION 0x11 -#define PGM_TRANSLATION_SPEC 0x12 -#define PGM_SPECIAL_OPERATION 0x13 -#define PGM_OPERAND 0x15 -#define PGM_TRACE_TABEL 0x16 -#define PGM_VECTOR_PROCESSING 0x1b -#define PGM_SPACE_SWITCH 0x1c -#define PGM_HFP_SQUARE_ROOT 0x1d -#define PGM_PC_TRANSLATION_SPEC 0x1f -#define PGM_AFX_TRANSLATION 0x20 -#define PGM_ASX_TRANSLATION 0x21 -#define PGM_LX_TRANSLATION 0x22 -#define PGM_EX_TRANSLATION 0x23 -#define PGM_PRIMARY_AUTHORITY 0x24 -#define PGM_SECONDARY_AUTHORITY 0x25 -#define PGM_LFX_TRANSLATION 0x26 -#define PGM_LSX_TRANSLATION 0x27 -#define PGM_ALET_SPECIFICATION 0x28 -#define PGM_ALEN_TRANSLATION 0x29 -#define PGM_ALE_SEQUENCE 0x2a -#define PGM_ASTE_VALIDITY 0x2b -#define PGM_ASTE_SEQUENCE 0x2c -#define PGM_EXTENDED_AUTHORITY 0x2d -#define PGM_LSTE_SEQUENCE 0x2e -#define PGM_ASTE_INSTANCE 0x2f -#define PGM_STACK_FULL 0x30 -#define PGM_STACK_EMPTY 0x31 -#define PGM_STACK_SPECIFICATION 0x32 -#define PGM_STACK_TYPE 0x33 -#define PGM_STACK_OPERATION 0x34 -#define PGM_ASCE_TYPE 0x38 -#define PGM_REGION_FIRST_TRANS 0x39 -#define PGM_REGION_SECOND_TRANS 0x3a -#define PGM_REGION_THIRD_TRANS 0x3b -#define PGM_SECURE_STORAGE_ACCESS 0x3d -#define PGM_NON_SECURE_STORAGE_ACCESS 0x3e -#define PGM_SECURE_STORAGE_VIOLATION 0x3f -#define PGM_MONITOR 0x40 -#define PGM_PER 0x80 -#define PGM_CRYPTO_OPERATION 0x119 - /* irq types in ascend order of priorities */ enum irq_types { IRQ_PEND_SET_PREFIX = 0, From d487a24041c2ea9999ff6e8384002b874de64c94 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Thu, 2 Apr 2026 06:00:00 +0200 Subject: [PATCH 71/82] KVM: s390: Prepare gmap for a second KVM implementation Refactor gmap code such that a second s390 (host) KVM implementation can use the gmap code as well. Move mmu code from s390 to gmap so the other KVM implementation can use it as well. No functional change. Signed-off-by: Steffen Eiden Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/gmap/Makefile | 2 +- arch/s390/kvm/gmap/kvm_mmu.c | 133 +++++++++++++++++++++++++++++++++++ arch/s390/kvm/gmap/kvm_mmu.h | 18 +++++ arch/s390/kvm/s390/s390.c | 126 ++++----------------------------- arch/s390/kvm/s390/s390.h | 8 +++ 5 files changed, 175 insertions(+), 112 deletions(-) create mode 100644 arch/s390/kvm/gmap/kvm_mmu.c create mode 100644 arch/s390/kvm/gmap/kvm_mmu.h diff --git a/arch/s390/kvm/gmap/Makefile b/arch/s390/kvm/gmap/Makefile index 3a4f8df11622..dd4ef062c536 100644 --- a/arch/s390/kvm/gmap/Makefile +++ b/arch/s390/kvm/gmap/Makefile @@ -2,4 +2,4 @@ GMAP ?= ../gmap -gmap-y += $(GMAP)/dat.o $(GMAP)/gmap.o $(GMAP)/faultin.o +gmap-y += $(GMAP)/dat.o $(GMAP)/gmap.o $(GMAP)/faultin.o $(GMAP)/kvm_mmu.o diff --git a/arch/s390/kvm/gmap/kvm_mmu.c b/arch/s390/kvm/gmap/kvm_mmu.c new file mode 100644 index 000000000000..b08b8229bb6f --- /dev/null +++ b/arch/s390/kvm/gmap/kvm_mmu.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include "s390.h" +#include "gmap.h" +#include "dat.h" +#include "kvm_mmu.h" + +/* + * Get (and clear) the dirty memory log for a memory slot. + */ +int s390_kvm_mmu_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) +{ + int r; + unsigned long n; + struct kvm_memory_slot *memslot; + int is_dirty; + + if (kvm_is_ucontrol(kvm)) + return -EINVAL; + + mutex_lock(&kvm->slots_lock); + + r = -EINVAL; + if (log->slot >= KVM_USER_MEM_SLOTS) + goto out; + + r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot); + if (r) + goto out; + + /* Clear the dirty log */ + if (is_dirty) { + n = kvm_dirty_bitmap_bytes(memslot); + memset(memslot->dirty_bitmap, 0, n); + } + r = 0; +out: + mutex_unlock(&kvm->slots_lock); + return r; +} + +int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + if (kvm_is_ucontrol(kvm) && new && new->id < KVM_USER_MEM_SLOTS) + return -EINVAL; + + /* When we are protected, we should not change the memory slots */ + if (kvm_s390_pv_get_handle(kvm)) + return -EINVAL; + + if (change != KVM_MR_DELETE && change != KVM_MR_FLAGS_ONLY) { + /* + * A few sanity checks. The memory in userland is ok to be + * fragmented into various different vmas. It is okay to mmap() + * and munmap() stuff in this slot after doing this call at any + * time. + */ + if (new->userspace_addr & ~PAGE_MASK) + return -EINVAL; + if ((new->base_gfn + new->npages) * PAGE_SIZE > kvm->arch.mem_limit) + return -EINVAL; + if (!asce_contains_gfn(kvm->arch.gmap->asce, new->base_gfn + new->npages - 1)) + return -EINVAL; + } + + if (!kvm_s390_is_migration_mode(kvm)) + return 0; + + /* + * Turn off migration mode when: + * - userspace creates a new memslot with dirty logging off, + * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and + * dirty logging is turned off. + * Migration mode expects dirty page logging being enabled to store + * its dirty bitmap. + */ + if (change != KVM_MR_DELETE && + !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) + WARN(kvm_s390_vm_stop_migration(kvm), + "Failed to stop migration mode"); + + return 0; +} + +void s390_kvm_mmu_commit_memory_region(struct kvm *kvm, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + int rc = 0; + + guard(mutex)(&kvm->slots_arch_lock); + + if (change == KVM_MR_FLAGS_ONLY) + return; + + mc = kvm_s390_new_mmu_cache(); + if (!mc) { + rc = -ENOMEM; + goto out; + } + + scoped_guard(write_lock, &kvm->mmu_lock) { + kvm_s390_update_cmma_dirty(kvm, old); + switch (change) { + case KVM_MR_DELETE: + rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); + break; + case KVM_MR_MOVE: + rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); + if (rc) + break; + fallthrough; + case KVM_MR_CREATE: + rc = dat_create_slot(mc, kvm->arch.gmap->asce, new->base_gfn, new->npages); + break; + case KVM_MR_FLAGS_ONLY: + break; + default: + WARN(1, "Unknown KVM MR CHANGE: %d\n", change); + } + } +out: + if (rc) + pr_warn("failed to commit memory region\n"); +} diff --git a/arch/s390/kvm/gmap/kvm_mmu.h b/arch/s390/kvm/gmap/kvm_mmu.h new file mode 100644 index 000000000000..cdbd390bd33c --- /dev/null +++ b/arch/s390/kvm/gmap/kvm_mmu.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef ARCH_KVM_GMAP_KVM_MMU_H +#define ARCH_KVM_GMAP_KVM_MMU_H + +#include + +int s390_kvm_mmu_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log); +int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change); +void s390_kvm_mmu_commit_memory_region(struct kvm *kvm, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change); + +#endif /* ARCH_KVM_GMAP_KVM_MMU_H */ diff --git a/arch/s390/kvm/s390/s390.c b/arch/s390/kvm/s390/s390.c index bb8ef40e3ab5..d90b53c36706 100644 --- a/arch/s390/kvm/s390/s390.c +++ b/arch/s390/kvm/s390/s390.c @@ -55,6 +55,7 @@ #include "gmap.h" #include "faultin.h" #include "pci.h" +#include "kvm_mmu.h" #define CREATE_TRACE_POINTS #include "trace.h" @@ -746,33 +747,7 @@ static void sca_del_vcpu(struct kvm_vcpu *vcpu); int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) { - int r; - unsigned long n; - struct kvm_memory_slot *memslot; - int is_dirty; - - if (kvm_is_ucontrol(kvm)) - return -EINVAL; - - mutex_lock(&kvm->slots_lock); - - r = -EINVAL; - if (log->slot >= KVM_USER_MEM_SLOTS) - goto out; - - r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot); - if (r) - goto out; - - /* Clear the dirty log */ - if (is_dirty) { - n = kvm_dirty_bitmap_bytes(memslot); - memset(memslot->dirty_bitmap, 0, n); - } - r = 0; -out: - mutex_unlock(&kvm->slots_lock); - return r; + return s390_kvm_mmu_get_dirty_log(kvm, log); } static void icpt_operexc_on_all_vcpus(struct kvm *kvm) @@ -1268,7 +1243,7 @@ static int kvm_s390_vm_start_migration(struct kvm *kvm) * Must be called with kvm->slots_arch_lock to avoid races with ourselves, * kvm_s390_vm_start_migration() and kvm_s390_get_cmma_bits(). */ -static int kvm_s390_vm_stop_migration(struct kvm *kvm) +int kvm_s390_vm_stop_migration(struct kvm *kvm) { /* migration mode already disabled */ if (!kvm->arch.migration_mode) @@ -5776,45 +5751,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *new, enum kvm_mr_change change) { - if (kvm_is_ucontrol(kvm) && new && new->id < KVM_USER_MEM_SLOTS) - return -EINVAL; - - /* When we are protected, we should not change the memory slots */ - if (kvm_s390_pv_get_handle(kvm)) - return -EINVAL; - - if (change != KVM_MR_DELETE && change != KVM_MR_FLAGS_ONLY) { - /* - * A few sanity checks. The memory in userland is ok to be - * fragmented into various different vmas. It is okay to mmap() - * and munmap() stuff in this slot after doing this call at any - * time. - */ - if (new->userspace_addr & ~PAGE_MASK) - return -EINVAL; - if ((new->base_gfn + new->npages) * PAGE_SIZE > kvm->arch.mem_limit) - return -EINVAL; - if (!asce_contains_gfn(kvm->arch.gmap->asce, new->base_gfn + new->npages - 1)) - return -EINVAL; - } - - if (!kvm->arch.migration_mode) - return 0; - - /* - * Turn off migration mode when: - * - userspace creates a new memslot with dirty logging off, - * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and - * dirty logging is turned off. - * Migration mode expects dirty page logging being enabled to store - * its dirty bitmap. - */ - if (change != KVM_MR_DELETE && - !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) - WARN(kvm_s390_vm_stop_migration(kvm), - "Failed to stop migration mode"); - - return 0; + return s390_kvm_mmu_prepare_memory_region(kvm, old, new, change); } static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) @@ -5830,55 +5767,22 @@ static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_ return 0; } -void kvm_arch_commit_memory_region(struct kvm *kvm, - struct kvm_memory_slot *old, - const struct kvm_memory_slot *new, - enum kvm_mr_change change) +void kvm_s390_update_cmma_dirty(struct kvm *kvm, struct kvm_memory_slot *old) { const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, }; - struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; - int rc = 0; - guard(mutex)(&kvm->slots_arch_lock); - - if (change == KVM_MR_FLAGS_ONLY) - return; - - mc = kvm_s390_new_mmu_cache(); - if (!mc) { - rc = -ENOMEM; - goto out; + if (kvm->arch.migration_mode && kvm->arch.use_cmma && old) { + _dat_walk_gfn_range(old->base_gfn, old->base_gfn + old->npages, + kvm->arch.gmap->asce, &ops, DAT_WALK_IGN_HOLES, + &kvm->arch.cmma_dirty_pages); } +} - scoped_guard(write_lock, &kvm->mmu_lock) { - if (kvm->arch.migration_mode && kvm->arch.use_cmma && old) { - _dat_walk_gfn_range(old->base_gfn, old->base_gfn + old->npages, - kvm->arch.gmap->asce, &ops, DAT_WALK_IGN_HOLES, - &kvm->arch.cmma_dirty_pages); - } - - switch (change) { - case KVM_MR_DELETE: - rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); - break; - case KVM_MR_MOVE: - rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); - if (rc) - break; - fallthrough; - case KVM_MR_CREATE: - rc = dat_create_slot(mc, kvm->arch.gmap->asce, new->base_gfn, new->npages); - break; - case KVM_MR_FLAGS_ONLY: - break; - default: - WARN(1, "Unknown KVM MR CHANGE: %d\n", change); - } - } -out: - if (rc) - pr_warn("failed to commit memory region\n"); - return; +void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + s390_kvm_mmu_commit_memory_region(kvm, old, new, change); } /** diff --git a/arch/s390/kvm/s390/s390.h b/arch/s390/kvm/s390/s390.h index e144ca2e6d5e..d284a263ba70 100644 --- a/arch/s390/kvm/s390/s390.h +++ b/arch/s390/kvm/s390/s390.h @@ -472,6 +472,9 @@ int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned i unsigned long bits); bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); +void kvm_s390_update_cmma_dirty(struct kvm *kvm, struct kvm_memory_slot *old); +int kvm_s390_vm_stop_migration(struct kvm *kvm); + /* implemented in diag.c */ int kvm_s390_handle_diag(struct kvm_vcpu *vcpu); @@ -594,6 +597,11 @@ static inline bool kvm_s390_cur_gmap_fault_is_write(void) return test_facility(75) && (current->thread.gmap_teid.fsi == TEID_FSI_STORE); } +static __always_inline int kvm_s390_is_migration_mode(struct kvm *kvm) +{ + return kvm->arch.migration_mode; +} + /** * kvm_s390_vcpu_crypto_reset_all * From a07276d5d18810fe716edb88f530c19fa067b1b1 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 3 Aug 2026 13:22:39 +0200 Subject: [PATCH 72/82] KVM: s390: gmap: Make storage keys optional Guard guest storage key handling behind `KVM_S390_MANAGES_S390_GUEST`. This enables other KVM implementations to use gmap without implementing storage key infrastructure. Define KVM_S390_MANAGES_S390_GUEST to 1 for KVM hosts managing s390 guests (in kvm_host_s390.h). A KVM implementation not implementing those features must define KVM_S390_MANAGES_S390_GUEST to 0 in its kvm_host_.h. No functional changes besides the new guard. Signed-off-by: Steffen Eiden Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/include/asm/kvm_host_s390.h | 2 ++ arch/s390/kvm/gmap/dat.c | 2 ++ arch/s390/kvm/gmap/dat.h | 4 ++++ arch/s390/kvm/gmap/gmap.c | 3 +++ arch/s390/kvm/gmap/gmap.h | 6 ++++++ 5 files changed, 17 insertions(+) diff --git a/arch/s390/include/asm/kvm_host_s390.h b/arch/s390/include/asm/kvm_host_s390.h index 22e8d7c4a755..cd692f8fb764 100644 --- a/arch/s390/include/asm/kvm_host_s390.h +++ b/arch/s390/include/asm/kvm_host_s390.h @@ -32,6 +32,8 @@ #define KVM_INTERNAL_MEM_SLOTS 1 +#define KVM_S390_MANAGES_S390_GUEST 1 + /* * These seem to be used for allocating ->chip in the routing table, which we * don't use. 1 is as small as we can get to reduce the needed memory. If we diff --git a/arch/s390/kvm/gmap/dat.c b/arch/s390/kvm/gmap/dat.c index f2ea013cb33e..d0790b3c96f4 100644 --- a/arch/s390/kvm/gmap/dat.c +++ b/arch/s390/kvm/gmap/dat.c @@ -613,6 +613,7 @@ long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce, return dat_crste_walk_range(start, min(end, asce_end(asce)), table, &walk); } +#if KVM_S390_MANAGES_S390_GUEST int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey) { union crste *crstep; @@ -843,6 +844,7 @@ long dat_reset_skeys(union asce asce, gfn_t start) return _dat_walk_gfn_range(start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, NULL); } +#endif /* KVM_S390_MANAGES_S390_GUEST */ struct slot_priv { unsigned long token; diff --git a/arch/s390/kvm/gmap/dat.h b/arch/s390/kvm/gmap/dat.h index 87108f51228e..2cc072168810 100644 --- a/arch/s390/kvm/gmap/dat.h +++ b/arch/s390/kvm/gmap/dat.h @@ -9,6 +9,7 @@ #ifndef ARCH_KVM_GMAP_DAT_H #define ARCH_KVM_GMAP_DAT_H +#include #include #include #include @@ -532,6 +533,8 @@ int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, in void dat_free_level(struct crst_table *table, bool owns_ptes); struct crst_table *dat_alloc_crst_sleepable(unsigned long init); int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype); + +#if KVM_S390_MANAGES_S390_GUEST int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey); int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, union skey skey, bool nq); @@ -539,6 +542,7 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf union skey skey, union skey *oldkey, bool nq, bool mr, bool mc); int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey); long dat_reset_skeys(union asce asce, gfn_t start); +#endif /* KVM_S390_MANAGES_S390_GUEST */ unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param); void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val); diff --git a/arch/s390/kvm/gmap/gmap.c b/arch/s390/kvm/gmap/gmap.c index 9ae55a1d6f09..0c5e20f033b6 100644 --- a/arch/s390/kvm/gmap/gmap.c +++ b/arch/s390/kvm/gmap/gmap.c @@ -945,6 +945,8 @@ void gmap_split_huge_pages(struct gmap *gmap) } while (start); } +#if KVM_S390_MANAGES_S390_GUEST + static int _gmap_enable_skeys(struct gmap *gmap) { gfn_t start = 0; @@ -977,6 +979,7 @@ int gmap_enable_skeys(struct gmap *gmap) mmap_write_unlock(gmap->kvm->mm); return rc; } +#endif /* KVM_S390_MANAGES_S390_GUEST */ static long _destroy_pages_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { diff --git a/arch/s390/kvm/gmap/gmap.h b/arch/s390/kvm/gmap/gmap.h index c54c35e47d6d..328a81c72c7a 100644 --- a/arch/s390/kvm/gmap/gmap.h +++ b/arch/s390/kvm/gmap/gmap.h @@ -10,6 +10,8 @@ #ifndef ARCH_KVM_GMAP_GMAP_H #define ARCH_KVM_GMAP_GMAP_H +#include + #include "dat.h" /** @@ -98,7 +100,11 @@ int gmap_set_limit(struct gmap *gmap, gfn_t limit); int gmap_ucas_translate(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, gpa_t *gaddr); int gmap_ucas_map(struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, unsigned long count); void gmap_ucas_unmap(struct gmap *gmap, gfn_t c_gfn, unsigned long count); + +#if KVM_S390_MANAGES_S390_GUEST int gmap_enable_skeys(struct gmap *gmap); +#endif /* KVM_S390_MANAGES_S390_GUEST */ + int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interruptible); int gmap_insert_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level); From ce4bee6a91da37948dd3c4de9dea0aa9fbdece16 Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 3 Aug 2026 13:39:42 +0200 Subject: [PATCH 73/82] KVM: s390: gmap: Make CMMA optional Guard guest CMMA behind `KVM_S390_MANAGES_S390_GUEST`. This enables other KVM implementations to use gmap without implementing CMMA. No functional changes. Signed-off-by: Steffen Eiden Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/gmap/dat.c | 2 ++ arch/s390/kvm/gmap/dat.h | 2 ++ arch/s390/kvm/gmap/gmap.c | 2 ++ arch/s390/kvm/gmap/gmap.h | 4 +++- 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/s390/kvm/gmap/dat.c b/arch/s390/kvm/gmap/dat.c index d0790b3c96f4..f4bad4333d8e 100644 --- a/arch/s390/kvm/gmap/dat.c +++ b/arch/s390/kvm/gmap/dat.c @@ -1060,6 +1060,7 @@ int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn) return 0; } +#if KVM_S390_MANAGES_S390_GUEST /** * dat_perform_essa() - Perform ESSA actions on the PGSTE. * @asce: The asce to operate on. @@ -1337,3 +1338,4 @@ int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, } return _dat_walk_gfn_range(gfn, gfn + count, asce, &ops, DAT_WALK_IGN_HOLES, &state); } +#endif /* KVM_S390_MANAGES_S390_GUEST */ diff --git a/arch/s390/kvm/gmap/dat.h b/arch/s390/kvm/gmap/dat.h index 2cc072168810..4f11f9bbd83a 100644 --- a/arch/s390/kvm/gmap/dat.h +++ b/arch/s390/kvm/gmap/dat.h @@ -552,12 +552,14 @@ int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gf int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn); bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end); +#if KVM_S390_MANAGES_S390_GUEST int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty); long dat_reset_cmma(union asce asce, gfn_t start_gfn); int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values); int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem); int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, unsigned long count, unsigned long mask, const uint8_t *bits); +#endif /* KVM_S390_MANAGES_S390_GUEST */ int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc); diff --git a/arch/s390/kvm/gmap/gmap.c b/arch/s390/kvm/gmap/gmap.c index 0c5e20f033b6..96feaebf09c8 100644 --- a/arch/s390/kvm/gmap/gmap.c +++ b/arch/s390/kvm/gmap/gmap.c @@ -1101,6 +1101,7 @@ int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gf return 0; } +#if KVM_S390_MANAGES_S390_GUEST static long __set_cmma_clean_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { union pgste pgste; @@ -1144,6 +1145,7 @@ void _gmap_set_cmma_all(struct gmap *gmap, bool dirty) cond_resched(); } while (gfn); } +#endif /* KVM_S390_MANAGES_S390_GUEST */ static void gmap_unshadow_level(struct gmap *sg, gfn_t r_gfn, int level) { diff --git a/arch/s390/kvm/gmap/gmap.h b/arch/s390/kvm/gmap/gmap.h index 328a81c72c7a..6f3bf11d2166 100644 --- a/arch/s390/kvm/gmap/gmap.h +++ b/arch/s390/kvm/gmap/gmap.h @@ -110,7 +110,6 @@ int gmap_insert_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn gfn_t r_gfn, int level); int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, kvm_pfn_t pfn, int level, bool wr); -void _gmap_set_cmma_all(struct gmap *gmap, bool dirty); void _gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn); struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, union asce asce, int edat_level); @@ -204,6 +203,8 @@ static inline bool pte_needs_unshadow(union pte oldpte, union pte newpte, union return !newpte.h.p || !newpte.s.pr; } +#if KVM_S390_MANAGES_S390_GUEST +void _gmap_set_cmma_all(struct gmap *gmap, bool dirty); static inline void gmap_set_cmma_all_dirty(struct gmap *gmap) { _gmap_set_cmma_all(gmap, true); @@ -213,6 +214,7 @@ static inline void gmap_set_cmma_all_clean(struct gmap *gmap) { _gmap_set_cmma_all(gmap, false); } +#endif /* KVM_S390_MANAGES_S390_GUEST */ static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, union pte newpte, union pgste pgste, gfn_t gfn, bool needs_lock) From 188a15cc7fa544e4a041825aacf565354d23d4cb Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Thu, 2 Jul 2026 14:35:05 +0200 Subject: [PATCH 74/82] KVM: s390: gmap: Make prefix handling optional Guard guest prefix handling behind `KVM_S390_MANAGES_S390_GUEST`. This enables other KVM implementations to use gmap without implementing prefix handling. The prefix handling is integrated deeply in the gmap implementation. Therefore, provide safe default implementations for the guarded functions. No functional changes. Signed-off-by: Steffen Eiden Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/gmap/dat.c | 2 +- arch/s390/kvm/gmap/dat.h | 9 +++++++++ arch/s390/kvm/gmap/gmap.c | 12 +++++++----- arch/s390/kvm/gmap/gmap.h | 9 ++++++++- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/arch/s390/kvm/gmap/dat.c b/arch/s390/kvm/gmap/dat.c index f4bad4333d8e..24547e39fab2 100644 --- a/arch/s390/kvm/gmap/dat.c +++ b/arch/s390/kvm/gmap/dat.c @@ -1012,6 +1012,7 @@ bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end) return _dat_walk_gfn_range(start, end, asce, &test_age_ops, 0, NULL) > 0; } +#if KVM_S390_MANAGES_S390_GUEST static long dat_set_pn_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { union crste newcrste, oldcrste; @@ -1060,7 +1061,6 @@ int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn) return 0; } -#if KVM_S390_MANAGES_S390_GUEST /** * dat_perform_essa() - Perform ESSA actions on the PGSTE. * @asce: The asce to operate on. diff --git a/arch/s390/kvm/gmap/dat.h b/arch/s390/kvm/gmap/dat.h index 4f11f9bbd83a..e452c141b841 100644 --- a/arch/s390/kvm/gmap/dat.h +++ b/arch/s390/kvm/gmap/dat.h @@ -549,7 +549,16 @@ void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end, u16 type, u16 param); + +#if KVM_S390_MANAGES_S390_GUEST int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn); +#else +static inline int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn) +{ + return 0; +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ + bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end); #if KVM_S390_MANAGES_S390_GUEST diff --git a/arch/s390/kvm/gmap/gmap.c b/arch/s390/kvm/gmap/gmap.c index 96feaebf09c8..66a21b5751a0 100644 --- a/arch/s390/kvm/gmap/gmap.c +++ b/arch/s390/kvm/gmap/gmap.c @@ -24,11 +24,6 @@ #include "s390.h" #include "faultin.h" -static inline bool kvm_s390_is_in_sie(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.sie_block->prog0c & PROG_IN_SIE; -} - static int gmap_limit_to_type(gfn_t limit) { if (!limit) @@ -256,6 +251,12 @@ int s390_replace_asce(struct gmap *gmap) return 0; } +#if KVM_S390_MANAGES_S390_GUEST +static inline bool kvm_s390_is_in_sie(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sie_block->prog0c & PROG_IN_SIE; +} + bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint) { struct kvm *kvm = gmap->kvm; @@ -278,6 +279,7 @@ bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint) } return true; } +#endif /* KVM_S390_MANAGES_S390_GUEST */ struct clear_young_pte_priv { struct gmap *gmap; diff --git a/arch/s390/kvm/gmap/gmap.h b/arch/s390/kvm/gmap/gmap.h index 6f3bf11d2166..8f47f29c89fd 100644 --- a/arch/s390/kvm/gmap/gmap.h +++ b/arch/s390/kvm/gmap/gmap.h @@ -85,7 +85,6 @@ struct gmap_cache { for (pos = (head); n = pos ? pos->next : NULL, pos; pos = n) int s390_replace_asce(struct gmap *gmap); -bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint); bool gmap_age_gfn(struct gmap *gmap, gfn_t start, gfn_t end); bool gmap_unmap_gfn_range(struct gmap *gmap, struct kvm_memory_slot *slot, gfn_t start, gfn_t end); int gmap_try_fixup_minor(struct gmap *gmap, struct guest_fault *fault); @@ -163,6 +162,14 @@ static inline void gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gf _gmap_handle_vsie_unshadow_event(parent, gfn); } +#if KVM_S390_MANAGES_S390_GUEST +bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint); +#else +static inline bool _gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end, bool hint) +{ + return true; +} +#endif /* KVM_S390_MANAGES_S390_GUEST */ static inline bool gmap_mkold_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end) { return _gmap_unmap_prefix(gmap, gfn, end, true); From b1f092d94f621307927f145e3cc31893da51fc08 Mon Sep 17 00:00:00 2001 From: Anthony Krowiak Date: Wed, 12 Aug 2026 16:02:32 -0400 Subject: [PATCH 75/82] s390/vfio-ap: Fix stale do_remove flag across iterations in vfio_ap_mdev_cfg_remove The do_remove flag in vfio_ap_mdev_cfg_remove() is initialised to zero before the loop that iterates over the list of matrix mdevs, but is never reset at the start of each iteration. Since do_remove is OR-accumulated across iterations, a positive result from one mdev carries over to subsequent mdevs. The fix is to set the do_remove flag with the first call to bitmap_and; for example: do_remove = bitmap_an rather than do_remove |= bitmap_and. Fixes: eeb386aeb5b7 ("s390/vfio-ap: handle config changed and scan complete notification") Cc: stable@vger.kernel.org Signed-off-by: Anthony Krowiak Reviewed-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- drivers/s390/crypto/vfio_ap_ops.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 99a0efd999ef..5a0932d011d4 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -2598,15 +2598,15 @@ static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove, DECLARE_BITMAP(aprem, AP_DEVICES); DECLARE_BITMAP(aqrem, AP_DOMAINS); DECLARE_BITMAP(cdrem, AP_DOMAINS); - int do_remove = 0; + int do_remove; list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { mutex_lock(&matrix_mdev->kvm->lock); mutex_lock(&matrix_dev->mdevs_lock); - do_remove |= bitmap_and(aprem, ap_remove, - matrix_mdev->matrix.apm, - AP_DEVICES); + do_remove = bitmap_and(aprem, ap_remove, + matrix_mdev->matrix.apm, + AP_DEVICES); do_remove |= bitmap_and(aqrem, aq_remove, matrix_mdev->matrix.aqm, AP_DOMAINS); From d50346801b4f144e42b49cd4f1496010498ab114 Mon Sep 17 00:00:00 2001 From: Anthony Krowiak Date: Wed, 12 Aug 2026 16:02:33 -0400 Subject: [PATCH 76/82] s390/vfio-ap: Fix dereference matrix_mdev->kvm without checking for NULL The ap_driver structure has two fields which are function pointers to callbacks: * .on_config_changed: called at the start of the AP bus scan function to notify the device driver that the host AP configuration has changed and the associated AP devices will be added or removed accordingly. This gives the implementor a chance to evaluate the configuration changes and respond to them before the associated devices are added or removed. * .on_scan_complete: Called at the end of the AP bus scan function to notify the device driver that the host AP configuration has changed and the AP devices have been added or removed accordingly. This gives the implementor the opportunity to respond to the changes after the associated devices are added or removed. These two callbacks are implemented in the vfio_ap device driver via the vfio_ap_on_cfg_changed and vfio_ap_on_scan_complete functions respectively. Within the call stack of these two callback functions the matrix_mdev->kvm->lock mutex is taken without checking whether matrix_mdev->kvm is NULL or not. If matrix_mdev->kvm has never been set, trying to take the lock will trigger a NULL pointer dereference. This patch adds checks for matrix_mdev->kvm == NULL before taking the matrix_mdev->kvm->lock mutex. Note that the matrix_mdev->kvm->lock mutex taken in the vfio_ap_mdev_hot_plug_config function is moved to the calling function along with the matrix_dev->mdevs_lock which is needed there to access the fields of the matrix_mdev. It makes little sense to make the change the check for matrix_mdev->kvm there before taking the kvm->lock mutex only to have to move it out via another patch, so it is done in this patch. It is important to make note of the following: 1. The matrix_dev->guests_lock is acquired at the start of both callback functions. This ensures that matrix_mdev will not be removed via the vfio_ap_mdev_remove function because it too takes matrix_dev_guests_lock before removing the object; so, matrix_mdev will be available for the duration of the callback functions. 2. The matrix_dev->mdevs_lock mutex must be taken in order to access fields within the matrix_mdev structure 3. matrix_mdev->kvm->lock mutex must be taken before the matrix_dev->mdevs_lock to prevent a lockdep splat. 4: The kvm->lock must be held while plugging the guest's AP configuration into its SIE state description via the vfio_ap_mdev_update_guest_apcb function. 5. The vfio_ap_mdev_update_guest_apcb checks matrix_mdev->kvm to verify it is not NULL before doing the hot plug of the guest's AP configuration. Fixes: eeb386aeb5b7c ("s390/vfio-ap: handle config changed and scan complete notification") Cc: stable@vger.kernel.org Signed-off-by: Anthony Krowiak Reviewed-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- drivers/s390/crypto/vfio_ap_ops.c | 39 ++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 5a0932d011d4..1076a9982923 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -2600,8 +2600,20 @@ static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove, DECLARE_BITMAP(cdrem, AP_DOMAINS); int do_remove; + /* + * It is safe to traverse this list here because the + * required guard - matrix_dev->guests_lock - is taken in the + * vfio_ap_on_cfg_changed function prior to this function getting + * called. + */ list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { - mutex_lock(&matrix_mdev->kvm->lock); + /* + * The mdevs_lock must be held to access fields within matrix_mdev, + * and kvm->lock must be taken before mdevs_lock to satisfy the lock + * ordering requirement and prevent a lockdep splat. + */ + if (matrix_mdev->kvm) + mutex_lock(&matrix_mdev->kvm->lock); mutex_lock(&matrix_dev->mdevs_lock); do_remove = bitmap_and(aprem, ap_remove, @@ -2619,7 +2631,8 @@ static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove, cdrem); mutex_unlock(&matrix_dev->mdevs_lock); - mutex_unlock(&matrix_mdev->kvm->lock); + if (matrix_mdev->kvm) + mutex_unlock(&matrix_mdev->kvm->lock); } } @@ -2816,9 +2829,6 @@ static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) DECLARE_BITMAP(apm_filtered, AP_DEVICES); bool filter_domains, filter_adapters, filter_cdoms, do_hotplug = false; - mutex_lock(&matrix_mdev->kvm->lock); - mutex_lock(&matrix_dev->mdevs_lock); - filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm, matrix_mdev->apm_add, AP_DEVICES); filter_domains = bitmap_intersects(matrix_mdev->matrix.aqm, @@ -2836,9 +2846,6 @@ static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) vfio_ap_mdev_update_guest_apcb(matrix_mdev); reset_queues_for_apids(matrix_mdev, apm_filtered); - - mutex_unlock(&matrix_dev->mdevs_lock); - mutex_unlock(&matrix_mdev->kvm->lock); } void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info, @@ -2849,15 +2856,29 @@ void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info, mutex_lock(&matrix_dev->guests_lock); list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + /* + * The mdevs_lock must be held to access fields within matrix_mdev, + * and kvm->lock must be taken before mdevs_lock to satisfy the lock + * ordering requirement and prevent a lockdep splat. + */ + if (matrix_mdev->kvm) + mutex_lock(&matrix_mdev->kvm->lock); + mutex_lock(&matrix_dev->mdevs_lock); + if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) && bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) && bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS)) - continue; + goto do_unlock; vfio_ap_mdev_hot_plug_cfg(matrix_mdev); bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES); bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS); bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS); + +do_unlock: + mutex_unlock(&matrix_dev->mdevs_lock); + if (matrix_mdev->kvm) + mutex_unlock(&matrix_mdev->kvm->lock); } mutex_unlock(&matrix_dev->guests_lock); From 7fa61c29850d05e40ca9ed41bfdf57673023f581 Mon Sep 17 00:00:00 2001 From: Anthony Krowiak Date: Wed, 12 Aug 2026 16:02:34 -0400 Subject: [PATCH 77/82] s390/vfio-ap: Fix missing lock required to access list of ap_matrix_mdev objects In order to traverse or add/remove ap_matrix_mdev objects in the matrix_dev->mdev_list, the matrix_dev->guests_lock mutex must be held. There are two functions that access the list without holding the mutex: vfio_ap_mdev_probe function ~~~~~~~~~~~~~~~~~~~~~~~~~~~ The vfio_ap_mdev_probe function uses the matrix_dev->mdevs_lock mutex to guard the add of a newly created ap_matrix_mdev object to the matrix_dev->mdev_list. This mutex does not protect list access; its purpose is to guard against concurrent access to fields contained in an ap_matrix_mdev object. This could lead to kernel memory corruption or use-after-free if another mdev is created or removed concurrently. The adding of an ap_matrix_mdev object to matrix_dev->mdev_list is now guarded by the matrix_dev->guests_lock which is the correct way to protect against concurrent mdev_list access. Also removed the following two lines of code because the matrix_mdev is allocated via vfio_alloc_device macro which uses kzalloc, so req_trigger and cfg_chg_trigger are already zero-initialised when the struct is allocated before the call to vfio_register_emulated_iommu_dev. This prevents a window whereby these triggers are set to NULL after the device is exposed to userspace. matrix_mdev->req_trigger = NULL; matrix_mdev->cfg_chg_trigger = NULL; vfio_ap_mdev_for_queue function ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The status_show function that supports display of the status attribute of the devices in /sys/bus/ap/devices calls the vfio_ap_mdev_for_queue function which iterates the matrix_dev->mdev_list to find the object representing the queue device whose status is to be displayed. In order to traverse this list, the matrix_dev->guests_lock mutex must be held. To fix this, the guests_lock mutex is taken prior to taking the matrix_dev->mdevs_lock mutex in the status_show function. It is taken there rather than the vfio_ap_mdev_for_queue function - where it is needed - because it must be taken prior to the mdevs_lock mutex in order to adhere to the proper locking order and prevent a lockdep splat; also because the mdevs_lock is needed there to access fields within the matrix_mdev object in that function. See the vfio-ap-locking.rst in the linux kernel tree. Fixes: 2c1ee8983aa3 ("s390/vfio-ap: prepare for dynamic update of guest's APCB on queue probe/remove") Cc: stable@vger.kernel.org Signed-off-by: Anthony Krowiak Reviewed-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- drivers/s390/crypto/vfio_ap_ops.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 1076a9982923..a403d0f59122 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -807,12 +807,17 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev) ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev); if (ret) goto err_put_vdev; - matrix_mdev->req_trigger = NULL; - matrix_mdev->cfg_chg_trigger = NULL; + + /* + * Take the matrix_dev->guests_lock mutex before adding the matrix_mdev + * to the mdev_list. All functions that traverse the list must also hold + * this lock to guard against additions to or removals from the list + * while it is being traversed. + */ + mutex_lock(&matrix_dev->guests_lock); dev_set_drvdata(&mdev->dev, matrix_mdev); - mutex_lock(&matrix_dev->mdevs_lock); list_add(&matrix_mdev->node, &matrix_dev->mdev_list); - mutex_unlock(&matrix_dev->mdevs_lock); + mutex_unlock(&matrix_dev->guests_lock); return 0; err_put_vdev: @@ -2292,6 +2297,8 @@ static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q) unsigned long apid = AP_QID_CARD(q->apqn); unsigned long apqi = AP_QID_QUEUE(q->apqn); + lockdep_assert_held(&matrix_dev->guests_lock); + list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { if (test_bit_inv(apid, matrix_mdev->matrix.apm) && test_bit_inv(apqi, matrix_mdev->matrix.aqm)) @@ -2311,6 +2318,7 @@ static ssize_t status_show(struct device *dev, struct ap_matrix_mdev *matrix_mdev; struct ap_device *apdev = to_ap_dev(dev); + mutex_lock(&matrix_dev->guests_lock); mutex_lock(&matrix_dev->mdevs_lock); q = dev_get_drvdata(&apdev->device); matrix_mdev = vfio_ap_mdev_for_queue(q); @@ -2338,6 +2346,7 @@ static ssize_t status_show(struct device *dev, } mutex_unlock(&matrix_dev->mdevs_lock); + mutex_unlock(&matrix_dev->guests_lock); return nchars; } @@ -2756,6 +2765,12 @@ static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add, vfio_ap_filter_apid_by_qtype(apm_add, aqm_add); + /* + * It is safe to traverse this list here because the + * required guard - matrix_dev->guests_lock - is taken in the + * vfio_ap_on_cfg_changed function prior to this function getting + * called. + */ list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { bitmap_and(matrix_mdev->apm_add, matrix_mdev->matrix.apm, apm_add, AP_DEVICES); @@ -2815,6 +2830,10 @@ void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info, if (!cur_cfg_info || !prev_cfg_info) return; + /* + * Take the guests_lock mutex here to guard access to the + * matrix_dev->mdev_list in the two functions called below. + */ mutex_lock(&matrix_dev->guests_lock); vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info); From 5883528250be57fa92270459b33603ff52de0a91 Mon Sep 17 00:00:00 2001 From: Anthony Krowiak Date: Wed, 12 Aug 2026 16:02:35 -0400 Subject: [PATCH 78/82] s390/vfio-ap: Fix required lock not held during update of ap_matrix_mdev object In the vfio_ap_mdev_cfg_add function, the apm_add, aqm_add and adm_add fields of an ap_matrix_mdev object fields are modified while not holding the matrix_dev->mdevs_lock. This lock must be held while making these to guard against a race condition with another caller that may be concurrently modifying these fields or any of the fields in the matrix_mdev->matrix. Fixes: eeb386aeb5b7c ("s390/vfio-ap: handle config changed and scan complete notification") Cc: stable@vger.kernel.org Signed-off-by: Anthony Krowiak Reviewed-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- drivers/s390/crypto/vfio_ap_ops.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index a403d0f59122..d2f99079f619 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -2772,12 +2772,20 @@ static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add, * called. */ list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + /* + * The mdevs_lock must be held in order to access fields + * within matrix_mdev + */ + mutex_lock(&matrix_dev->mdevs_lock); + bitmap_and(matrix_mdev->apm_add, matrix_mdev->matrix.apm, apm_add, AP_DEVICES); bitmap_and(matrix_mdev->aqm_add, matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS); bitmap_and(matrix_mdev->adm_add, matrix_mdev->matrix.adm, adm_add, AP_DEVICES); + + mutex_unlock(&matrix_dev->mdevs_lock); } } From 6b8a02e216f6b520cc029e43ddc83956605135d5 Mon Sep 17 00:00:00 2001 From: Anthony Krowiak Date: Wed, 12 Aug 2026 16:02:36 -0400 Subject: [PATCH 79/82] s390/vfio-ap: Fix control domain removal in vfio_ap_mdev_cfg_remove The vfio_ap_config_remove function uses the bitmap_andnot function to clear bits from the matrix_mdev->matrix.adm bitmap (specifies the control domains assigned to the mdev). This prevents the explicitly unplugged control domains from being removed the KVM guest. The bitmap_and function is used instead. Fixes: eeb386aeb5b7c ("s390/vfio-ap: handle config changed and scan complete notification") Cc: stable@vger.kernel.org Signed-off-by: Anthony Krowiak Reviewed-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- drivers/s390/crypto/vfio_ap_ops.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index d2f99079f619..78552886612e 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -2631,9 +2631,9 @@ static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove, do_remove |= bitmap_and(aqrem, aq_remove, matrix_mdev->matrix.aqm, AP_DOMAINS); - do_remove |= bitmap_andnot(cdrem, cd_remove, - matrix_mdev->matrix.adm, - AP_DOMAINS); + do_remove |= bitmap_and(cdrem, cd_remove, + matrix_mdev->matrix.adm, + AP_DOMAINS); if (do_remove) vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem, From bf09b9d7cd7890bc3a3b7eb63d5ece15f88bfde7 Mon Sep 17 00:00:00 2001 From: Anthony Krowiak Date: Wed, 12 Aug 2026 16:02:37 -0400 Subject: [PATCH 80/82] s390/vfio-ap: fix potential use of uninitialized apm_filtered bitmap The DECLARE_BITMAP(apm_filtered, AP_DEVICES) macro allocates the bitmap on the stack without zero-initializing it. In vfio_ap_mdev_hot_plug_cfg(), the vfio_ap_mdev_filter_matrix() function is only called to initialize and populate apm_filtered if either filter_adapters or filter_domains is true. If the hot plug configuration change only adds control domains (meaning filter_cdoms is true, but filter_adapters and filter_domains are both false), vfio_ap_mdev_filter_matrix() is bypassed. Consequently, apm_filtered is passed to reset_queues_for_apids() with uninitialized stack garbage. This can cause reset_queues_for_apids() to interpret arbitrary stack garbage bits as valid APIDs to reset, potentially performing unintended guest hardware queue resets. Fix this by zero-initializing the apm_filtered bitmap at the beginning of vfio_ap_mdev_hot_plug_cfg() using bitmap_zero(). Fixes: eeb386aeb5b7c ("s390/vfio-ap: handle config changed and scan complete notification") Cc: stable@vger.kernel.org Signed-off-by: Anthony Krowiak Reviewed-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- drivers/s390/crypto/vfio_ap_ops.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 78552886612e..3b4cacb7bc47 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -2856,6 +2856,15 @@ static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) DECLARE_BITMAP(apm_filtered, AP_DEVICES); bool filter_domains, filter_adapters, filter_cdoms, do_hotplug = false; + /* + * Zero out the apm_filtered bitmap in case there are no adapters or + * domains to be added, but only control domains. In that case, + * vfio_ap_mdev_filter_matrix() - which initializes apm_filtered - will + * not get called and the reset_queues_for_apids will crash because it + * will access an uninitialized bitmap. + */ + bitmap_zero(apm_filtered, AP_DEVICES); + filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm, matrix_mdev->apm_add, AP_DEVICES); filter_domains = bitmap_intersects(matrix_mdev->matrix.aqm, From 917f509bfb88048094dbb85c4e9dbc4d6fe4a886 Mon Sep 17 00:00:00 2001 From: Anthony Krowiak Date: Wed, 12 Aug 2026 16:02:38 -0400 Subject: [PATCH 81/82] s390/vfio-ap: Fix hot-unplug skipped when last AP adapter or domain removed The vfio_ap_mdev_hot_unplug_cfg() function uses the return value of bitmap_andnot() to determine whether the guest APCB needs to be updated. However, bitmap_andnot() returns false when the resulting destination bitmap is empty. This means that if the only adapter, domain or control domain assigned to an mdev is removed from the host's AP configuration, the bit is correctly cleared from the shadow APCB, but bitmap_andnot() returns false because the result is an empty bitmap. Consequently, do_hotplug remains 0 and vfio_ap_mdev_update_guest_apcb() is never called, leaving the KVM guest with stale hardware access to the unplugged AP devices. Fix this by replacing the bitmap_andnot() return value check with bitmap_intersects() to determine whether the shadow APCB actually overlaps with the removal mask. If there is an intersection, call bitmap_andnot() solely for its side effect of clearing the bits, then unconditionally set do_hotplug to trigger the guest APCB update. Fixes: eeb386aeb5b7c ("s390/vfio-ap: handle config changed and scan complete notification") Cc: stable@vger.kernel.org Signed-off-by: Anthony Krowiak Reviewed-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- drivers/s390/crypto/vfio_ap_ops.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 3b4cacb7bc47..1546a216295b 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -2563,24 +2563,28 @@ static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev, unsigned long *aqrem, unsigned long *cdrem) { - int do_hotplug = 0; + bool do_hotplug = false; - if (!bitmap_empty(aprem, AP_DEVICES)) { - do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm, - matrix_mdev->shadow_apcb.apm, - aprem, AP_DEVICES); + if (bitmap_intersects(matrix_mdev->shadow_apcb.apm, aprem, AP_DEVICES)) { + bitmap_andnot(matrix_mdev->shadow_apcb.apm, + matrix_mdev->shadow_apcb.apm, + aprem, AP_DEVICES); + do_hotplug = true; } - if (!bitmap_empty(aqrem, AP_DOMAINS)) { - do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm, - matrix_mdev->shadow_apcb.aqm, - aqrem, AP_DEVICES); + if (bitmap_intersects(matrix_mdev->shadow_apcb.aqm, aqrem, AP_DOMAINS)) { + bitmap_andnot(matrix_mdev->shadow_apcb.aqm, + matrix_mdev->shadow_apcb.aqm, + aqrem, AP_DOMAINS); + do_hotplug = true; } - if (!bitmap_empty(cdrem, AP_DOMAINS)) - do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm, - matrix_mdev->shadow_apcb.adm, - cdrem, AP_DOMAINS); + if (bitmap_intersects(matrix_mdev->shadow_apcb.adm, cdrem, AP_DOMAINS)) { + bitmap_andnot(matrix_mdev->shadow_apcb.adm, + matrix_mdev->shadow_apcb.adm, + cdrem, AP_DOMAINS); + do_hotplug = true; + } if (do_hotplug) vfio_ap_mdev_update_guest_apcb(matrix_mdev); From dd6f4ef6f8a37412909ad787c837332fb070159c Mon Sep 17 00:00:00 2001 From: Anthony Krowiak Date: Wed, 12 Aug 2026 16:02:39 -0400 Subject: [PATCH 82/82] s390/vfio-ap: Fix NULL deref in status_show() during queue probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When vfio_ap_mdev_probe_queue() creates the sysfs attribute group, the queue's driver data has not yet been set. A concurrent read of the 'status' attribute can therefore call dev_get_drvdata() and get NULL, which is then passed directly to vfio_ap_mdev_for_queue() where q->apqn is unconditionally dereferenced, causing a NULL pointer dereference. Fix this by acquiring the update locks before calling sysfs_create_group(). The status_show() function acquires guests_lock before reading the driver data, so any concurrent read will block until after dev_set_drvdata() has been called and the update locks are released. As a bonus, the APQN no longer needs to be read from the queue struct after allocation — it can be read directly from apdev before allocation and stored in a local variable, which is then assigned to q->apqn once the allocation succeeds. Fixes: 260f3ea141382 ("s390/vfio-ap: move probe and remove callbacks to vfio_ap_ops.c") Cc: stable@vger.kernel.org Signed-off-by: Anthony Krowiak Reviewed-by: Matthew Rosato Signed-off-by: Christian Borntraeger --- drivers/s390/crypto/vfio_ap_ops.c | 33 +++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 1546a216295b..940c0ff668be 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -2321,6 +2321,23 @@ static ssize_t status_show(struct device *dev, mutex_lock(&matrix_dev->guests_lock); mutex_lock(&matrix_dev->mdevs_lock); q = dev_get_drvdata(&apdev->device); + + /* + * Make sure the drvdata has been set before proceeding. There is a + * possibility that the drvdata was not set if the vfio_ap_queue object + * could not be allocated when the queue device was probed. In that case, + * the locks used in vfio_ap_mdev_probe_queue() are released prior to + * removing the sysfs status attribute to avoid a lockdep + * splat. That opens a very small window where the status attribute is + * still available without the vfio_ap_queue object having been + * stored in the device drvdata. In that case, indicate the queue is not + * assigned. + */ + if (!q) { + nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_UNASSIGNED); + goto done; + } + matrix_mdev = vfio_ap_mdev_for_queue(q); /* If the queue is assigned to the matrix mediated device, then @@ -2345,6 +2362,7 @@ static ssize_t status_show(struct device *dev, nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_UNASSIGNED); } +done: mutex_unlock(&matrix_dev->mdevs_lock); mutex_unlock(&matrix_dev->guests_lock); @@ -2419,14 +2437,17 @@ void vfio_ap_mdev_unregister(void) int vfio_ap_mdev_probe_queue(struct ap_device *apdev) { - int ret; + int ret, apqn; struct vfio_ap_queue *q; DECLARE_BITMAP(apm_filtered, AP_DEVICES); struct ap_matrix_mdev *matrix_mdev; + apqn = to_ap_queue(&apdev->device)->qid; + matrix_mdev = get_update_locks_by_apqn(apqn); + ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group); if (ret) - return ret; + goto err_release_locks; q = kzalloc_obj(*q); if (!q) { @@ -2434,11 +2455,10 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) goto err_remove_group; } - q->apqn = to_ap_queue(&apdev->device)->qid; + q->apqn = apqn; q->saved_isc = VFIO_AP_ISC_INVALID; memset(&q->reset_status, 0, sizeof(q->reset_status)); INIT_WORK(&q->reset_work, apq_reset_check); - matrix_mdev = get_update_locks_by_apqn(q->apqn); if (matrix_mdev) { vfio_ap_mdev_link_queue(matrix_mdev, q); @@ -2467,8 +2487,13 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) return ret; err_remove_group: + release_update_locks_for_mdev(matrix_mdev); sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); return ret; + +err_release_locks: + release_update_locks_for_mdev(matrix_mdev); + return ret; } void vfio_ap_mdev_remove_queue(struct ap_device *apdev)