From b32ecac779bba6e7ac7caeae8c40db4280941634 Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Thu, 5 Nov 2026 16:32:15 +0000 Subject: [PATCH 1/5] iommu/amd: Enable PerfOpt automatically for integrated GPUs PerfOpt is usable by integrated GPUs in identity mode. Enable it from the IOMMU driver's domain attachment paths instead of requiring an amdgpu policy knob and exported enable/disable interfaces. Mark supported integrated GPUs as PerfOpt-eligible when selecting their default identity domain. Skip GCR3 setup and ATS/PRI/PASID enablement for these identity attachments, then enable PerfOpt after installing the DTE. Clear PerfOpt before attaching an eligible GPU to a translated or blocked domain, including unmanaged domains used by VFIO. Propagate failures to clear bypass, but keep a completed identity attachment successful if the optional PerfOpt enable fails. Clear inherited PerfOpt state during initialization, including when a crash kernel reuses device tables. Read feature support directly from hardware because cached features may not be available yet, and stop initialization if the initial clear fails. Each IOMMU has at most one PerfOpt-eligible GPU, so track activation with a boolean under the IOMMU lock. Preserve this state while clearing hardware at suspend and shutdown, and restore the bit on resume only where it was active. Do not enable PerfOpt unconditionally during early IOMMU initialization. Remove the amdgpu parameter, driver callbacks, and exported interfaces. Translated attachments retain their normal device capability setup. Suggested-by: Jason Gunthorpe Signed-off-by: Mario Limonciello (AMD) Acked-by: Alex Deucher Signed-off-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 - drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 50 ---- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 12 - drivers/iommu/amd/amd_iommu.h | 3 +- drivers/iommu/amd/amd_iommu_types.h | 6 +- drivers/iommu/amd/init.c | 91 ++++---- drivers/iommu/amd/iommu.c | 252 ++++----------------- include/linux/amd-iommu.h | 11 - 8 files changed, 95 insertions(+), 331 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 78a6f743137e..c7577b608f3e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -157,7 +157,6 @@ struct amdgpu_watchdog_timer { * Modules parameters. */ extern int amdgpu_modeset; -extern int amdgpu_iommu_perfopt; extern unsigned int amdgpu_vram_limit; extern int amdgpu_vis_vram_limit; extern int amdgpu_gart_size; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 53f05ee7193f..c65bc575a57a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include @@ -3770,28 +3769,6 @@ amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px) apple_gmux_detect(NULL, NULL))); } -static inline bool amdgpu_device_identity(struct amdgpu_device *adev) -{ - struct pci_dev *pdev = adev->pdev; - struct iommu_domain *domain = iommu_get_domain_for_dev(&pdev->dev); - - if (!domain) - return false; - - return domain->type == IOMMU_DOMAIN_IDENTITY; -} - -static bool amdgpu_device_use_perfopt(struct amdgpu_device *adev) -{ - if (amdgpu_iommu_perfopt == 0) - return false; - - if (!(adev->flags & AMD_IS_APU)) - return false; - - return amdgpu_device_identity(adev); -} - /** * amdgpu_device_init - initialize the driver * @@ -4001,16 +3978,6 @@ int amdgpu_device_init(struct amdgpu_device *adev, if (r) return r; - if (amdgpu_device_use_perfopt(adev)) { - int perfopt_ret = amd_iommu_enable_perfopt(pdev); - - /* Optional optimization; a failure to arm it must not abort probe. */ - if (perfopt_ret) - dev_warn(adev->dev, - "Failed to enable IOMMU PerfOpt (%d); continuing without it\n", - perfopt_ret); - } - /* * No need to remove conflicting FBs for non-display class devices. * This prevents the sysfb from being freed accidently. @@ -4380,9 +4347,6 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_gart_dummy_page_fini(adev); - if (amdgpu_device_use_perfopt(adev)) - amd_iommu_disable_perfopt(adev->pdev); - if (pci_dev_is_disconnected(adev->pdev)) amdgpu_device_unmap_mmio(adev); @@ -4746,20 +4710,6 @@ int amdgpu_device_resume(struct drm_device *dev, bool notify_clients) if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - if (amdgpu_device_use_perfopt(adev)) { - int perfopt_ret = amd_iommu_enable_perfopt(adev->pdev); - - /* - * Must not return on failure: a bare return would leak the - * SR-IOV VF exclusive-mode acquisition taken above (released - * via the exit: path). - */ - if (perfopt_ret) - dev_warn(adev->dev, - "Failed to enable IOMMU PerfOpt (%d); continuing without it\n", - perfopt_ret); - } - if (adev->in_s0ix) amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 731fe8f9e922..3e5634de077f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -188,7 +188,6 @@ char *amdgpu_disable_cu; char *amdgpu_virtual_display; int amdgpu_enforce_isolation = -1; int amdgpu_modeset = -1; -int amdgpu_iommu_perfopt = -1; /* Specifies the default granularity for SVM, used in buffer * migration and restoration of backing memory when handling @@ -399,17 +398,6 @@ module_param_named(fw_load_type, amdgpu_fw_load_type, int, 0444); MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)"); module_param_named(aspm, amdgpu_aspm, int, 0444); -/** - * DOC: iommu_perfopt (int) - * Control the AMD IOMMU PerfOpt DMA-latency optimization - * (0 = disable; -1 = enable on supported devices). - * This arms the IOMMU PerfOpt control (IOMMU spec, MMIO Offset 016Ch, EFR PerfOptSup / PerfOptEn) - * Arming it disables ATS, PRI, PASID and SVA for the GPU and removes IOMMU DMA containment for it, - * trading isolation for lower DMA latency. - */ -MODULE_PARM_DESC(iommu_perfopt, "Control IOMMU PerfOpt DMA-latency optimization (-1 = enable on supported devices, 0 = disable)"); -module_param_named(iommu_perfopt, amdgpu_iommu_perfopt, int, 0444); - /** * DOC: runpm (int) * Override for runtime power management control for dGPUs. The amdgpu driver can dynamically power down diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 93845b3c3fbf..29355810db7c 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -48,8 +48,7 @@ extern u8 amd_iommu_hpt_vasize; extern unsigned long amd_iommu_pgsize_bitmap; extern bool amd_iommu_hatdis; -int amd_iommu_perfopt_clear(struct amd_iommu *iommu); -int amd_iommu_perfopt_restore(struct amd_iommu *iommu); +int amd_iommu_perfopt_set(struct amd_iommu *iommu, bool enable); /* Protection domain ops */ void amd_iommu_init_identity_domain(void); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 0e79c43be001..f98770db0090 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -660,8 +660,8 @@ struct amd_iommu { /* Extended features 2 */ u64 features2; - /* Devices requesting PerfOpt; the shared PERF_OPT_EN bit is on while >0. Protected by @lock. */ - int perfopt_refcount; + /* Saved PerfOpt activation state, protected by @lock. */ + bool perfopt_enabled; /* PCI device id of the IOMMU device */ u16 devid; @@ -824,7 +824,7 @@ struct iommu_dev_data { u8 ppr :1; /* Enable device PPR support */ bool use_vapic; /* Enable device to use vapic mode */ bool defer_attach; - bool perfopt; + bool perfopt; /* Device is eligible for PerfOpt */ struct ratelimit_state rs; /* Ratelimit IOPF messages */ }; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index de6db5621194..0cb9aeb0597b 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -432,8 +432,41 @@ static void iommu_enable(struct amd_iommu *iommu) iommu_feature_enable(iommu, CONTROL_IOMMU_EN); } +/* + * Program or clear the per-IOMMU PerfOpt enable bit. + * Gate on this IOMMU's own EFR, not the global AND-mask. + * Caller serializes access using iommu->lock or during init/suspend/resume. + */ +int amd_iommu_perfopt_set(struct amd_iommu *iommu, bool enable) +{ + u32 old, val, readback; + + if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT)) + return enable ? -ENODEV : 0; + + old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + if (old == U32_MAX) + goto fail; + + val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN; + if (val != old) + writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + if (readback == U32_MAX || + (readback & PERF_OPT_EN) != (val & PERF_OPT_EN)) + goto fail; + + iommu->perfopt_enabled = enable; + return 0; +fail: + pr_err("IOMMU%d: failed to set PerfOpt to %d\n", iommu->index, enable); + return -EIO; +} + static void iommu_disable(struct amd_iommu *iommu) { + bool perfopt_enabled = iommu->perfopt_enabled; + if (!iommu->mmio_base) return; @@ -457,6 +490,10 @@ static void iommu_disable(struct amd_iommu *iommu) /* Clear IRTE cache disabling bit */ iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS); + + /* Clear hardware while preserving the activation state for resume. */ + amd_iommu_perfopt_set(iommu, false); + iommu->perfopt_enabled = perfopt_enabled; } /* @@ -1888,6 +1925,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, struct acpi_table_header *ivrs_base) { struct amd_iommu_pci_seg *pci_seg; + int ret; pci_seg = get_pci_segment(h->pci_seg, ivrs_base); if (pci_seg == NULL) @@ -1960,8 +1998,10 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, if (!iommu->mmio_base) return -ENOMEM; - if (amd_iommu_perfopt_clear(iommu)) - pr_err("IOMMU%d: failed to clear PerfOpt\n", iommu->index); + /* A crash kernel may reuse device tables with PerfOpt still enabled. */ + ret = amd_iommu_perfopt_set(iommu, false); + if (ret) + return ret; return init_iommu_from_acpi(iommu, h); } @@ -2331,6 +2371,9 @@ static void print_iommu_info(void) if (check_feature2(FEATURE_SEVSNPIO_SUP)) pr_cont(" SEV-TIO"); + if (check_feature(FEATURE_PERF_OPT)) + pr_cont(" PerfOpt"); + pr_cont("\n"); } @@ -3067,46 +3110,10 @@ static void enable_iommus_vapic(void) #endif } -static int clear_perfopt_all(void) -{ - struct amd_iommu *iommu; - int err, ret = 0; - - for_each_iommu(iommu) { - err = amd_iommu_perfopt_clear(iommu); - if (err) - ret = err; - } - - return ret; -} - -static int restore_perfopt_all(void) -{ - struct amd_iommu *iommu; - int err, ret = 0; - - for_each_iommu(iommu) { - err = amd_iommu_perfopt_restore(iommu); - if (err) - ret = err; - } - - return ret; -} - static void disable_iommus(void) { struct amd_iommu *iommu; - /* - * PerfOpt is an optional performance bit, so a failure to clear it must - * not skip the mandatory disable below. This also runs from the void - * amd_iommu_disable() shutdown/kexec path, which cannot report an error. - */ - if (clear_perfopt_all()) - pr_err("Failed to clear PerfOpt while disabling IOMMUs\n"); - for_each_iommu(iommu) iommu_disable(iommu); @@ -3132,9 +3139,11 @@ static void amd_iommu_resume(void *data) for_each_iommu(iommu) early_enable_iommu(iommu); - /* early_enable_iommu() cleared PERF_OPT_EN; re-assert it from the refcount. */ - if (restore_perfopt_all()) - pr_err("Failed to restore PerfOpt after IOMMU resume\n"); + /* Restore the activation state saved at suspend. */ + for_each_iommu(iommu) { + if (iommu->perfopt_enabled) + amd_iommu_perfopt_set(iommu, true); + } iommu_enable_event_buffer(); amd_iommu_enable_interrupts(); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 960e1dc05ec6..54465e041155 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -2368,7 +2369,7 @@ static int attach_device(struct device *dev, if (ret) goto out; - if (dev_data->perfopt) + if (dev_data->perfopt && pdom_is_in_pt_mode(domain)) goto skip_caps; /* Setup GCR3 table */ @@ -2510,192 +2511,6 @@ static int iommu_init_device_caps(struct iommu_dev_data *dev_data, return 0; } -/* Program the per-IOMMU PerfOpt enable bit. Caller must hold iommu->lock. */ -static int __perfopt_write(struct amd_iommu *iommu, bool enable) -{ - u32 old, val, readback; - - if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT)) - return enable ? -ENODEV : 0; - - old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); - if (old == U32_MAX) - return -EIO; - - val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN; - if (val != old) - writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET); - readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); - if (readback == U32_MAX || - (readback & PERF_OPT_EN) != (val & PERF_OPT_EN)) - return -EIO; - return 0; -} - -/* - * PERF_OPT_EN is a single bit shared by every device behind @iommu, so it is - * reference counted: armed on the first requesting device, cleared on the last. - */ -static int perfopt_get(struct amd_iommu *iommu) -{ - unsigned long flags; - int ret = 0; - - if (!iommu->mmio_base) - return 0; - - raw_spin_lock_irqsave(&iommu->lock, flags); - if (iommu->perfopt_refcount == 0) { - ret = __perfopt_write(iommu, true); - if (ret) - goto out; - } - iommu->perfopt_refcount++; -out: - raw_spin_unlock_irqrestore(&iommu->lock, flags); - return ret; -} - -static int perfopt_put(struct amd_iommu *iommu) -{ - unsigned long flags; - int ret = 0; - - if (!iommu->mmio_base) - return 0; - - raw_spin_lock_irqsave(&iommu->lock, flags); - if (iommu->perfopt_refcount > 0 && --iommu->perfopt_refcount == 0) - ret = __perfopt_write(iommu, false); - raw_spin_unlock_irqrestore(&iommu->lock, flags); - return ret; -} - -/* - * Force PERF_OPT_EN off without touching the refcount (used on init, shutdown, - * and suspend). The count is preserved so amd_iommu_perfopt_restore() can - * re-arm on resume. - */ -int amd_iommu_perfopt_clear(struct amd_iommu *iommu) -{ - unsigned long flags; - int ret; - - if (!iommu->mmio_base) - return 0; - - raw_spin_lock_irqsave(&iommu->lock, flags); - ret = __perfopt_write(iommu, false); - raw_spin_unlock_irqrestore(&iommu->lock, flags); - return ret; -} - -/* - * Re-assert PERF_OPT_EN from the refcount after the hardware was reprogrammed on - * resume, so devices armed before suspend keep the optimization without each - * consumer driver re-arming. - */ -int amd_iommu_perfopt_restore(struct amd_iommu *iommu) -{ - unsigned long flags; - int ret; - - if (!iommu->mmio_base) - return 0; - - raw_spin_lock_irqsave(&iommu->lock, flags); - ret = __perfopt_write(iommu, iommu->perfopt_refcount > 0); - raw_spin_unlock_irqrestore(&iommu->lock, flags); - return ret; -} - -int amd_iommu_enable_perfopt(struct pci_dev *pdev) -{ - struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); - struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); - struct protection_domain *domain; - int ret; - - if (!iommu || !dev_data) - return -ENODEV; - - if (!(iommu->features & FEATURE_PERF_OPT)) - return -ENODEV; - - domain = dev_data->domain; - if (!domain) - return -ENODEV; - - /* Already armed for this device (e.g. re-entry on resume). */ - if (dev_data->perfopt) - return 0; - - /* - * The bit is only architecturally valid while the device is untranslated: - * identity domain with ATS/PRI/PASID off. The identity domain is - * SVA-capable so attach_device() enabled ATS/PRI/PASID and built a GCR3 - * table. Re-home the device onto the same identity domain with - * perfopt set, so the attach_device() skip_caps path leaves - * ATS/PRI/PASID off and no GCR3 table. This follows the detach/attach - * pattern used by amd_iommu_attach_device(). - * - * Locking: this and amd_iommu_disable_perfopt() run only from the - * consumer driver's bind/unbind path. group->mutex is not exposed to - * drivers, but a device bound to its native driver cannot have its domain - * changed concurrently by the core (VFIO ownership is mutually exclusive; - * sysfs domain changes require an unused group), so the detach/attach pair - * is serialized without it. - */ - dev_data->perfopt = true; - detach_device(&pdev->dev); - ret = attach_device(&pdev->dev, domain); - if (ret) - goto err_restore; - - ret = perfopt_get(iommu); - if (ret) - goto err_rearm; - - dev_info_once(&pdev->dev, "PerfOpt armed on IOMMU%d\n", iommu->index); - return 0; - -err_rearm: - detach_device(&pdev->dev); -err_restore: - dev_data->perfopt = false; - if (attach_device(&pdev->dev, domain)) - pci_err(pdev, "failed to restore state after PerfOpt setup; device left detached\n"); - dev_err_once(&pdev->dev, "PerfOpt failed to arm on IOMMU%d (%d)\n", - iommu->index, ret); - return ret; -} -EXPORT_SYMBOL_GPL(amd_iommu_enable_perfopt); - -void amd_iommu_disable_perfopt(struct pci_dev *pdev) -{ - struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); - struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); - struct protection_domain *domain; - - if (!iommu || !dev_data || !dev_data->perfopt || !dev_data->domain) - return; - - if (WARN_ON(perfopt_put(iommu))) - pci_err(pdev, "failed to clear PerfOpt\n"); - - /* - * Restore ATS/PRI/PASID (and thus SVA) by re-homing the device onto its - * identity domain with the flag cleared, so a later bind without PerfOpt - * sees a normally-capable device. See the locking note in - * amd_iommu_enable_perfopt(). - */ - domain = dev_data->domain; - dev_data->perfopt = false; - detach_device(&pdev->dev); - if (attach_device(&pdev->dev, domain)) - pci_err(pdev, "failed to restore caps after PerfOpt disable\n"); -} -EXPORT_SYMBOL_GPL(amd_iommu_disable_perfopt); static struct iommu_device *amd_iommu_probe_device(struct device *dev) { struct iommu_device *iommu_dev; @@ -2739,14 +2554,6 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) static void amd_iommu_release_device(struct device *dev) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); - struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); - - if (dev_data->perfopt) { - if (WARN_ON(perfopt_put(iommu))) - dev_err(dev, "IOMMU%d: failed to clear PerfOpt on release\n", - iommu->index); - dev_data->perfopt = false; - } WARN_ON(dev_data->domain); @@ -3116,24 +2923,32 @@ void amd_iommu_domain_free(struct iommu_domain *dom) kfree(domain); } +/* Each IOMMU has at most one PerfOpt-eligible GPU. */ +static int set_device_perfopt(struct iommu_dev_data *dev_data, bool enable) +{ + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + + if (!dev_data->perfopt) + return 0; + + guard(raw_spinlock_irqsave)(&iommu->lock); + if (iommu->perfopt_enabled == enable) + return 0; + + return amd_iommu_perfopt_set(iommu, enable); +} + static int blocked_domain_attach_device(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); - struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + int ret; - /* - * blocked_domain is also the .release_domain, so this is the normal - * teardown path: drop the reference and clear the flag here too, and - * don't fail teardown if the WARN-guarded write doesn't stick. - */ - if (dev_data->perfopt) { - if (WARN_ON(perfopt_put(iommu))) - dev_err(dev, "IOMMU%d: failed to clear PerfOpt for blocked domain\n", - iommu->index); - dev_data->perfopt = false; - } + /* Clearing the DTE cannot block DMA while PerfOpt bypasses it. */ + ret = set_device_perfopt(dev_data, false); + if (ret) + return ret; if (dev_data->domain) detach_device(dev); @@ -3167,6 +2982,9 @@ static struct protection_domain identity_domain; static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *dev, struct iommu_domain *old) { + struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + int ret; + /* * Don't allow attaching a device to the identity domain if SNP is * enabled and SNP Mode0 support is not present. @@ -3174,7 +2992,13 @@ static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *de if (amd_iommu_snp_en && !amd_iommu_snp_mode0_sup) return -EINVAL; - return amd_iommu_attach_device(dom, dev, old); + ret = amd_iommu_attach_device(dom, dev, old); + if (ret) + return ret; + + /* PerfOpt is optional; the identity attachment is already complete. */ + set_device_perfopt(dev_data, true); + return 0; } static const struct iommu_domain_ops identity_domain_ops = { @@ -3202,9 +3026,6 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev, struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); int ret; - if (dev_data->perfopt && !pdom_is_in_pt_mode(domain)) - return -EBUSY; - /* * Skip attach device to domain if new domain is same as * devices current domain @@ -3221,6 +3042,12 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev, if (dom->dirty_ops && !amd_iommu_hd_support(iommu)) return -EINVAL; + if (!pdom_is_in_pt_mode(domain)) { + ret = set_device_perfopt(dev_data, false); + if (ret) + return ret; + } + if (dev_data->domain) detach_device(dev); @@ -3415,6 +3242,9 @@ static int amd_iommu_def_domain_type(struct device *dev) if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) return 0; + dev_data->perfopt = get_amd_iommu_from_dev_data(dev_data)->features & + FEATURE_PERF_OPT; + return IOMMU_DOMAIN_IDENTITY; } diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h index 104817e8d897..5c9dcf720f84 100644 --- a/include/linux/amd-iommu.h +++ b/include/linux/amd-iommu.h @@ -76,15 +76,4 @@ static inline int amd_iommu_snp_disable(void) { return 0; } static inline bool amd_iommu_sev_tio_supported(void) { return false; } #endif -#ifdef CONFIG_AMD_IOMMU -int amd_iommu_enable_perfopt(struct pci_dev *pdev); -void amd_iommu_disable_perfopt(struct pci_dev *pdev); -#else -static inline int amd_iommu_enable_perfopt(struct pci_dev *pdev) -{ - return 0; -} -static inline void amd_iommu_disable_perfopt(struct pci_dev *pdev) { } -#endif - #endif /* _ASM_X86_AMD_IOMMU_H */ From ec91bb159dc24eefe144ef8867937f4a055dd0e2 Mon Sep 17 00:00:00 2001 From: Dheeraj Kumar Srivastava Date: Thu, 5 Nov 2026 16:32:16 +0000 Subject: [PATCH 2/5] iommu/amd: Clear amd_iommu_snp_en if snp_rmptable_init() fails iommu_snp_enable() checks that the IOMMU can support SNP and sets amd_iommu_snp_en, then calls snp_rmptable_init() to map the RMP table. If that call fails, CC_ATTR_HOST_SEV_SNP is cleared but amd_iommu_snp_en is left set. Later IOMMU initialization uses amd_iommu_snp_en to configure the IOMMU for SNP, on the assumption that RMP setup succeeded. Clear amd_iommu_snp_en along with CC_ATTR_HOST_SEV_SNP. Fixes: 409f45387c93 ("x86/sev: Fix broken SNP support with KVM module built-in") Signed-off-by: Dheeraj Kumar Srivastava Reviewed-by: Ankit Soni Signed-off-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 0cb9aeb0597b..78c7079dec9d 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -3514,6 +3514,7 @@ static __init void iommu_snp_enable(void) return; disable_snp: + amd_iommu_snp_en = false; cc_platform_clear(CC_ATTR_HOST_SEV_SNP); #endif } From e62df37e0eff4d670e78bbd415be2c1675883c15 Mon Sep 17 00:00:00 2001 From: Akshay Jindal Date: Thu, 5 Nov 2026 16:32:17 +0000 Subject: [PATCH 3/5] iommu/amd: Skip non-IVMD entries when parsing IVRS memory definitions init_memory_definitions() walks every top-level subtable including the IVHD blocks (type 0x10/0x11/0x40) in the IVRS ACPI table. It interprets each one as a 'struct ivmd_header', solely on m->flags, without first checking that the subtable's type actually is one of the IVMD types (ACPI_IVMD_TYPE, ACPI_IVMD_TYPE_ALL, ACPI_IVMD_TYPE_RANGE). if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE)) init_unity_map_range(m, table); IVMD_FLAG_UNITY_MAP (0x01) occupies the same bit position as the IVHD flags field's HtTunEn bit. Unlike real hardware, QEMU's AMD IOMMU emulation sets HtTunEn unconditionally (flags=0xD1 for type 0x10, 0x11 for type 0x11), so every QEMU-emitted IVHD block is misidentified as a unity-map/exclusion-range IVMD entry, which subsequently results in the creation of a bogus pci segment. Eg: AMD-Vi: PCI segment : 0x4000, last bdf : 0x0000 Fix by checking the subtable's type before treating it as IVMD. Fixes: be2a022c0dd0 ("x86, AMD IOMMU: add functions to parse IOMMU memory mapping requirements for devices") Cc: stable@vger.kernel.org Signed-off-by: Akshay Jindal Signed-off-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 78c7079dec9d..3dd4ff145e55 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2776,6 +2776,15 @@ static int __init init_unity_map_range(struct ivmd_header *m, return 0; } +static bool ivrs_block_type_ivmd(u8 type) +{ + if (type == ACPI_IVMD_TYPE || + type == ACPI_IVMD_TYPE_ALL || + type == ACPI_IVMD_TYPE_RANGE) + return true; + return false; +} + /* iterates over all memory definitions we find in the ACPI table */ static int __init init_memory_definitions(struct acpi_table_header *table) { @@ -2787,10 +2796,11 @@ static int __init init_memory_definitions(struct acpi_table_header *table) while (p < end) { m = (struct ivmd_header *)p; + p += m->length; + if (!ivrs_block_type_ivmd(m->type)) + continue; if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE)) init_unity_map_range(m, table); - - p += m->length; } return 0; From fa77b023eb80cdb297ff32f6b13786e483a81ca8 Mon Sep 17 00:00:00 2001 From: Kevin Mitchell Date: Thu, 5 Nov 2026 16:32:18 +0000 Subject: [PATCH 4/5] iommu/amd: remove obsolete domain pointer warning on release A warning is triggered in amd_iommu_release_device() on removal of a device with an IOMMU_RESV_DIRECT region. The warning is based on the false assumption that correct attachment to the release_domain will always result in a NULL dev_data->domain pointer. For devices with IOMMU_RESV_DIRECT (and consequently dev->iommu->require_direct), the release domain is the permanent identity_domain which sets itself to dev_data->domain on attach. This behaviour was added in commit e94160488e65 ("iommu: Generic support for RMRs during device release"). Commit c21b34762e2f ("iommu/amd: Set release_domain to blocked_domain") caused it to take effect for AMD IOMMU. Remove the warning as it is no longer strictly an error to have a non-NULL domain pointer. As this leaves amd_iommu_release_device() empty, remove it. The warning was observed for a device with direct mapped region: /sys/bus/pci/devices/0000:04:00.0/iommu_group/reserved_regions: 0x00000000fee00000 0x00000000feefffff msi 0x000000fd00000000 0x000000ffffffffff reserved 0x0000fffd00000000 0x0000ffffffffffff direct Upon hotunplug: [ 28.715475] pcieport 0000:00:03.1: pciehp: Slot(0-3): Link Down [ 28.715482] pcieport 0000:00:03.1: pciehp: Slot(0-3): Card not present [ 28.715601] ------------[ cut here ]------------ [ 28.715603] WARNING: drivers/iommu/amd/iommu.c:2521 at amd_iommu_release_device+0x5d/0x70, CPU#4: irq/29-pciehp/99 [ 28.715657] RIP: 0010:amd_iommu_release_device+0x5d/0x70 [ 28.715687] Call Trace: [ 28.715689] [ 28.715694] iommu_deinit_device+0x84/0x110 [ 28.715699] __iommu_group_remove_device+0xa8/0xd0 [ 28.715703] iommu_bus_notifier+0x48/0x70 [ 28.715707] notifier_call_chain+0x48/0xe0 [ 28.715712] blocking_notifier_call_chain+0x45/0x60 [ 28.715716] bus_notify+0x3b/0x60 [ 28.715720] device_del+0x259/0x3b0 [ 28.715730] pci_remove_bus_device+0x85/0x100 [ 28.715735] pciehp_unconfigure_device+0x9d/0x180 [ 28.715743] __pciehp_disable_slot+0x47/0xe0 [ 28.715747] pciehp_handle_presence_or_link_change+0x80/0x5e0 [ 28.715753] pciehp_ist+0x162/0x290 Fixes: c21b34762e2f ("iommu/amd: Set release_domain to blocked_domain") Cc: stable@vger.kernel.org Signed-off-by: Kevin Mitchell Reviewed-by: Jason Gunthorpe Signed-off-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 54465e041155..91981c8a9614 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -488,6 +488,10 @@ static void setup_aliases(struct amd_iommu *iommu, struct device *dev) clone_aliases(iommu, dev); } +/* + * We keep dev_data around for unplugged devices and reuse it when the + * device is re-plugged - not doing so would introduce a ton of races. + */ static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid) { struct iommu_dev_data *dev_data; @@ -2551,18 +2555,6 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) return iommu_dev; } -static void amd_iommu_release_device(struct device *dev) -{ - struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); - - WARN_ON(dev_data->domain); - - /* - * We keep dev_data around for unplugged devices and reuse it when the - * device is re-plugged - not doing so would introduce a ton of races. - */ -} - static struct iommu_group *amd_iommu_device_group(struct device *dev) { if (dev_is_pci(dev)) @@ -3266,7 +3258,6 @@ const struct iommu_ops amd_iommu_ops = { .domain_alloc_paging_flags = amd_iommu_domain_alloc_paging_flags, .domain_alloc_sva = amd_iommu_domain_alloc_sva, .probe_device = amd_iommu_probe_device, - .release_device = amd_iommu_release_device, .device_group = amd_iommu_device_group, .get_resv_regions = amd_iommu_get_resv_regions, .is_attach_deferred = amd_iommu_is_attach_deferred, From 2cf4be57b301ed2d7af82e70cff5dfd96894d8de Mon Sep 17 00:00:00 2001 From: Jean-Christophe Guillain Date: Thu, 5 Nov 2026 16:32:19 +0000 Subject: [PATCH 5/5] iommu/amd: Fix NULL pointer dereference for GPUs without upstream bridge pci_upstream_bridge() can return NULL for devices without an upstream bridge. The AMD GPU identity mapping quirk dereferences the returned pointer unconditionally, causing a kernel panic during boot. Check the pointer before accessing the bridge vendor ID. This fixes a boot regression on a Lenovo system with an AMD PRO A10-9700E APU. The system now boots with AMD IOMMU enabled. Fixes: bf5bdfd1f8d4 ("iommu/amd: Force identity mode for selected GPUs only") Signed-off-by: Jean-Christophe Guillain Signed-off-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 91981c8a9614..41ee4ad1dcf8 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -3191,6 +3191,7 @@ static bool amd_iommu_is_attach_deferred(struct device *dev) static bool quirks_force_identity_mapping(struct pci_dev *pdev) { + struct pci_dev *bridge; int class = pdev->class >> 8; /* AMD GPU vendor ID */ @@ -3202,7 +3203,8 @@ static bool quirks_force_identity_mapping(struct pci_dev *pdev) class != PCI_CLASS_DISPLAY_OTHER) return false; - if (pci_upstream_bridge(pdev)->vendor == PCI_VENDOR_ID_ATI) + bridge = pci_upstream_bridge(pdev); + if (bridge && bridge->vendor == PCI_VENDOR_ID_ATI) return false; /* It is the GPU in an APU, force identity domain */