diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 78a6f743137e..c7577b608f3e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -157,7 +157,6 @@ struct amdgpu_watchdog_timer { * Modules parameters. */ extern int amdgpu_modeset; -extern int amdgpu_iommu_perfopt; extern unsigned int amdgpu_vram_limit; extern int amdgpu_vis_vram_limit; extern int amdgpu_gart_size; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 53f05ee7193f..c65bc575a57a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include @@ -3770,28 +3769,6 @@ amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px) apple_gmux_detect(NULL, NULL))); } -static inline bool amdgpu_device_identity(struct amdgpu_device *adev) -{ - struct pci_dev *pdev = adev->pdev; - struct iommu_domain *domain = iommu_get_domain_for_dev(&pdev->dev); - - if (!domain) - return false; - - return domain->type == IOMMU_DOMAIN_IDENTITY; -} - -static bool amdgpu_device_use_perfopt(struct amdgpu_device *adev) -{ - if (amdgpu_iommu_perfopt == 0) - return false; - - if (!(adev->flags & AMD_IS_APU)) - return false; - - return amdgpu_device_identity(adev); -} - /** * amdgpu_device_init - initialize the driver * @@ -4001,16 +3978,6 @@ int amdgpu_device_init(struct amdgpu_device *adev, if (r) return r; - if (amdgpu_device_use_perfopt(adev)) { - int perfopt_ret = amd_iommu_enable_perfopt(pdev); - - /* Optional optimization; a failure to arm it must not abort probe. */ - if (perfopt_ret) - dev_warn(adev->dev, - "Failed to enable IOMMU PerfOpt (%d); continuing without it\n", - perfopt_ret); - } - /* * No need to remove conflicting FBs for non-display class devices. * This prevents the sysfb from being freed accidently. @@ -4380,9 +4347,6 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) amdgpu_gart_dummy_page_fini(adev); - if (amdgpu_device_use_perfopt(adev)) - amd_iommu_disable_perfopt(adev->pdev); - if (pci_dev_is_disconnected(adev->pdev)) amdgpu_device_unmap_mmio(adev); @@ -4746,20 +4710,6 @@ int amdgpu_device_resume(struct drm_device *dev, bool notify_clients) if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; - if (amdgpu_device_use_perfopt(adev)) { - int perfopt_ret = amd_iommu_enable_perfopt(adev->pdev); - - /* - * Must not return on failure: a bare return would leak the - * SR-IOV VF exclusive-mode acquisition taken above (released - * via the exit: path). - */ - if (perfopt_ret) - dev_warn(adev->dev, - "Failed to enable IOMMU PerfOpt (%d); continuing without it\n", - perfopt_ret); - } - if (adev->in_s0ix) amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 731fe8f9e922..3e5634de077f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -188,7 +188,6 @@ char *amdgpu_disable_cu; char *amdgpu_virtual_display; int amdgpu_enforce_isolation = -1; int amdgpu_modeset = -1; -int amdgpu_iommu_perfopt = -1; /* Specifies the default granularity for SVM, used in buffer * migration and restoration of backing memory when handling @@ -399,17 +398,6 @@ module_param_named(fw_load_type, amdgpu_fw_load_type, int, 0444); MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)"); module_param_named(aspm, amdgpu_aspm, int, 0444); -/** - * DOC: iommu_perfopt (int) - * Control the AMD IOMMU PerfOpt DMA-latency optimization - * (0 = disable; -1 = enable on supported devices). - * This arms the IOMMU PerfOpt control (IOMMU spec, MMIO Offset 016Ch, EFR PerfOptSup / PerfOptEn) - * Arming it disables ATS, PRI, PASID and SVA for the GPU and removes IOMMU DMA containment for it, - * trading isolation for lower DMA latency. - */ -MODULE_PARM_DESC(iommu_perfopt, "Control IOMMU PerfOpt DMA-latency optimization (-1 = enable on supported devices, 0 = disable)"); -module_param_named(iommu_perfopt, amdgpu_iommu_perfopt, int, 0444); - /** * DOC: runpm (int) * Override for runtime power management control for dGPUs. The amdgpu driver can dynamically power down diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 93845b3c3fbf..29355810db7c 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -48,8 +48,7 @@ extern u8 amd_iommu_hpt_vasize; extern unsigned long amd_iommu_pgsize_bitmap; extern bool amd_iommu_hatdis; -int amd_iommu_perfopt_clear(struct amd_iommu *iommu); -int amd_iommu_perfopt_restore(struct amd_iommu *iommu); +int amd_iommu_perfopt_set(struct amd_iommu *iommu, bool enable); /* Protection domain ops */ void amd_iommu_init_identity_domain(void); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 0e79c43be001..f98770db0090 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -660,8 +660,8 @@ struct amd_iommu { /* Extended features 2 */ u64 features2; - /* Devices requesting PerfOpt; the shared PERF_OPT_EN bit is on while >0. Protected by @lock. */ - int perfopt_refcount; + /* Saved PerfOpt activation state, protected by @lock. */ + bool perfopt_enabled; /* PCI device id of the IOMMU device */ u16 devid; @@ -824,7 +824,7 @@ struct iommu_dev_data { u8 ppr :1; /* Enable device PPR support */ bool use_vapic; /* Enable device to use vapic mode */ bool defer_attach; - bool perfopt; + bool perfopt; /* Device is eligible for PerfOpt */ struct ratelimit_state rs; /* Ratelimit IOPF messages */ }; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index de6db5621194..3dd4ff145e55 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -432,8 +432,41 @@ static void iommu_enable(struct amd_iommu *iommu) iommu_feature_enable(iommu, CONTROL_IOMMU_EN); } +/* + * Program or clear the per-IOMMU PerfOpt enable bit. + * Gate on this IOMMU's own EFR, not the global AND-mask. + * Caller serializes access using iommu->lock or during init/suspend/resume. + */ +int amd_iommu_perfopt_set(struct amd_iommu *iommu, bool enable) +{ + u32 old, val, readback; + + if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT)) + return enable ? -ENODEV : 0; + + old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + if (old == U32_MAX) + goto fail; + + val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN; + if (val != old) + writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); + if (readback == U32_MAX || + (readback & PERF_OPT_EN) != (val & PERF_OPT_EN)) + goto fail; + + iommu->perfopt_enabled = enable; + return 0; +fail: + pr_err("IOMMU%d: failed to set PerfOpt to %d\n", iommu->index, enable); + return -EIO; +} + static void iommu_disable(struct amd_iommu *iommu) { + bool perfopt_enabled = iommu->perfopt_enabled; + if (!iommu->mmio_base) return; @@ -457,6 +490,10 @@ static void iommu_disable(struct amd_iommu *iommu) /* Clear IRTE cache disabling bit */ iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS); + + /* Clear hardware while preserving the activation state for resume. */ + amd_iommu_perfopt_set(iommu, false); + iommu->perfopt_enabled = perfopt_enabled; } /* @@ -1888,6 +1925,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, struct acpi_table_header *ivrs_base) { struct amd_iommu_pci_seg *pci_seg; + int ret; pci_seg = get_pci_segment(h->pci_seg, ivrs_base); if (pci_seg == NULL) @@ -1960,8 +1998,10 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, if (!iommu->mmio_base) return -ENOMEM; - if (amd_iommu_perfopt_clear(iommu)) - pr_err("IOMMU%d: failed to clear PerfOpt\n", iommu->index); + /* A crash kernel may reuse device tables with PerfOpt still enabled. */ + ret = amd_iommu_perfopt_set(iommu, false); + if (ret) + return ret; return init_iommu_from_acpi(iommu, h); } @@ -2331,6 +2371,9 @@ static void print_iommu_info(void) if (check_feature2(FEATURE_SEVSNPIO_SUP)) pr_cont(" SEV-TIO"); + if (check_feature(FEATURE_PERF_OPT)) + pr_cont(" PerfOpt"); + pr_cont("\n"); } @@ -2733,6 +2776,15 @@ static int __init init_unity_map_range(struct ivmd_header *m, return 0; } +static bool ivrs_block_type_ivmd(u8 type) +{ + if (type == ACPI_IVMD_TYPE || + type == ACPI_IVMD_TYPE_ALL || + type == ACPI_IVMD_TYPE_RANGE) + return true; + return false; +} + /* iterates over all memory definitions we find in the ACPI table */ static int __init init_memory_definitions(struct acpi_table_header *table) { @@ -2744,10 +2796,11 @@ static int __init init_memory_definitions(struct acpi_table_header *table) while (p < end) { m = (struct ivmd_header *)p; + p += m->length; + if (!ivrs_block_type_ivmd(m->type)) + continue; if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE)) init_unity_map_range(m, table); - - p += m->length; } return 0; @@ -3067,46 +3120,10 @@ static void enable_iommus_vapic(void) #endif } -static int clear_perfopt_all(void) -{ - struct amd_iommu *iommu; - int err, ret = 0; - - for_each_iommu(iommu) { - err = amd_iommu_perfopt_clear(iommu); - if (err) - ret = err; - } - - return ret; -} - -static int restore_perfopt_all(void) -{ - struct amd_iommu *iommu; - int err, ret = 0; - - for_each_iommu(iommu) { - err = amd_iommu_perfopt_restore(iommu); - if (err) - ret = err; - } - - return ret; -} - static void disable_iommus(void) { struct amd_iommu *iommu; - /* - * PerfOpt is an optional performance bit, so a failure to clear it must - * not skip the mandatory disable below. This also runs from the void - * amd_iommu_disable() shutdown/kexec path, which cannot report an error. - */ - if (clear_perfopt_all()) - pr_err("Failed to clear PerfOpt while disabling IOMMUs\n"); - for_each_iommu(iommu) iommu_disable(iommu); @@ -3132,9 +3149,11 @@ static void amd_iommu_resume(void *data) for_each_iommu(iommu) early_enable_iommu(iommu); - /* early_enable_iommu() cleared PERF_OPT_EN; re-assert it from the refcount. */ - if (restore_perfopt_all()) - pr_err("Failed to restore PerfOpt after IOMMU resume\n"); + /* Restore the activation state saved at suspend. */ + for_each_iommu(iommu) { + if (iommu->perfopt_enabled) + amd_iommu_perfopt_set(iommu, true); + } iommu_enable_event_buffer(); amd_iommu_enable_interrupts(); @@ -3505,6 +3524,7 @@ static __init void iommu_snp_enable(void) return; disable_snp: + amd_iommu_snp_en = false; cc_platform_clear(CC_ATTR_HOST_SEV_SNP); #endif } diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 960e1dc05ec6..41ee4ad1dcf8 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -487,6 +488,10 @@ static void setup_aliases(struct amd_iommu *iommu, struct device *dev) clone_aliases(iommu, dev); } +/* + * We keep dev_data around for unplugged devices and reuse it when the + * device is re-plugged - not doing so would introduce a ton of races. + */ static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid) { struct iommu_dev_data *dev_data; @@ -2368,7 +2373,7 @@ static int attach_device(struct device *dev, if (ret) goto out; - if (dev_data->perfopt) + if (dev_data->perfopt && pdom_is_in_pt_mode(domain)) goto skip_caps; /* Setup GCR3 table */ @@ -2510,192 +2515,6 @@ static int iommu_init_device_caps(struct iommu_dev_data *dev_data, return 0; } -/* Program the per-IOMMU PerfOpt enable bit. Caller must hold iommu->lock. */ -static int __perfopt_write(struct amd_iommu *iommu, bool enable) -{ - u32 old, val, readback; - - if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT)) - return enable ? -ENODEV : 0; - - old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); - if (old == U32_MAX) - return -EIO; - - val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN; - if (val != old) - writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET); - readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET); - if (readback == U32_MAX || - (readback & PERF_OPT_EN) != (val & PERF_OPT_EN)) - return -EIO; - return 0; -} - -/* - * PERF_OPT_EN is a single bit shared by every device behind @iommu, so it is - * reference counted: armed on the first requesting device, cleared on the last. - */ -static int perfopt_get(struct amd_iommu *iommu) -{ - unsigned long flags; - int ret = 0; - - if (!iommu->mmio_base) - return 0; - - raw_spin_lock_irqsave(&iommu->lock, flags); - if (iommu->perfopt_refcount == 0) { - ret = __perfopt_write(iommu, true); - if (ret) - goto out; - } - iommu->perfopt_refcount++; -out: - raw_spin_unlock_irqrestore(&iommu->lock, flags); - return ret; -} - -static int perfopt_put(struct amd_iommu *iommu) -{ - unsigned long flags; - int ret = 0; - - if (!iommu->mmio_base) - return 0; - - raw_spin_lock_irqsave(&iommu->lock, flags); - if (iommu->perfopt_refcount > 0 && --iommu->perfopt_refcount == 0) - ret = __perfopt_write(iommu, false); - raw_spin_unlock_irqrestore(&iommu->lock, flags); - return ret; -} - -/* - * Force PERF_OPT_EN off without touching the refcount (used on init, shutdown, - * and suspend). The count is preserved so amd_iommu_perfopt_restore() can - * re-arm on resume. - */ -int amd_iommu_perfopt_clear(struct amd_iommu *iommu) -{ - unsigned long flags; - int ret; - - if (!iommu->mmio_base) - return 0; - - raw_spin_lock_irqsave(&iommu->lock, flags); - ret = __perfopt_write(iommu, false); - raw_spin_unlock_irqrestore(&iommu->lock, flags); - return ret; -} - -/* - * Re-assert PERF_OPT_EN from the refcount after the hardware was reprogrammed on - * resume, so devices armed before suspend keep the optimization without each - * consumer driver re-arming. - */ -int amd_iommu_perfopt_restore(struct amd_iommu *iommu) -{ - unsigned long flags; - int ret; - - if (!iommu->mmio_base) - return 0; - - raw_spin_lock_irqsave(&iommu->lock, flags); - ret = __perfopt_write(iommu, iommu->perfopt_refcount > 0); - raw_spin_unlock_irqrestore(&iommu->lock, flags); - return ret; -} - -int amd_iommu_enable_perfopt(struct pci_dev *pdev) -{ - struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); - struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); - struct protection_domain *domain; - int ret; - - if (!iommu || !dev_data) - return -ENODEV; - - if (!(iommu->features & FEATURE_PERF_OPT)) - return -ENODEV; - - domain = dev_data->domain; - if (!domain) - return -ENODEV; - - /* Already armed for this device (e.g. re-entry on resume). */ - if (dev_data->perfopt) - return 0; - - /* - * The bit is only architecturally valid while the device is untranslated: - * identity domain with ATS/PRI/PASID off. The identity domain is - * SVA-capable so attach_device() enabled ATS/PRI/PASID and built a GCR3 - * table. Re-home the device onto the same identity domain with - * perfopt set, so the attach_device() skip_caps path leaves - * ATS/PRI/PASID off and no GCR3 table. This follows the detach/attach - * pattern used by amd_iommu_attach_device(). - * - * Locking: this and amd_iommu_disable_perfopt() run only from the - * consumer driver's bind/unbind path. group->mutex is not exposed to - * drivers, but a device bound to its native driver cannot have its domain - * changed concurrently by the core (VFIO ownership is mutually exclusive; - * sysfs domain changes require an unused group), so the detach/attach pair - * is serialized without it. - */ - dev_data->perfopt = true; - detach_device(&pdev->dev); - ret = attach_device(&pdev->dev, domain); - if (ret) - goto err_restore; - - ret = perfopt_get(iommu); - if (ret) - goto err_rearm; - - dev_info_once(&pdev->dev, "PerfOpt armed on IOMMU%d\n", iommu->index); - return 0; - -err_rearm: - detach_device(&pdev->dev); -err_restore: - dev_data->perfopt = false; - if (attach_device(&pdev->dev, domain)) - pci_err(pdev, "failed to restore state after PerfOpt setup; device left detached\n"); - dev_err_once(&pdev->dev, "PerfOpt failed to arm on IOMMU%d (%d)\n", - iommu->index, ret); - return ret; -} -EXPORT_SYMBOL_GPL(amd_iommu_enable_perfopt); - -void amd_iommu_disable_perfopt(struct pci_dev *pdev) -{ - struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); - struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); - struct protection_domain *domain; - - if (!iommu || !dev_data || !dev_data->perfopt || !dev_data->domain) - return; - - if (WARN_ON(perfopt_put(iommu))) - pci_err(pdev, "failed to clear PerfOpt\n"); - - /* - * Restore ATS/PRI/PASID (and thus SVA) by re-homing the device onto its - * identity domain with the flag cleared, so a later bind without PerfOpt - * sees a normally-capable device. See the locking note in - * amd_iommu_enable_perfopt(). - */ - domain = dev_data->domain; - dev_data->perfopt = false; - detach_device(&pdev->dev); - if (attach_device(&pdev->dev, domain)) - pci_err(pdev, "failed to restore caps after PerfOpt disable\n"); -} -EXPORT_SYMBOL_GPL(amd_iommu_disable_perfopt); static struct iommu_device *amd_iommu_probe_device(struct device *dev) { struct iommu_device *iommu_dev; @@ -2736,26 +2555,6 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) return iommu_dev; } -static void amd_iommu_release_device(struct device *dev) -{ - struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); - struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); - - if (dev_data->perfopt) { - if (WARN_ON(perfopt_put(iommu))) - dev_err(dev, "IOMMU%d: failed to clear PerfOpt on release\n", - iommu->index); - dev_data->perfopt = false; - } - - WARN_ON(dev_data->domain); - - /* - * We keep dev_data around for unplugged devices and reuse it when the - * device is re-plugged - not doing so would introduce a ton of races. - */ -} - static struct iommu_group *amd_iommu_device_group(struct device *dev) { if (dev_is_pci(dev)) @@ -3116,24 +2915,32 @@ void amd_iommu_domain_free(struct iommu_domain *dom) kfree(domain); } +/* Each IOMMU has at most one PerfOpt-eligible GPU. */ +static int set_device_perfopt(struct iommu_dev_data *dev_data, bool enable) +{ + struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + + if (!dev_data->perfopt) + return 0; + + guard(raw_spinlock_irqsave)(&iommu->lock); + if (iommu->perfopt_enabled == enable) + return 0; + + return amd_iommu_perfopt_set(iommu, enable); +} + static int blocked_domain_attach_device(struct iommu_domain *domain, struct device *dev, struct iommu_domain *old) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); - struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); + int ret; - /* - * blocked_domain is also the .release_domain, so this is the normal - * teardown path: drop the reference and clear the flag here too, and - * don't fail teardown if the WARN-guarded write doesn't stick. - */ - if (dev_data->perfopt) { - if (WARN_ON(perfopt_put(iommu))) - dev_err(dev, "IOMMU%d: failed to clear PerfOpt for blocked domain\n", - iommu->index); - dev_data->perfopt = false; - } + /* Clearing the DTE cannot block DMA while PerfOpt bypasses it. */ + ret = set_device_perfopt(dev_data, false); + if (ret) + return ret; if (dev_data->domain) detach_device(dev); @@ -3167,6 +2974,9 @@ static struct protection_domain identity_domain; static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *dev, struct iommu_domain *old) { + struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + int ret; + /* * Don't allow attaching a device to the identity domain if SNP is * enabled and SNP Mode0 support is not present. @@ -3174,7 +2984,13 @@ static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *de if (amd_iommu_snp_en && !amd_iommu_snp_mode0_sup) return -EINVAL; - return amd_iommu_attach_device(dom, dev, old); + ret = amd_iommu_attach_device(dom, dev, old); + if (ret) + return ret; + + /* PerfOpt is optional; the identity attachment is already complete. */ + set_device_perfopt(dev_data, true); + return 0; } static const struct iommu_domain_ops identity_domain_ops = { @@ -3202,9 +3018,6 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev, struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); int ret; - if (dev_data->perfopt && !pdom_is_in_pt_mode(domain)) - return -EBUSY; - /* * Skip attach device to domain if new domain is same as * devices current domain @@ -3221,6 +3034,12 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev, if (dom->dirty_ops && !amd_iommu_hd_support(iommu)) return -EINVAL; + if (!pdom_is_in_pt_mode(domain)) { + ret = set_device_perfopt(dev_data, false); + if (ret) + return ret; + } + if (dev_data->domain) detach_device(dev); @@ -3372,6 +3191,7 @@ static bool amd_iommu_is_attach_deferred(struct device *dev) static bool quirks_force_identity_mapping(struct pci_dev *pdev) { + struct pci_dev *bridge; int class = pdev->class >> 8; /* AMD GPU vendor ID */ @@ -3383,7 +3203,8 @@ static bool quirks_force_identity_mapping(struct pci_dev *pdev) class != PCI_CLASS_DISPLAY_OTHER) return false; - if (pci_upstream_bridge(pdev)->vendor == PCI_VENDOR_ID_ATI) + bridge = pci_upstream_bridge(pdev); + if (bridge && bridge->vendor == PCI_VENDOR_ID_ATI) return false; /* It is the GPU in an APU, force identity domain */ @@ -3415,6 +3236,9 @@ static int amd_iommu_def_domain_type(struct device *dev) if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) return 0; + dev_data->perfopt = get_amd_iommu_from_dev_data(dev_data)->features & + FEATURE_PERF_OPT; + return IOMMU_DOMAIN_IDENTITY; } @@ -3436,7 +3260,6 @@ const struct iommu_ops amd_iommu_ops = { .domain_alloc_paging_flags = amd_iommu_domain_alloc_paging_flags, .domain_alloc_sva = amd_iommu_domain_alloc_sva, .probe_device = amd_iommu_probe_device, - .release_device = amd_iommu_release_device, .device_group = amd_iommu_device_group, .get_resv_regions = amd_iommu_get_resv_regions, .is_attach_deferred = amd_iommu_is_attach_deferred, diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h index 104817e8d897..5c9dcf720f84 100644 --- a/include/linux/amd-iommu.h +++ b/include/linux/amd-iommu.h @@ -76,15 +76,4 @@ static inline int amd_iommu_snp_disable(void) { return 0; } static inline bool amd_iommu_sev_tio_supported(void) { return false; } #endif -#ifdef CONFIG_AMD_IOMMU -int amd_iommu_enable_perfopt(struct pci_dev *pdev); -void amd_iommu_disable_perfopt(struct pci_dev *pdev); -#else -static inline int amd_iommu_enable_perfopt(struct pci_dev *pdev) -{ - return 0; -} -static inline void amd_iommu_disable_perfopt(struct pci_dev *pdev) { } -#endif - #endif /* _ASM_X86_AMD_IOMMU_H */