github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

IOMMU Fixes for Linux v7.4-rc1

Browse Source 
Including:
 
   - AMD IOMMU:
     - Move PerfOpt management into the IOMMU domain attachment paths, enabling
       it automatically for supported integrated GPUs in identity mode and
       clearing it before switching to translated or blocked domains.
 
     - Fix a boot panic in the GPU identity-mapping quirk when a GPU has no
       upstream PCI bridge.
 
     - Clear the IOMMU SNP enable flag when RMP table initialization fails,
       preventing subsequent initialization from assuming SNP setup succeeded.
 
     - Check IVRS subtable types before parsing memory definitions, avoiding
       bogus PCI segments caused by misinterpreting IVHD entries under QEMU.
 
     - Remove an obsolete warning on device release that incorrectly fires for
       devices required to remain attached to an identity domain.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEr9jSbILcajRFYWYyK/BELZcBGuMFAmrt9NsACgkQK/BELZcB
 GuMRKBAAhj43RcJXi8OXkXdwt13M3dUVGRfQ5YyKdTUgwWl4qtRoXpFrlnlFh3FB
 +WmWn4BwpRZ1eKQ9FfPRv6auJ8ItfHDJxwPGRDADYgYuNmuPN2eWhzfOLkRiLCd8
 a1sYTqtTAZfmyPJ3FevmCJeYifR+v6k7wkGLHC+316s/2JiVKo9QC8XUldR197Z2
 0O/B1kfV6vX1zWfBVJXTM2lTMn5LEBRkAeOXmZIWY2uFCYWr0B0HuUhPegfAMbCT
 5qkQStnrkvjlDPj5uI8pi4TdTvbewyz7LwAw0g5qRHSZm2CCUR+av4mkXkp4Ud6O
 d6+8hiTwWzuclz0+iEeaYyd124P1m3uZTVo6wqkz4PUsqlOKXwRMg49y9W4mT2NX
 exYBcL6dZKISntEoBk9P/Xr7MQ7UXHL5Kg+mojiBBSAs/irRTXymGmBBuhHhCIw+
 IDbgc0PY1+C60knS0s/EIj1RbHppS6foCQwJH6GaJLWKqe9KM6QSib+vJYRT9EOj
 K6S/Ys8f3XqCMus0BmxTDIqfI3OPAeMEqUpP2vyHpkv54iTnU3ouVeUkrFa0jV65
 0Iaq+EE5uUqYwvNxAEiuoTK+t6trAQddeBafCTi2QJEwsSPqjw33YzFOUCnwMsYP
 PFL6bpUwr6JCJosnU+qk2nWYZKSxwCEKTSdsMFJ9Zw+niV4Pyqc=
 =B1KC
 -----END PGP SIGNATURE-----

Merge tag 'iommu-fixes-v7.4-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux

Pull iommu fixes from Joerg Roedel:
 "AMD IOMMU:

   - Move PerfOpt management into the IOMMU domain attachment paths,
     enabling it automatically for supported integrated GPUs in identity
     mode and clearing it before switching to translated or blocked
     domains

   - Fix a boot panic in the GPU identity-mapping quirk when a GPU has
     no upstream PCI bridge

   - Clear the IOMMU SNP enable flag when RMP table initialization
     fails, preventing subsequent initialization from assuming SNP setup
     succeeded

   - Check IVRS subtable types before parsing memory definitions,
     avoiding bogus PCI segments caused by misinterpreting IVHD entries
     under QEMU

   - Remove an obsolete warning on device release that incorrectly fires
     for devices required to remain attached to an identity domain"

* tag 'iommu-fixes-v7.4-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux:
  iommu/amd: Fix NULL pointer dereference for GPUs without upstream bridge
  iommu/amd: remove obsolete domain pointer warning on release
  iommu/amd: Skip non-IVMD entries when parsing IVRS memory definitions
  iommu/amd: Clear amd_iommu_snp_en if snp_rmptable_init() fails
  iommu/amd: Enable PerfOpt automatically for integrated GPUs
This commit is contained in:
Linus Torvalds 2026-11-06 13:02:02 -08:00
commit a7963b12ca
8 changed files with 115 additions and 347 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/gpu/drm/amd/amdgpu/amdgpu.h
View File

@ -157,7 +157,6 @@ struct amdgpu_watchdog_timer {
* Modules parameters.
*/
extern int amdgpu_modeset;
extern int amdgpu_iommu_perfopt;
extern unsigned int amdgpu_vram_limit;
extern int amdgpu_vis_vram_limit;
extern int amdgpu_gart_size;

50
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
View File

@ -33,7 +33,6 @@
#include <linux/console.h>
#include <linux/slab.h>
#include <linux/iommu.h>
#include <linux/amd-iommu.h>
#include <linux/pci.h>
#include <linux/pci-p2pdma.h>
#include <linux/apple-gmux.h>
@ -3770,28 +3769,6 @@ amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px)
apple_gmux_detect(NULL, NULL)));
}
static inline bool amdgpu_device_identity(struct amdgpu_device *adev)
{
struct pci_dev *pdev = adev->pdev;
struct iommu_domain *domain = iommu_get_domain_for_dev(&pdev->dev);
if (!domain)
return false;
return domain->type == IOMMU_DOMAIN_IDENTITY;
}
static bool amdgpu_device_use_perfopt(struct amdgpu_device *adev)
{
if (amdgpu_iommu_perfopt == 0)
return false;
if (!(adev->flags & AMD_IS_APU))
return false;
return amdgpu_device_identity(adev);
}
/**
* amdgpu_device_init - initialize the driver
*
@ -4001,16 +3978,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
if (r)
return r;
if (amdgpu_device_use_perfopt(adev)) {
int perfopt_ret = amd_iommu_enable_perfopt(pdev);
/* Optional optimization; a failure to arm it must not abort probe. */
if (perfopt_ret)
dev_warn(adev->dev,
"Failed to enable IOMMU PerfOpt (%d); continuing without it\n",
perfopt_ret);
}
/*
* No need to remove conflicting FBs for non-display class devices.
* This prevents the sysfb from being freed accidently.
@ -4380,9 +4347,6 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
amdgpu_gart_dummy_page_fini(adev);
if (amdgpu_device_use_perfopt(adev))
amd_iommu_disable_perfopt(adev->pdev);
if (pci_dev_is_disconnected(adev->pdev))
amdgpu_device_unmap_mmio(adev);
@ -4746,20 +4710,6 @@ int amdgpu_device_resume(struct drm_device *dev, bool notify_clients)
if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
return 0;
if (amdgpu_device_use_perfopt(adev)) {
int perfopt_ret = amd_iommu_enable_perfopt(adev->pdev);
/*
* Must not return on failure: a bare return would leak the
* SR-IOV VF exclusive-mode acquisition taken above (released
* via the exit: path).
*/
if (perfopt_ret)
dev_warn(adev->dev,
"Failed to enable IOMMU PerfOpt (%d); continuing without it\n",
perfopt_ret);
}
if (adev->in_s0ix)
amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);

12
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
View File

@ -188,7 +188,6 @@ char *amdgpu_disable_cu;
char *amdgpu_virtual_display;
int amdgpu_enforce_isolation = -1;
int amdgpu_modeset = -1;
int amdgpu_iommu_perfopt = -1;
/* Specifies the default granularity for SVM, used in buffer
* migration and restoration of backing memory when handling
@ -399,17 +398,6 @@ module_param_named(fw_load_type, amdgpu_fw_load_type, int, 0444);
MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)");
module_param_named(aspm, amdgpu_aspm, int, 0444);
/**
* DOC: iommu_perfopt (int)
* Control the AMD IOMMU PerfOpt DMA-latency optimization
* (0 = disable; -1 = enable on supported devices).
* This arms the IOMMU PerfOpt control (IOMMU spec, MMIO Offset 016Ch, EFR PerfOptSup / PerfOptEn)
* Arming it disables ATS, PRI, PASID and SVA for the GPU and removes IOMMU DMA containment for it,
* trading isolation for lower DMA latency.
*/
MODULE_PARM_DESC(iommu_perfopt, "Control IOMMU PerfOpt DMA-latency optimization (-1 = enable on supported devices, 0 = disable)");
module_param_named(iommu_perfopt, amdgpu_iommu_perfopt, int, 0444);
/**
* DOC: runpm (int)
* Override for runtime power management control for dGPUs. The amdgpu driver can dynamically power down

3
drivers/iommu/amd/amd_iommu.h
View File

@ -48,8 +48,7 @@ extern u8 amd_iommu_hpt_vasize;
extern unsigned long amd_iommu_pgsize_bitmap;
extern bool amd_iommu_hatdis;
int amd_iommu_perfopt_clear(struct amd_iommu *iommu);
int amd_iommu_perfopt_restore(struct amd_iommu *iommu);
int amd_iommu_perfopt_set(struct amd_iommu *iommu, bool enable);
/* Protection domain ops */
void amd_iommu_init_identity_domain(void);

6
drivers/iommu/amd/amd_iommu_types.h
View File

@ -660,8 +660,8 @@ struct amd_iommu {
/* Extended features 2 */
u64 features2;
/* Devices requesting PerfOpt; the shared PERF_OPT_EN bit is on while >0. Protected by @lock. */
int perfopt_refcount;
/* Saved PerfOpt activation state, protected by @lock. */
bool perfopt_enabled;
/* PCI device id of the IOMMU device */
u16 devid;
@ -824,7 +824,7 @@ struct iommu_dev_data {
u8 ppr :1; /* Enable device PPR support */
bool use_vapic; /* Enable device to use vapic mode */
bool defer_attach;
bool perfopt;
bool perfopt; /* Device is eligible for PerfOpt */
struct ratelimit_state rs; /* Ratelimit IOPF messages */
};

106
drivers/iommu/amd/init.c
View File

@ -432,8 +432,41 @@ static void iommu_enable(struct amd_iommu *iommu)
iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
}
/*
* Program or clear the per-IOMMU PerfOpt enable bit.
* Gate on this IOMMU's own EFR, not the global AND-mask.
* Caller serializes access using iommu->lock or during init/suspend/resume.
*/
int amd_iommu_perfopt_set(struct amd_iommu *iommu, bool enable)
{
u32 old, val, readback;
if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT))
return enable ? -ENODEV : 0;
old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
if (old == U32_MAX)
goto fail;
val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN;
if (val != old)
writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
if (readback == U32_MAX ||
(readback & PERF_OPT_EN) != (val & PERF_OPT_EN))
goto fail;
iommu->perfopt_enabled = enable;
return 0;
fail:
pr_err("IOMMU%d: failed to set PerfOpt to %d\n", iommu->index, enable);
return -EIO;
}
static void iommu_disable(struct amd_iommu *iommu)
{
bool perfopt_enabled = iommu->perfopt_enabled;
if (!iommu->mmio_base)
return;
@ -457,6 +490,10 @@ static void iommu_disable(struct amd_iommu *iommu)
/* Clear IRTE cache disabling bit */
iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);
/* Clear hardware while preserving the activation state for resume. */
amd_iommu_perfopt_set(iommu, false);
iommu->perfopt_enabled = perfopt_enabled;
}
/*
@ -1888,6 +1925,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
struct acpi_table_header *ivrs_base)
{
struct amd_iommu_pci_seg *pci_seg;
int ret;
pci_seg = get_pci_segment(h->pci_seg, ivrs_base);
if (pci_seg == NULL)
@ -1960,8 +1998,10 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
if (!iommu->mmio_base)
return -ENOMEM;
if (amd_iommu_perfopt_clear(iommu))
pr_err("IOMMU%d: failed to clear PerfOpt\n", iommu->index);
/* A crash kernel may reuse device tables with PerfOpt still enabled. */
ret = amd_iommu_perfopt_set(iommu, false);
if (ret)
return ret;
return init_iommu_from_acpi(iommu, h);
}
@ -2331,6 +2371,9 @@ static void print_iommu_info(void)
if (check_feature2(FEATURE_SEVSNPIO_SUP))
pr_cont(" SEV-TIO");
if (check_feature(FEATURE_PERF_OPT))
pr_cont(" PerfOpt");
pr_cont("\n");
}
@ -2733,6 +2776,15 @@ static int __init init_unity_map_range(struct ivmd_header *m,
return 0;
}
static bool ivrs_block_type_ivmd(u8 type)
{
if (type == ACPI_IVMD_TYPE ||
type == ACPI_IVMD_TYPE_ALL ||
type == ACPI_IVMD_TYPE_RANGE)
return true;
return false;
}
/* iterates over all memory definitions we find in the ACPI table */
static int __init init_memory_definitions(struct acpi_table_header *table)
{
@ -2744,10 +2796,11 @@ static int __init init_memory_definitions(struct acpi_table_header *table)
while (p < end) {
m = (struct ivmd_header *)p;
p += m->length;
if (!ivrs_block_type_ivmd(m->type))
continue;
if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE))
init_unity_map_range(m, table);
p += m->length;
}
return 0;
@ -3067,46 +3120,10 @@ static void enable_iommus_vapic(void)
#endif
}
static int clear_perfopt_all(void)
{
struct amd_iommu *iommu;
int err, ret = 0;
for_each_iommu(iommu) {
err = amd_iommu_perfopt_clear(iommu);
if (err)
ret = err;
}
return ret;
}
static int restore_perfopt_all(void)
{
struct amd_iommu *iommu;
int err, ret = 0;
for_each_iommu(iommu) {
err = amd_iommu_perfopt_restore(iommu);
if (err)
ret = err;
}
return ret;
}
static void disable_iommus(void)
{
struct amd_iommu *iommu;
/*
* PerfOpt is an optional performance bit, so a failure to clear it must
* not skip the mandatory disable below. This also runs from the void
* amd_iommu_disable() shutdown/kexec path, which cannot report an error.
*/
if (clear_perfopt_all())
pr_err("Failed to clear PerfOpt while disabling IOMMUs\n");
for_each_iommu(iommu)
iommu_disable(iommu);
@ -3132,9 +3149,11 @@ static void amd_iommu_resume(void *data)
for_each_iommu(iommu)
early_enable_iommu(iommu);
/* early_enable_iommu() cleared PERF_OPT_EN; re-assert it from the refcount. */
if (restore_perfopt_all())
pr_err("Failed to restore PerfOpt after IOMMU resume\n");
/* Restore the activation state saved at suspend. */
for_each_iommu(iommu) {
if (iommu->perfopt_enabled)
amd_iommu_perfopt_set(iommu, true);
}
iommu_enable_event_buffer();
amd_iommu_enable_interrupts();
@ -3505,6 +3524,7 @@ static __init void iommu_snp_enable(void)
return;
disable_snp:
amd_iommu_snp_en = false;
cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
#endif
}

273
drivers/iommu/amd/iommu.c
View File

@ -11,6 +11,7 @@
#include <linux/ratelimit.h>
#include <linux/pci.h>
#include <linux/acpi.h>
#include <linux/cleanup.h>
#include <linux/pci-ats.h>
#include <linux/bitmap.h>
#include <linux/slab.h>
@ -487,6 +488,10 @@ static void setup_aliases(struct amd_iommu *iommu, struct device *dev)
clone_aliases(iommu, dev);
}
/*
* We keep dev_data around for unplugged devices and reuse it when the
* device is re-plugged - not doing so would introduce a ton of races.
*/
static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid)
{
struct iommu_dev_data *dev_data;
@ -2368,7 +2373,7 @@ static int attach_device(struct device *dev,
if (ret)
goto out;
if (dev_data->perfopt)
if (dev_data->perfopt && pdom_is_in_pt_mode(domain))
goto skip_caps;
/* Setup GCR3 table */
@ -2510,192 +2515,6 @@ static int iommu_init_device_caps(struct iommu_dev_data *dev_data,
return 0;
}
/* Program the per-IOMMU PerfOpt enable bit. Caller must hold iommu->lock. */
static int __perfopt_write(struct amd_iommu *iommu, bool enable)
{
u32 old, val, readback;
if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT))
return enable ? -ENODEV : 0;
old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
if (old == U32_MAX)
return -EIO;
val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN;
if (val != old)
writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
if (readback == U32_MAX ||
(readback & PERF_OPT_EN) != (val & PERF_OPT_EN))
return -EIO;
return 0;
}
/*
* PERF_OPT_EN is a single bit shared by every device behind @iommu, so it is
* reference counted: armed on the first requesting device, cleared on the last.
*/
static int perfopt_get(struct amd_iommu *iommu)
{
unsigned long flags;
int ret = 0;
if (!iommu->mmio_base)
return 0;
raw_spin_lock_irqsave(&iommu->lock, flags);
if (iommu->perfopt_refcount == 0) {
ret = __perfopt_write(iommu, true);
if (ret)
goto out;
}
iommu->perfopt_refcount++;
out:
raw_spin_unlock_irqrestore(&iommu->lock, flags);
return ret;
}
static int perfopt_put(struct amd_iommu *iommu)
{
unsigned long flags;
int ret = 0;
if (!iommu->mmio_base)
return 0;
raw_spin_lock_irqsave(&iommu->lock, flags);
if (iommu->perfopt_refcount > 0 && --iommu->perfopt_refcount == 0)
ret = __perfopt_write(iommu, false);
raw_spin_unlock_irqrestore(&iommu->lock, flags);
return ret;
}
/*
* Force PERF_OPT_EN off without touching the refcount (used on init, shutdown,
* and suspend). The count is preserved so amd_iommu_perfopt_restore() can
* re-arm on resume.
*/
int amd_iommu_perfopt_clear(struct amd_iommu *iommu)
{
unsigned long flags;
int ret;
if (!iommu->mmio_base)
return 0;
raw_spin_lock_irqsave(&iommu->lock, flags);
ret = __perfopt_write(iommu, false);
raw_spin_unlock_irqrestore(&iommu->lock, flags);
return ret;
}
/*
* Re-assert PERF_OPT_EN from the refcount after the hardware was reprogrammed on
* resume, so devices armed before suspend keep the optimization without each
* consumer driver re-arming.
*/
int amd_iommu_perfopt_restore(struct amd_iommu *iommu)
{
unsigned long flags;
int ret;
if (!iommu->mmio_base)
return 0;
raw_spin_lock_irqsave(&iommu->lock, flags);
ret = __perfopt_write(iommu, iommu->perfopt_refcount > 0);
raw_spin_unlock_irqrestore(&iommu->lock, flags);
return ret;
}
int amd_iommu_enable_perfopt(struct pci_dev *pdev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);
struct protection_domain *domain;
int ret;
if (!iommu || !dev_data)
return -ENODEV;
if (!(iommu->features & FEATURE_PERF_OPT))
return -ENODEV;
domain = dev_data->domain;
if (!domain)
return -ENODEV;
/* Already armed for this device (e.g. re-entry on resume). */
if (dev_data->perfopt)
return 0;
/*
* The bit is only architecturally valid while the device is untranslated:
* identity domain with ATS/PRI/PASID off. The identity domain is
* SVA-capable so attach_device() enabled ATS/PRI/PASID and built a GCR3
* table. Re-home the device onto the same identity domain with
* perfopt set, so the attach_device() skip_caps path leaves
* ATS/PRI/PASID off and no GCR3 table. This follows the detach/attach
* pattern used by amd_iommu_attach_device().
*
* Locking: this and amd_iommu_disable_perfopt() run only from the
* consumer driver's bind/unbind path. group->mutex is not exposed to
* drivers, but a device bound to its native driver cannot have its domain
* changed concurrently by the core (VFIO ownership is mutually exclusive;
* sysfs domain changes require an unused group), so the detach/attach pair
* is serialized without it.
*/
dev_data->perfopt = true;
detach_device(&pdev->dev);
ret = attach_device(&pdev->dev, domain);
if (ret)
goto err_restore;
ret = perfopt_get(iommu);
if (ret)
goto err_rearm;
dev_info_once(&pdev->dev, "PerfOpt armed on IOMMU%d\n", iommu->index);
return 0;
err_rearm:
detach_device(&pdev->dev);
err_restore:
dev_data->perfopt = false;
if (attach_device(&pdev->dev, domain))
pci_err(pdev, "failed to restore state after PerfOpt setup; device left detached\n");
dev_err_once(&pdev->dev, "PerfOpt failed to arm on IOMMU%d (%d)\n",
iommu->index, ret);
return ret;
}
EXPORT_SYMBOL_GPL(amd_iommu_enable_perfopt);
void amd_iommu_disable_perfopt(struct pci_dev *pdev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);
struct protection_domain *domain;
if (!iommu || !dev_data || !dev_data->perfopt || !dev_data->domain)
return;
if (WARN_ON(perfopt_put(iommu)))
pci_err(pdev, "failed to clear PerfOpt\n");
/*
* Restore ATS/PRI/PASID (and thus SVA) by re-homing the device onto its
* identity domain with the flag cleared, so a later bind without PerfOpt
* sees a normally-capable device. See the locking note in
* amd_iommu_enable_perfopt().
*/
domain = dev_data->domain;
dev_data->perfopt = false;
detach_device(&pdev->dev);
if (attach_device(&pdev->dev, domain))
pci_err(pdev, "failed to restore caps after PerfOpt disable\n");
}
EXPORT_SYMBOL_GPL(amd_iommu_disable_perfopt);
static struct iommu_device *amd_iommu_probe_device(struct device *dev)
{
struct iommu_device *iommu_dev;
@ -2736,26 +2555,6 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev)
return iommu_dev;
}
static void amd_iommu_release_device(struct device *dev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
if (dev_data->perfopt) {
if (WARN_ON(perfopt_put(iommu)))
dev_err(dev, "IOMMU%d: failed to clear PerfOpt on release\n",
iommu->index);
dev_data->perfopt = false;
}
WARN_ON(dev_data->domain);
/*
* We keep dev_data around for unplugged devices and reuse it when the
* device is re-plugged - not doing so would introduce a ton of races.
*/
}
static struct iommu_group *amd_iommu_device_group(struct device *dev)
{
if (dev_is_pci(dev))
@ -3116,24 +2915,32 @@ void amd_iommu_domain_free(struct iommu_domain *dom)
kfree(domain);
}
/* Each IOMMU has at most one PerfOpt-eligible GPU. */
static int set_device_perfopt(struct iommu_dev_data *dev_data, bool enable)
{
struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
if (!dev_data->perfopt)
return 0;
guard(raw_spinlock_irqsave)(&iommu->lock);
if (iommu->perfopt_enabled == enable)
return 0;
return amd_iommu_perfopt_set(iommu, enable);
}
static int blocked_domain_attach_device(struct iommu_domain *domain,
struct device *dev,
struct iommu_domain *old)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
int ret;
/*
* blocked_domain is also the .release_domain, so this is the normal
* teardown path: drop the reference and clear the flag here too, and
* don't fail teardown if the WARN-guarded write doesn't stick.
*/
if (dev_data->perfopt) {
if (WARN_ON(perfopt_put(iommu)))
dev_err(dev, "IOMMU%d: failed to clear PerfOpt for blocked domain\n",
iommu->index);
dev_data->perfopt = false;
}
/* Clearing the DTE cannot block DMA while PerfOpt bypasses it. */
ret = set_device_perfopt(dev_data, false);
if (ret)
return ret;
if (dev_data->domain)
detach_device(dev);
@ -3167,6 +2974,9 @@ static struct protection_domain identity_domain;
static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *dev,
struct iommu_domain *old)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
int ret;
/*
* Don't allow attaching a device to the identity domain if SNP is
* enabled and SNP Mode0 support is not present.
@ -3174,7 +2984,13 @@ static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *de
if (amd_iommu_snp_en && !amd_iommu_snp_mode0_sup)
return -EINVAL;
return amd_iommu_attach_device(dom, dev, old);
ret = amd_iommu_attach_device(dom, dev, old);
if (ret)
return ret;
/* PerfOpt is optional; the identity attachment is already complete. */
set_device_perfopt(dev_data, true);
return 0;
}
static const struct iommu_domain_ops identity_domain_ops = {
@ -3202,9 +3018,6 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev,
struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);
int ret;
if (dev_data->perfopt && !pdom_is_in_pt_mode(domain))
return -EBUSY;
/*
* Skip attach device to domain if new domain is same as
* devices current domain
@ -3221,6 +3034,12 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev,
if (dom->dirty_ops && !amd_iommu_hd_support(iommu))
return -EINVAL;
if (!pdom_is_in_pt_mode(domain)) {
ret = set_device_perfopt(dev_data, false);
if (ret)
return ret;
}
if (dev_data->domain)
detach_device(dev);
@ -3372,6 +3191,7 @@ static bool amd_iommu_is_attach_deferred(struct device *dev)
static bool quirks_force_identity_mapping(struct pci_dev *pdev)
{
struct pci_dev *bridge;
int class = pdev->class >> 8;
/* AMD GPU vendor ID */
@ -3383,7 +3203,8 @@ static bool quirks_force_identity_mapping(struct pci_dev *pdev)
class != PCI_CLASS_DISPLAY_OTHER)
return false;
if (pci_upstream_bridge(pdev)->vendor == PCI_VENDOR_ID_ATI)
bridge = pci_upstream_bridge(pdev);
if (bridge && bridge->vendor == PCI_VENDOR_ID_ATI)
return false;
/* It is the GPU in an APU, force identity domain */
@ -3415,6 +3236,9 @@ static int amd_iommu_def_domain_type(struct device *dev)
if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
return 0;
dev_data->perfopt = get_amd_iommu_from_dev_data(dev_data)->features &
FEATURE_PERF_OPT;
return IOMMU_DOMAIN_IDENTITY;
}
@ -3436,7 +3260,6 @@ const struct iommu_ops amd_iommu_ops = {
.domain_alloc_paging_flags = amd_iommu_domain_alloc_paging_flags,
.domain_alloc_sva = amd_iommu_domain_alloc_sva,
.probe_device = amd_iommu_probe_device,
.release_device = amd_iommu_release_device,
.device_group = amd_iommu_device_group,
.get_resv_regions = amd_iommu_get_resv_regions,
.is_attach_deferred = amd_iommu_is_attach_deferred,

11
include/linux/amd-iommu.h
View File

@ -76,15 +76,4 @@ static inline int amd_iommu_snp_disable(void) { return 0; }
static inline bool amd_iommu_sev_tio_supported(void) { return false; }
#endif
#ifdef CONFIG_AMD_IOMMU
int amd_iommu_enable_perfopt(struct pci_dev *pdev);
void amd_iommu_disable_perfopt(struct pci_dev *pdev);
#else
static inline int amd_iommu_enable_perfopt(struct pci_dev *pdev)
{
return 0;
}
static inline void amd_iommu_disable_perfopt(struct pci_dev *pdev) { }
#endif
#endif /* _ASM_X86_AMD_IOMMU_H */