mirror of
https://github.com/torvalds/linux.git
synced 2026-05-28 00:53:34 +02:00
Merge branch 'intel/vt-d' into next
* intel/vt-d: iommu/vt-d: Fix UAF on sva unbind with pending IOPFs iommu/vt-d: Make iotlb_sync_map a static property of dmar_domain iommu/vt-d: Deduplicate cache_tag_flush_all by reusing flush_range iommu/vt-d: Fix missing PASID in dev TLB flush with cache_tag_flush_all iommu/vt-d: Split paging_domain_compatible() iommu/vt-d: Split intel_iommu_enforce_cache_coherency() iommu/vt-d: Create unique domain ops for each stage iommu/vt-d: Split intel_iommu_domain_alloc_paging_flags() iommu/vt-d: Do not wipe out the page table NID when devices detach iommu/vt-d: Fold domain_exit() into intel_iommu_domain_free() iommu/vt-d: Lift the __pa to domain_setup_first_level/intel_svm_set_dev_pasid() iommu/vt-d: Optimize iotlb_sync_map for non-caching/non-RWBF modes iommu/vt-d: Remove the CONFIG_X86 wrapping from iommu init hook
This commit is contained in:
Will Deacon
commit
9f341a2aeb
|
|
@ -371,7 +371,7 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *
|
|||
struct intel_iommu *iommu = tag->iommu;
|
||||
u64 type = DMA_TLB_PSI_FLUSH;
|
||||
|
||||
if (domain->use_first_level) {
|
||||
if (intel_domain_is_fs_paging(domain)) {
|
||||
qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr,
|
||||
pages, ih, domain->qi_batch);
|
||||
return;
|
||||
|
|
@ -423,22 +423,6 @@ static void cache_tag_flush_devtlb_psi(struct dmar_domain *domain, struct cache_
|
|||
domain->qi_batch);
|
||||
}
|
||||
|
||||
static void cache_tag_flush_devtlb_all(struct dmar_domain *domain, struct cache_tag *tag)
|
||||
{
|
||||
struct intel_iommu *iommu = tag->iommu;
|
||||
struct device_domain_info *info;
|
||||
u16 sid;
|
||||
|
||||
info = dev_iommu_priv_get(tag->dev);
|
||||
sid = PCI_DEVID(info->bus, info->devfn);
|
||||
|
||||
qi_batch_add_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep, 0,
|
||||
MAX_AGAW_PFN_WIDTH, domain->qi_batch);
|
||||
if (info->dtlb_extra_inval)
|
||||
qi_batch_add_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep, 0,
|
||||
MAX_AGAW_PFN_WIDTH, domain->qi_batch);
|
||||
}
|
||||
|
||||
/*
|
||||
* Invalidates a range of IOVA from @start (inclusive) to @end (inclusive)
|
||||
* when the memory mappings in the target domain have been modified.
|
||||
|
|
@ -451,7 +435,13 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start,
|
|||
struct cache_tag *tag;
|
||||
unsigned long flags;
|
||||
|
||||
addr = calculate_psi_aligned_address(start, end, &pages, &mask);
|
||||
if (start == 0 && end == ULONG_MAX) {
|
||||
addr = 0;
|
||||
pages = -1;
|
||||
mask = MAX_AGAW_PFN_WIDTH;
|
||||
} else {
|
||||
addr = calculate_psi_aligned_address(start, end, &pages, &mask);
|
||||
}
|
||||
|
||||
spin_lock_irqsave(&domain->cache_lock, flags);
|
||||
list_for_each_entry(tag, &domain->cache_tags, node) {
|
||||
|
|
@ -492,31 +482,7 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start,
|
|||
*/
|
||||
void cache_tag_flush_all(struct dmar_domain *domain)
|
||||
{
|
||||
struct intel_iommu *iommu = NULL;
|
||||
struct cache_tag *tag;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&domain->cache_lock, flags);
|
||||
list_for_each_entry(tag, &domain->cache_tags, node) {
|
||||
if (iommu && iommu != tag->iommu)
|
||||
qi_batch_flush_descs(iommu, domain->qi_batch);
|
||||
iommu = tag->iommu;
|
||||
|
||||
switch (tag->type) {
|
||||
case CACHE_TAG_IOTLB:
|
||||
case CACHE_TAG_NESTING_IOTLB:
|
||||
cache_tag_flush_iotlb(domain, tag, 0, -1, 0, 0);
|
||||
break;
|
||||
case CACHE_TAG_DEVTLB:
|
||||
case CACHE_TAG_NESTING_DEVTLB:
|
||||
cache_tag_flush_devtlb_all(domain, tag);
|
||||
break;
|
||||
}
|
||||
|
||||
trace_cache_tag_flush_all(tag);
|
||||
}
|
||||
qi_batch_flush_descs(iommu, domain->qi_batch);
|
||||
spin_unlock_irqrestore(&domain->cache_lock, flags);
|
||||
cache_tag_flush_range(domain, 0, ULONG_MAX, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -546,7 +512,8 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start,
|
|||
qi_batch_flush_descs(iommu, domain->qi_batch);
|
||||
iommu = tag->iommu;
|
||||
|
||||
if (!cap_caching_mode(iommu->cap) || domain->use_first_level) {
|
||||
if (!cap_caching_mode(iommu->cap) ||
|
||||
intel_domain_is_fs_paging(domain)) {
|
||||
iommu_flush_write_buffer(iommu);
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -935,14 +935,11 @@ void __init detect_intel_iommu(void)
|
|||
pci_request_acs();
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86
|
||||
if (!ret) {
|
||||
x86_init.iommu.iommu_init = intel_iommu_init;
|
||||
x86_platform.iommu_shutdown = intel_iommu_shutdown;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
if (dmar_tbl) {
|
||||
acpi_put_table(dmar_tbl);
|
||||
dmar_tbl = NULL;
|
||||
|
|
|
|||
|
|
@ -57,6 +57,8 @@
|
|||
static void __init check_tylersburg_isoch(void);
|
||||
static int rwbf_quirk;
|
||||
|
||||
#define rwbf_required(iommu) (rwbf_quirk || cap_rwbf((iommu)->cap))
|
||||
|
||||
/*
|
||||
* set to 1 to panic kernel if can't successfully enable VT-d
|
||||
* (used when kernel is launched w/ TXT)
|
||||
|
|
@ -1391,28 +1393,10 @@ void domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
|
|||
if (--info->refcnt == 0) {
|
||||
ida_free(&iommu->domain_ida, info->did);
|
||||
xa_erase(&domain->iommu_array, iommu->seq_id);
|
||||
domain->nid = NUMA_NO_NODE;
|
||||
kfree(info);
|
||||
}
|
||||
}
|
||||
|
||||
static void domain_exit(struct dmar_domain *domain)
|
||||
{
|
||||
if (domain->pgd) {
|
||||
struct iommu_pages_list freelist =
|
||||
IOMMU_PAGES_LIST_INIT(freelist);
|
||||
|
||||
domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist);
|
||||
iommu_put_pages_list(&freelist);
|
||||
}
|
||||
|
||||
if (WARN_ON(!list_empty(&domain->devices)))
|
||||
return;
|
||||
|
||||
kfree(domain->qi_batch);
|
||||
kfree(domain);
|
||||
}
|
||||
|
||||
/*
|
||||
* For kdump cases, old valid entries may be cached due to the
|
||||
* in-flight DMA and copied pgtable, but there is no unmapping
|
||||
|
|
@ -1480,6 +1464,9 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
|
|||
struct context_entry *context;
|
||||
int ret;
|
||||
|
||||
if (WARN_ON(!intel_domain_is_ss_paging(domain)))
|
||||
return -EINVAL;
|
||||
|
||||
pr_debug("Set context mapping for %02x:%02x.%d\n",
|
||||
bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
|
||||
|
||||
|
|
@ -1736,15 +1723,14 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8
|
|||
intel_context_flush_no_pasid(info, context, did);
|
||||
}
|
||||
|
||||
int __domain_setup_first_level(struct intel_iommu *iommu,
|
||||
struct device *dev, ioasid_t pasid,
|
||||
u16 did, pgd_t *pgd, int flags,
|
||||
struct iommu_domain *old)
|
||||
int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev,
|
||||
ioasid_t pasid, u16 did, phys_addr_t fsptptr,
|
||||
int flags, struct iommu_domain *old)
|
||||
{
|
||||
if (!old)
|
||||
return intel_pasid_setup_first_level(iommu, dev, pgd,
|
||||
pasid, did, flags);
|
||||
return intel_pasid_replace_first_level(iommu, dev, pgd, pasid, did,
|
||||
return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid,
|
||||
did, flags);
|
||||
return intel_pasid_replace_first_level(iommu, dev, fsptptr, pasid, did,
|
||||
iommu_domain_did(old, iommu),
|
||||
flags);
|
||||
}
|
||||
|
|
@ -1793,7 +1779,7 @@ static int domain_setup_first_level(struct intel_iommu *iommu,
|
|||
|
||||
return __domain_setup_first_level(iommu, dev, pasid,
|
||||
domain_id_iommu(domain, iommu),
|
||||
(pgd_t *)pgd, flags, old);
|
||||
__pa(pgd), flags, old);
|
||||
}
|
||||
|
||||
static int dmar_domain_attach_device(struct dmar_domain *domain,
|
||||
|
|
@ -1819,12 +1805,14 @@ static int dmar_domain_attach_device(struct dmar_domain *domain,
|
|||
|
||||
if (!sm_supported(iommu))
|
||||
ret = domain_context_mapping(domain, dev);
|
||||
else if (domain->use_first_level)
|
||||
else if (intel_domain_is_fs_paging(domain))
|
||||
ret = domain_setup_first_level(iommu, domain, dev,
|
||||
IOMMU_NO_PASID, NULL);
|
||||
else
|
||||
else if (intel_domain_is_ss_paging(domain))
|
||||
ret = domain_setup_second_level(iommu, domain, dev,
|
||||
IOMMU_NO_PASID, NULL);
|
||||
else if (WARN_ON(true))
|
||||
ret = -EINVAL;
|
||||
|
||||
if (ret)
|
||||
goto out_block_translation;
|
||||
|
|
@ -3286,10 +3274,14 @@ static struct dmar_domain *paging_domain_alloc(struct device *dev, bool first_st
|
|||
spin_lock_init(&domain->lock);
|
||||
spin_lock_init(&domain->cache_lock);
|
||||
xa_init(&domain->iommu_array);
|
||||
INIT_LIST_HEAD(&domain->s1_domains);
|
||||
spin_lock_init(&domain->s1_lock);
|
||||
|
||||
domain->nid = dev_to_node(dev);
|
||||
domain->use_first_level = first_stage;
|
||||
|
||||
domain->domain.type = IOMMU_DOMAIN_UNMANAGED;
|
||||
|
||||
/* calculate the address width */
|
||||
addr_width = agaw_to_width(iommu->agaw);
|
||||
if (addr_width > cap_mgaw(iommu->cap))
|
||||
|
|
@ -3331,71 +3323,168 @@ static struct dmar_domain *paging_domain_alloc(struct device *dev, bool first_st
|
|||
}
|
||||
|
||||
static struct iommu_domain *
|
||||
intel_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags,
|
||||
const struct iommu_user_data *user_data)
|
||||
intel_iommu_domain_alloc_first_stage(struct device *dev,
|
||||
struct intel_iommu *iommu, u32 flags)
|
||||
{
|
||||
struct dmar_domain *dmar_domain;
|
||||
|
||||
if (flags & ~IOMMU_HWPT_ALLOC_PASID)
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
/* Only SL is available in legacy mode */
|
||||
if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
dmar_domain = paging_domain_alloc(dev, true);
|
||||
if (IS_ERR(dmar_domain))
|
||||
return ERR_CAST(dmar_domain);
|
||||
|
||||
dmar_domain->domain.ops = &intel_fs_paging_domain_ops;
|
||||
/*
|
||||
* iotlb sync for map is only needed for legacy implementations that
|
||||
* explicitly require flushing internal write buffers to ensure memory
|
||||
* coherence.
|
||||
*/
|
||||
if (rwbf_required(iommu))
|
||||
dmar_domain->iotlb_sync_map = true;
|
||||
|
||||
return &dmar_domain->domain;
|
||||
}
|
||||
|
||||
static struct iommu_domain *
|
||||
intel_iommu_domain_alloc_second_stage(struct device *dev,
|
||||
struct intel_iommu *iommu, u32 flags)
|
||||
{
|
||||
struct device_domain_info *info = dev_iommu_priv_get(dev);
|
||||
bool dirty_tracking = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
|
||||
bool nested_parent = flags & IOMMU_HWPT_ALLOC_NEST_PARENT;
|
||||
struct intel_iommu *iommu = info->iommu;
|
||||
struct dmar_domain *dmar_domain;
|
||||
struct iommu_domain *domain;
|
||||
bool first_stage;
|
||||
|
||||
if (flags &
|
||||
(~(IOMMU_HWPT_ALLOC_NEST_PARENT | IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
|
||||
IOMMU_HWPT_ALLOC_PASID)))
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
if (nested_parent && !nested_supported(iommu))
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
if (user_data || (dirty_tracking && !ssads_supported(iommu)))
|
||||
|
||||
if (((flags & IOMMU_HWPT_ALLOC_NEST_PARENT) &&
|
||||
!nested_supported(iommu)) ||
|
||||
((flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) &&
|
||||
!ssads_supported(iommu)))
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
/*
|
||||
* Always allocate the guest compatible page table unless
|
||||
* IOMMU_HWPT_ALLOC_NEST_PARENT or IOMMU_HWPT_ALLOC_DIRTY_TRACKING
|
||||
* is specified.
|
||||
*/
|
||||
if (nested_parent || dirty_tracking) {
|
||||
if (!sm_supported(iommu) || !ecap_slts(iommu->ecap))
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
first_stage = false;
|
||||
} else {
|
||||
first_stage = first_level_by_default(iommu);
|
||||
}
|
||||
/* Legacy mode always supports second stage */
|
||||
if (sm_supported(iommu) && !ecap_slts(iommu->ecap))
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
dmar_domain = paging_domain_alloc(dev, first_stage);
|
||||
dmar_domain = paging_domain_alloc(dev, false);
|
||||
if (IS_ERR(dmar_domain))
|
||||
return ERR_CAST(dmar_domain);
|
||||
domain = &dmar_domain->domain;
|
||||
domain->type = IOMMU_DOMAIN_UNMANAGED;
|
||||
domain->owner = &intel_iommu_ops;
|
||||
domain->ops = intel_iommu_ops.default_domain_ops;
|
||||
|
||||
if (nested_parent) {
|
||||
dmar_domain->nested_parent = true;
|
||||
INIT_LIST_HEAD(&dmar_domain->s1_domains);
|
||||
spin_lock_init(&dmar_domain->s1_lock);
|
||||
}
|
||||
dmar_domain->domain.ops = &intel_ss_paging_domain_ops;
|
||||
dmar_domain->nested_parent = flags & IOMMU_HWPT_ALLOC_NEST_PARENT;
|
||||
|
||||
if (dirty_tracking) {
|
||||
if (dmar_domain->use_first_level) {
|
||||
iommu_domain_free(domain);
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
}
|
||||
domain->dirty_ops = &intel_dirty_ops;
|
||||
}
|
||||
if (flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING)
|
||||
dmar_domain->domain.dirty_ops = &intel_dirty_ops;
|
||||
|
||||
return domain;
|
||||
/*
|
||||
* Besides the internal write buffer flush, the caching mode used for
|
||||
* legacy nested translation (which utilizes shadowing page tables)
|
||||
* also requires iotlb sync on map.
|
||||
*/
|
||||
if (rwbf_required(iommu) || cap_caching_mode(iommu->cap))
|
||||
dmar_domain->iotlb_sync_map = true;
|
||||
|
||||
return &dmar_domain->domain;
|
||||
}
|
||||
|
||||
static struct iommu_domain *
|
||||
intel_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags,
|
||||
const struct iommu_user_data *user_data)
|
||||
{
|
||||
struct device_domain_info *info = dev_iommu_priv_get(dev);
|
||||
struct intel_iommu *iommu = info->iommu;
|
||||
struct iommu_domain *domain;
|
||||
|
||||
if (user_data)
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
|
||||
/* Prefer first stage if possible by default. */
|
||||
domain = intel_iommu_domain_alloc_first_stage(dev, iommu, flags);
|
||||
if (domain != ERR_PTR(-EOPNOTSUPP))
|
||||
return domain;
|
||||
return intel_iommu_domain_alloc_second_stage(dev, iommu, flags);
|
||||
}
|
||||
|
||||
static void intel_iommu_domain_free(struct iommu_domain *domain)
|
||||
{
|
||||
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
|
||||
|
||||
WARN_ON(dmar_domain->nested_parent &&
|
||||
!list_empty(&dmar_domain->s1_domains));
|
||||
domain_exit(dmar_domain);
|
||||
if (WARN_ON(dmar_domain->nested_parent &&
|
||||
!list_empty(&dmar_domain->s1_domains)))
|
||||
return;
|
||||
|
||||
if (WARN_ON(!list_empty(&dmar_domain->devices)))
|
||||
return;
|
||||
|
||||
if (dmar_domain->pgd) {
|
||||
struct iommu_pages_list freelist =
|
||||
IOMMU_PAGES_LIST_INIT(freelist);
|
||||
|
||||
domain_unmap(dmar_domain, 0, DOMAIN_MAX_PFN(dmar_domain->gaw),
|
||||
&freelist);
|
||||
iommu_put_pages_list(&freelist);
|
||||
}
|
||||
|
||||
kfree(dmar_domain->qi_batch);
|
||||
kfree(dmar_domain);
|
||||
}
|
||||
|
||||
static int paging_domain_compatible_first_stage(struct dmar_domain *dmar_domain,
|
||||
struct intel_iommu *iommu)
|
||||
{
|
||||
if (WARN_ON(dmar_domain->domain.dirty_ops ||
|
||||
dmar_domain->nested_parent))
|
||||
return -EINVAL;
|
||||
|
||||
/* Only SL is available in legacy mode */
|
||||
if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))
|
||||
return -EINVAL;
|
||||
|
||||
/* Same page size support */
|
||||
if (!cap_fl1gp_support(iommu->cap) &&
|
||||
(dmar_domain->domain.pgsize_bitmap & SZ_1G))
|
||||
return -EINVAL;
|
||||
|
||||
/* iotlb sync on map requirement */
|
||||
if ((rwbf_required(iommu)) && !dmar_domain->iotlb_sync_map)
|
||||
return -EINVAL;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
paging_domain_compatible_second_stage(struct dmar_domain *dmar_domain,
|
||||
struct intel_iommu *iommu)
|
||||
{
|
||||
unsigned int sslps = cap_super_page_val(iommu->cap);
|
||||
|
||||
if (dmar_domain->domain.dirty_ops && !ssads_supported(iommu))
|
||||
return -EINVAL;
|
||||
if (dmar_domain->nested_parent && !nested_supported(iommu))
|
||||
return -EINVAL;
|
||||
|
||||
/* Legacy mode always supports second stage */
|
||||
if (sm_supported(iommu) && !ecap_slts(iommu->ecap))
|
||||
return -EINVAL;
|
||||
|
||||
/* Same page size support */
|
||||
if (!(sslps & BIT(0)) && (dmar_domain->domain.pgsize_bitmap & SZ_2M))
|
||||
return -EINVAL;
|
||||
if (!(sslps & BIT(1)) && (dmar_domain->domain.pgsize_bitmap & SZ_1G))
|
||||
return -EINVAL;
|
||||
|
||||
/* iotlb sync on map requirement */
|
||||
if ((rwbf_required(iommu) || cap_caching_mode(iommu->cap)) &&
|
||||
!dmar_domain->iotlb_sync_map)
|
||||
return -EINVAL;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int paging_domain_compatible(struct iommu_domain *domain, struct device *dev)
|
||||
|
|
@ -3403,28 +3492,29 @@ int paging_domain_compatible(struct iommu_domain *domain, struct device *dev)
|
|||
struct device_domain_info *info = dev_iommu_priv_get(dev);
|
||||
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
|
||||
struct intel_iommu *iommu = info->iommu;
|
||||
int ret = -EINVAL;
|
||||
int addr_width;
|
||||
|
||||
if (WARN_ON_ONCE(!(domain->type & __IOMMU_DOMAIN_PAGING)))
|
||||
return -EPERM;
|
||||
if (intel_domain_is_fs_paging(dmar_domain))
|
||||
ret = paging_domain_compatible_first_stage(dmar_domain, iommu);
|
||||
else if (intel_domain_is_ss_paging(dmar_domain))
|
||||
ret = paging_domain_compatible_second_stage(dmar_domain, iommu);
|
||||
else if (WARN_ON(true))
|
||||
ret = -EINVAL;
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
/*
|
||||
* FIXME this is locked wrong, it needs to be under the
|
||||
* dmar_domain->lock
|
||||
*/
|
||||
if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
|
||||
return -EINVAL;
|
||||
|
||||
if (domain->dirty_ops && !ssads_supported(iommu))
|
||||
return -EINVAL;
|
||||
|
||||
if (dmar_domain->iommu_coherency !=
|
||||
iommu_paging_structure_coherency(iommu))
|
||||
return -EINVAL;
|
||||
|
||||
if (dmar_domain->iommu_superpage !=
|
||||
iommu_superpage_capability(iommu, dmar_domain->use_first_level))
|
||||
return -EINVAL;
|
||||
|
||||
if (dmar_domain->use_first_level &&
|
||||
(!sm_supported(iommu) || !ecap_flts(iommu->ecap)))
|
||||
return -EINVAL;
|
||||
|
||||
/* check if this iommu agaw is sufficient for max mapped address */
|
||||
addr_width = agaw_to_width(iommu->agaw);
|
||||
|
|
@ -3610,44 +3700,41 @@ static bool domain_support_force_snooping(struct dmar_domain *domain)
|
|||
return support;
|
||||
}
|
||||
|
||||
static void domain_set_force_snooping(struct dmar_domain *domain)
|
||||
{
|
||||
struct device_domain_info *info;
|
||||
|
||||
assert_spin_locked(&domain->lock);
|
||||
/*
|
||||
* Second level page table supports per-PTE snoop control. The
|
||||
* iommu_map() interface will handle this by setting SNP bit.
|
||||
*/
|
||||
if (!domain->use_first_level) {
|
||||
domain->set_pte_snp = true;
|
||||
return;
|
||||
}
|
||||
|
||||
list_for_each_entry(info, &domain->devices, link)
|
||||
intel_pasid_setup_page_snoop_control(info->iommu, info->dev,
|
||||
IOMMU_NO_PASID);
|
||||
}
|
||||
|
||||
static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain)
|
||||
static bool intel_iommu_enforce_cache_coherency_fs(struct iommu_domain *domain)
|
||||
{
|
||||
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
|
||||
unsigned long flags;
|
||||
struct device_domain_info *info;
|
||||
|
||||
guard(spinlock_irqsave)(&dmar_domain->lock);
|
||||
|
||||
if (dmar_domain->force_snooping)
|
||||
return true;
|
||||
|
||||
spin_lock_irqsave(&dmar_domain->lock, flags);
|
||||
if (!domain_support_force_snooping(dmar_domain) ||
|
||||
(!dmar_domain->use_first_level && dmar_domain->has_mappings)) {
|
||||
spin_unlock_irqrestore(&dmar_domain->lock, flags);
|
||||
if (!domain_support_force_snooping(dmar_domain))
|
||||
return false;
|
||||
}
|
||||
|
||||
domain_set_force_snooping(dmar_domain);
|
||||
dmar_domain->force_snooping = true;
|
||||
spin_unlock_irqrestore(&dmar_domain->lock, flags);
|
||||
list_for_each_entry(info, &dmar_domain->devices, link)
|
||||
intel_pasid_setup_page_snoop_control(info->iommu, info->dev,
|
||||
IOMMU_NO_PASID);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool intel_iommu_enforce_cache_coherency_ss(struct iommu_domain *domain)
|
||||
{
|
||||
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
|
||||
|
||||
guard(spinlock_irqsave)(&dmar_domain->lock);
|
||||
if (!domain_support_force_snooping(dmar_domain) ||
|
||||
dmar_domain->has_mappings)
|
||||
return false;
|
||||
|
||||
/*
|
||||
* Second level page table supports per-PTE snoop control. The
|
||||
* iommu_map() interface will handle this by setting SNP bit.
|
||||
*/
|
||||
dmar_domain->set_pte_snp = true;
|
||||
dmar_domain->force_snooping = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -3945,7 +4032,10 @@ static bool risky_device(struct pci_dev *pdev)
|
|||
static int intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
|
||||
unsigned long iova, size_t size)
|
||||
{
|
||||
cache_tag_flush_range_np(to_dmar_domain(domain), iova, iova + size - 1);
|
||||
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
|
||||
|
||||
if (dmar_domain->iotlb_sync_map)
|
||||
cache_tag_flush_range_np(dmar_domain, iova, iova + size - 1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -3991,8 +4081,8 @@ static int blocking_domain_set_dev_pasid(struct iommu_domain *domain,
|
|||
{
|
||||
struct device_domain_info *info = dev_iommu_priv_get(dev);
|
||||
|
||||
iopf_for_domain_remove(old, dev);
|
||||
intel_pasid_tear_down_entry(info->iommu, dev, pasid, false);
|
||||
iopf_for_domain_remove(old, dev);
|
||||
domain_remove_dev_pasid(old, dev, pasid);
|
||||
|
||||
return 0;
|
||||
|
|
@ -4069,12 +4159,15 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
|
|||
if (ret)
|
||||
goto out_remove_dev_pasid;
|
||||
|
||||
if (dmar_domain->use_first_level)
|
||||
if (intel_domain_is_fs_paging(dmar_domain))
|
||||
ret = domain_setup_first_level(iommu, dmar_domain,
|
||||
dev, pasid, old);
|
||||
else
|
||||
else if (intel_domain_is_ss_paging(dmar_domain))
|
||||
ret = domain_setup_second_level(iommu, dmar_domain,
|
||||
dev, pasid, old);
|
||||
else if (WARN_ON(true))
|
||||
ret = -EINVAL;
|
||||
|
||||
if (ret)
|
||||
goto out_unwind_iopf;
|
||||
|
||||
|
|
@ -4349,6 +4442,32 @@ static struct iommu_domain identity_domain = {
|
|||
},
|
||||
};
|
||||
|
||||
const struct iommu_domain_ops intel_fs_paging_domain_ops = {
|
||||
.attach_dev = intel_iommu_attach_device,
|
||||
.set_dev_pasid = intel_iommu_set_dev_pasid,
|
||||
.map_pages = intel_iommu_map_pages,
|
||||
.unmap_pages = intel_iommu_unmap_pages,
|
||||
.iotlb_sync_map = intel_iommu_iotlb_sync_map,
|
||||
.flush_iotlb_all = intel_flush_iotlb_all,
|
||||
.iotlb_sync = intel_iommu_tlb_sync,
|
||||
.iova_to_phys = intel_iommu_iova_to_phys,
|
||||
.free = intel_iommu_domain_free,
|
||||
.enforce_cache_coherency = intel_iommu_enforce_cache_coherency_fs,
|
||||
};
|
||||
|
||||
const struct iommu_domain_ops intel_ss_paging_domain_ops = {
|
||||
.attach_dev = intel_iommu_attach_device,
|
||||
.set_dev_pasid = intel_iommu_set_dev_pasid,
|
||||
.map_pages = intel_iommu_map_pages,
|
||||
.unmap_pages = intel_iommu_unmap_pages,
|
||||
.iotlb_sync_map = intel_iommu_iotlb_sync_map,
|
||||
.flush_iotlb_all = intel_flush_iotlb_all,
|
||||
.iotlb_sync = intel_iommu_tlb_sync,
|
||||
.iova_to_phys = intel_iommu_iova_to_phys,
|
||||
.free = intel_iommu_domain_free,
|
||||
.enforce_cache_coherency = intel_iommu_enforce_cache_coherency_ss,
|
||||
};
|
||||
|
||||
const struct iommu_ops intel_iommu_ops = {
|
||||
.blocked_domain = &blocking_domain,
|
||||
.release_domain = &blocking_domain,
|
||||
|
|
@ -4366,18 +4485,6 @@ const struct iommu_ops intel_iommu_ops = {
|
|||
.is_attach_deferred = intel_iommu_is_attach_deferred,
|
||||
.def_domain_type = device_def_domain_type,
|
||||
.page_response = intel_iommu_page_response,
|
||||
.default_domain_ops = &(const struct iommu_domain_ops) {
|
||||
.attach_dev = intel_iommu_attach_device,
|
||||
.set_dev_pasid = intel_iommu_set_dev_pasid,
|
||||
.map_pages = intel_iommu_map_pages,
|
||||
.unmap_pages = intel_iommu_unmap_pages,
|
||||
.iotlb_sync_map = intel_iommu_iotlb_sync_map,
|
||||
.flush_iotlb_all = intel_flush_iotlb_all,
|
||||
.iotlb_sync = intel_iommu_tlb_sync,
|
||||
.iova_to_phys = intel_iommu_iova_to_phys,
|
||||
.free = intel_iommu_domain_free,
|
||||
.enforce_cache_coherency = intel_iommu_enforce_cache_coherency,
|
||||
}
|
||||
};
|
||||
|
||||
static void quirk_iommu_igfx(struct pci_dev *dev)
|
||||
|
|
|
|||
|
|
@ -614,6 +614,9 @@ struct dmar_domain {
|
|||
u8 has_mappings:1; /* Has mappings configured through
|
||||
* iommu_map() interface.
|
||||
*/
|
||||
u8 iotlb_sync_map:1; /* Need to flush IOTLB cache or write
|
||||
* buffer when creating mappings.
|
||||
*/
|
||||
|
||||
spinlock_t lock; /* Protect device tracking lists */
|
||||
struct list_head devices; /* all devices' list */
|
||||
|
|
@ -1252,10 +1255,9 @@ domain_add_dev_pasid(struct iommu_domain *domain,
|
|||
void domain_remove_dev_pasid(struct iommu_domain *domain,
|
||||
struct device *dev, ioasid_t pasid);
|
||||
|
||||
int __domain_setup_first_level(struct intel_iommu *iommu,
|
||||
struct device *dev, ioasid_t pasid,
|
||||
u16 did, pgd_t *pgd, int flags,
|
||||
struct iommu_domain *old);
|
||||
int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev,
|
||||
ioasid_t pasid, u16 did, phys_addr_t fsptptr,
|
||||
int flags, struct iommu_domain *old);
|
||||
|
||||
int dmar_ir_support(void);
|
||||
|
||||
|
|
@ -1376,6 +1378,18 @@ struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
|
|||
u8 devfn, int alloc);
|
||||
|
||||
extern const struct iommu_ops intel_iommu_ops;
|
||||
extern const struct iommu_domain_ops intel_fs_paging_domain_ops;
|
||||
extern const struct iommu_domain_ops intel_ss_paging_domain_ops;
|
||||
|
||||
static inline bool intel_domain_is_fs_paging(struct dmar_domain *domain)
|
||||
{
|
||||
return domain->domain.ops == &intel_fs_paging_domain_ops;
|
||||
}
|
||||
|
||||
static inline bool intel_domain_is_ss_paging(struct dmar_domain *domain)
|
||||
{
|
||||
return domain->domain.ops == &intel_ss_paging_domain_ops;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_INTEL_IOMMU
|
||||
extern int intel_iommu_sm;
|
||||
|
|
|
|||
|
|
@ -216,8 +216,7 @@ intel_iommu_domain_alloc_nested(struct device *dev, struct iommu_domain *parent,
|
|||
/* Must be nested domain */
|
||||
if (user_data->type != IOMMU_HWPT_DATA_VTD_S1)
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
if (parent->ops != intel_iommu_ops.default_domain_ops ||
|
||||
!s2_domain->nested_parent)
|
||||
if (!intel_domain_is_ss_paging(s2_domain) || !s2_domain->nested_parent)
|
||||
return ERR_PTR(-EINVAL);
|
||||
|
||||
ret = iommu_copy_struct_from_user(&vtd, user_data,
|
||||
|
|
@ -229,7 +228,6 @@ intel_iommu_domain_alloc_nested(struct device *dev, struct iommu_domain *parent,
|
|||
if (!domain)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
domain->use_first_level = true;
|
||||
domain->s2_domain = s2_domain;
|
||||
domain->s1_cfg = vtd;
|
||||
domain->domain.ops = &intel_nested_domain_ops;
|
||||
|
|
|
|||
|
|
@ -348,14 +348,15 @@ static void intel_pasid_flush_present(struct intel_iommu *iommu,
|
|||
*/
|
||||
static void pasid_pte_config_first_level(struct intel_iommu *iommu,
|
||||
struct pasid_entry *pte,
|
||||
pgd_t *pgd, u16 did, int flags)
|
||||
phys_addr_t fsptptr, u16 did,
|
||||
int flags)
|
||||
{
|
||||
lockdep_assert_held(&iommu->lock);
|
||||
|
||||
pasid_clear_entry(pte);
|
||||
|
||||
/* Setup the first level page table pointer: */
|
||||
pasid_set_flptr(pte, (u64)__pa(pgd));
|
||||
pasid_set_flptr(pte, fsptptr);
|
||||
|
||||
if (flags & PASID_FLAG_FL5LP)
|
||||
pasid_set_flpm(pte, 1);
|
||||
|
|
@ -372,9 +373,9 @@ static void pasid_pte_config_first_level(struct intel_iommu *iommu,
|
|||
pasid_set_present(pte);
|
||||
}
|
||||
|
||||
int intel_pasid_setup_first_level(struct intel_iommu *iommu,
|
||||
struct device *dev, pgd_t *pgd,
|
||||
u32 pasid, u16 did, int flags)
|
||||
int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev,
|
||||
phys_addr_t fsptptr, u32 pasid, u16 did,
|
||||
int flags)
|
||||
{
|
||||
struct pasid_entry *pte;
|
||||
|
||||
|
|
@ -402,7 +403,7 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu,
|
|||
return -EBUSY;
|
||||
}
|
||||
|
||||
pasid_pte_config_first_level(iommu, pte, pgd, did, flags);
|
||||
pasid_pte_config_first_level(iommu, pte, fsptptr, did, flags);
|
||||
|
||||
spin_unlock(&iommu->lock);
|
||||
|
||||
|
|
@ -412,7 +413,7 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu,
|
|||
}
|
||||
|
||||
int intel_pasid_replace_first_level(struct intel_iommu *iommu,
|
||||
struct device *dev, pgd_t *pgd,
|
||||
struct device *dev, phys_addr_t fsptptr,
|
||||
u32 pasid, u16 did, u16 old_did,
|
||||
int flags)
|
||||
{
|
||||
|
|
@ -430,7 +431,7 @@ int intel_pasid_replace_first_level(struct intel_iommu *iommu,
|
|||
return -EINVAL;
|
||||
}
|
||||
|
||||
pasid_pte_config_first_level(iommu, &new_pte, pgd, did, flags);
|
||||
pasid_pte_config_first_level(iommu, &new_pte, fsptptr, did, flags);
|
||||
|
||||
spin_lock(&iommu->lock);
|
||||
pte = intel_pasid_get_entry(dev, pasid);
|
||||
|
|
|
|||
|
|
@ -288,9 +288,9 @@ extern unsigned int intel_pasid_max_id;
|
|||
int intel_pasid_alloc_table(struct device *dev);
|
||||
void intel_pasid_free_table(struct device *dev);
|
||||
struct pasid_table *intel_pasid_get_table(struct device *dev);
|
||||
int intel_pasid_setup_first_level(struct intel_iommu *iommu,
|
||||
struct device *dev, pgd_t *pgd,
|
||||
u32 pasid, u16 did, int flags);
|
||||
int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev,
|
||||
phys_addr_t fsptptr, u32 pasid, u16 did,
|
||||
int flags);
|
||||
int intel_pasid_setup_second_level(struct intel_iommu *iommu,
|
||||
struct dmar_domain *domain,
|
||||
struct device *dev, u32 pasid);
|
||||
|
|
@ -302,9 +302,8 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
|
|||
int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,
|
||||
u32 pasid, struct dmar_domain *domain);
|
||||
int intel_pasid_replace_first_level(struct intel_iommu *iommu,
|
||||
struct device *dev, pgd_t *pgd,
|
||||
u32 pasid, u16 did, u16 old_did,
|
||||
int flags);
|
||||
struct device *dev, phys_addr_t fsptptr,
|
||||
u32 pasid, u16 did, u16 old_did, int flags);
|
||||
int intel_pasid_replace_second_level(struct intel_iommu *iommu,
|
||||
struct dmar_domain *domain,
|
||||
struct device *dev, u16 old_did,
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
|
|||
/* Setup the pasid table: */
|
||||
sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
|
||||
ret = __domain_setup_first_level(iommu, dev, pasid,
|
||||
FLPT_DEFAULT_DID, mm->pgd,
|
||||
FLPT_DEFAULT_DID, __pa(mm->pgd),
|
||||
sflags, old);
|
||||
if (ret)
|
||||
goto out_unwind_iopf;
|
||||
|
|
@ -214,7 +214,6 @@ struct iommu_domain *intel_svm_domain_alloc(struct device *dev,
|
|||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
domain->domain.ops = &intel_svm_domain_ops;
|
||||
domain->use_first_level = true;
|
||||
INIT_LIST_HEAD(&domain->dev_pasids);
|
||||
INIT_LIST_HEAD(&domain->cache_tags);
|
||||
spin_lock_init(&domain->cache_lock);
|
||||
|
|
|
|||
|
|
@ -130,11 +130,6 @@ DEFINE_EVENT(cache_tag_log, cache_tag_unassign,
|
|||
TP_ARGS(tag)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(cache_tag_log, cache_tag_flush_all,
|
||||
TP_PROTO(struct cache_tag *tag),
|
||||
TP_ARGS(tag)
|
||||
);
|
||||
|
||||
DECLARE_EVENT_CLASS(cache_tag_flush,
|
||||
TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end,
|
||||
unsigned long addr, unsigned long pages, unsigned long mask),
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user