github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-22 06:01:53 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branches 'arm/allwinner', 'arm/exynos', 'arm/mediatek', 'arm/rockchip', 'arm/smmu', 'ppc/pamu', 's390', 'x86/vt-d', 'x86/amd' and 'core' into next

Browse Source
This commit is contained in:
Joerg Roedel 2022-12-12 12:50:53 +01:00
commit e3eca2e4f6
54 changed files with 1974 additions and 1182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
Documentation/admin-guide/kernel-parameters.txt
View File

@ -2300,7 +2300,13 @@
Provide an override to the IOAPIC-ID<->DEVICE-ID
mapping provided in the IVRS ACPI table.
By default, PCI segment is 0, and can be omitted.
For example:
For example, to map IOAPIC-ID decimal 10 to
PCI segment 0x1 and PCI device 00:14.0,
write the parameter as:
ivrs_ioapic=10@0001:00:14.0
Deprecated formats:
* To map IOAPIC-ID decimal 10 to PCI device 00:14.0
write the parameter as:
ivrs_ioapic[10]=00:14.0
@ -2312,7 +2318,13 @@
Provide an override to the HPET-ID<->DEVICE-ID
mapping provided in the IVRS ACPI table.
By default, PCI segment is 0, and can be omitted.
For example:
For example, to map HPET-ID decimal 10 to
PCI segment 0x1 and PCI device 00:14.0,
write the parameter as:
ivrs_hpet=10@0001:00:14.0
Deprecated formats:
* To map HPET-ID decimal 0 to PCI device 00:14.0
write the parameter as:
ivrs_hpet[0]=00:14.0
@ -2323,15 +2335,20 @@
ivrs_acpihid [HW,X86-64]
Provide an override to the ACPI-HID:UID<->DEVICE-ID
mapping provided in the IVRS ACPI table.
By default, PCI segment is 0, and can be omitted.
For example, to map UART-HID:UID AMD0020:0 to
PCI segment 0x1 and PCI device ID 00:14.5,
write the parameter as:
ivrs_acpihid[0001:00:14.5]=AMD0020:0
ivrs_acpihid=AMD0020:0@0001:00:14.5
By default, PCI segment is 0, and can be omitted.
For example, PCI device 00:14.5 write the parameter as:
Deprecated formats:
* To map UART-HID:UID AMD0020:0 to PCI segment is 0,
PCI device ID 00:14.5, write the parameter as:
ivrs_acpihid[00:14.5]=AMD0020:0
* To map UART-HID:UID AMD0020:0 to PCI segment 0x1 and
PCI device ID 00:14.5, write the parameter as:
ivrs_acpihid[0001:00:14.5]=AMD0020:0
js= [HW,JOY] Analog joystick
See Documentation/input/joydev/joystick.rst.

182
Documentation/devicetree/bindings/iommu/arm,smmu.yaml
View File

@ -28,10 +28,42 @@ properties:
- enum:
- qcom,msm8996-smmu-v2
- qcom,msm8998-smmu-v2
- qcom,sdm630-smmu-v2
- const: qcom,smmu-v2
- description: Qcom SoCs implementing "arm,mmu-500"
- description: Qcom SoCs implementing "qcom,smmu-500" and "arm,mmu-500"
items:
- enum:
- qcom,qcm2290-smmu-500
- qcom,qdu1000-smmu-500
- qcom,sc7180-smmu-500
- qcom,sc7280-smmu-500
- qcom,sc8180x-smmu-500
- qcom,sc8280xp-smmu-500
- qcom,sdm670-smmu-500
- qcom,sdm845-smmu-500
- qcom,sm6115-smmu-500
- qcom,sm6350-smmu-500
- qcom,sm6375-smmu-500
- qcom,sm8150-smmu-500
- qcom,sm8250-smmu-500
- qcom,sm8350-smmu-500
- qcom,sm8450-smmu-500
- const: qcom,smmu-500
- const: arm,mmu-500
- description: Qcom SoCs implementing "arm,mmu-500" (non-qcom implementation)
deprecated: true
items:
- enum:
- qcom,sdx55-smmu-500
- qcom,sdx65-smmu-500
- const: arm,mmu-500
- description: Qcom SoCs implementing "arm,mmu-500" (legacy binding)
deprecated: true
items:
# Do not add additional SoC to this list. Instead use two previous lists.
- enum:
- qcom,qcm2290-smmu-500
- qcom,sc7180-smmu-500
@ -39,8 +71,7 @@ properties:
- qcom,sc8180x-smmu-500
- qcom,sc8280xp-smmu-500
- qcom,sdm845-smmu-500
- qcom,sdx55-smmu-500
- qcom,sdx65-smmu-500
- qcom,sm6115-smmu-500
- qcom,sm6350-smmu-500
- qcom,sm6375-smmu-500
- qcom,sm8150-smmu-500
@ -48,13 +79,28 @@ properties:
- qcom,sm8350-smmu-500
- qcom,sm8450-smmu-500
- const: arm,mmu-500
- description: Qcom Adreno GPUs implementing "arm,smmu-500"
items:
- enum:
- qcom,sc7280-smmu-500
- qcom,sm8250-smmu-500
- const: qcom,adreno-smmu
- const: arm,mmu-500
- description: Qcom Adreno GPUs implementing "arm,smmu-v2"
items:
- enum:
- qcom,msm8996-smmu-v2
- qcom,sc7180-smmu-v2
- qcom,sdm630-smmu-v2
- qcom,sdm845-smmu-v2
- qcom,sm6350-smmu-v2
- const: qcom,adreno-smmu
- const: qcom,smmu-v2
- description: Qcom Adreno GPUs on Google Cheza platform
items:
- const: qcom,sdm845-smmu-v2
- const: qcom,smmu-v2
- description: Marvell SoCs implementing "arm,mmu-500"
items:
- const: marvell,ap806-smmu-500
@ -147,16 +193,12 @@ properties:
present in such cases.
clock-names:
items:
- const: bus
- const: iface
minItems: 1
maxItems: 7
clocks:
items:
- description: bus clock required for downstream bus access and for the
smmu ptw
- description: interface clock required to access smmu's registers
through the TCU's programming interface.
minItems: 1
maxItems: 7
power-domains:
maxItems: 1
@ -206,6 +248,124 @@ allOf:
reg:
maxItems: 1
- if:
properties:
compatible:
contains:
enum:
- qcom,msm8998-smmu-v2
- qcom,sdm630-smmu-v2
then:
anyOf:
- properties:
clock-names:
items:
- const: bus
clocks:
items:
- description: bus clock required for downstream bus access and for
the smmu ptw
- properties:
clock-names:
items:
- const: iface
- const: mem
- const: mem_iface
clocks:
items:
- description: interface clock required to access smmu's registers
through the TCU's programming interface.
- description: bus clock required for memory access
- description: bus clock required for GPU memory access
- properties:
clock-names:
items:
- const: iface-mm
- const: iface-smmu
- const: bus-mm
- const: bus-smmu
clocks:
items:
- description: interface clock required to access mnoc's registers
through the TCU's programming interface.
- description: interface clock required to access smmu's registers
through the TCU's programming interface.
- description: bus clock required for downstream bus access
- description: bus clock required for the smmu ptw
- if:
properties:
compatible:
contains:
enum:
- qcom,msm8996-smmu-v2
- qcom,sc7180-smmu-v2
- qcom,sdm845-smmu-v2
then:
properties:
clock-names:
items:
- const: bus
- const: iface
clocks:
items:
- description: bus clock required for downstream bus access and for
the smmu ptw
- description: interface clock required to access smmu's registers
through the TCU's programming interface.
- if:
properties:
compatible:
contains:
const: qcom,sc7280-smmu-500
then:
properties:
clock-names:
items:
- const: gcc_gpu_memnoc_gfx_clk
- const: gcc_gpu_snoc_dvm_gfx_clk
- const: gpu_cc_ahb_clk
- const: gpu_cc_hlos1_vote_gpu_smmu_clk
- const: gpu_cc_cx_gmu_clk
- const: gpu_cc_hub_cx_int_clk
- const: gpu_cc_hub_aon_clk
clocks:
items:
- description: GPU memnoc_gfx clock
- description: GPU snoc_dvm_gfx clock
- description: GPU ahb clock
- description: GPU hlos1_vote_GPU smmu clock
- description: GPU cx_gmu clock
- description: GPU hub_cx_int clock
- description: GPU hub_aon clock
- if:
properties:
compatible:
contains:
enum:
- qcom,sm6350-smmu-v2
- qcom,sm8150-smmu-500
- qcom,sm8250-smmu-500
then:
properties:
clock-names:
items:
- const: ahb
- const: bus
- const: iface
clocks:
items:
- description: bus clock required for AHB bus access
- description: bus clock required for downstream bus access and for
the smmu ptw
- description: interface clock required to access smmu's registers
through the TCU's programming interface.
examples:
- |+
/* SMMU with stream matching or stream indexing */

2
Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml
View File

@ -82,6 +82,7 @@ properties:
- mediatek,mt8195-iommu-vdo # generation two
- mediatek,mt8195-iommu-vpp # generation two
- mediatek,mt8195-iommu-infra # generation two
- mediatek,mt8365-m4u # generation two
- description: mt7623 generation one
items:
@ -132,6 +133,7 @@ properties:
dt-binding/memory/mt8186-memory-port.h for mt8186,
dt-binding/memory/mt8192-larb-port.h for mt8192.
dt-binding/memory/mt8195-memory-port.h for mt8195.
dt-binding/memory/mediatek,mt8365-larb-port.h for mt8365.
power-domains:
maxItems: 1

5
arch/s390/include/asm/pci.h
View File

@ -117,7 +117,9 @@ struct zpci_bus {
struct zpci_dev {
struct zpci_bus *zbus;
struct list_head entry; /* list of all zpci_devices, needed for hotplug, etc. */
struct list_head iommu_list;
struct kref kref;
struct rcu_head rcu;
struct hotplug_slot hotplug_slot;
enum zpci_state state;
@ -155,7 +157,6 @@ struct zpci_dev {
/* DMA stuff */
unsigned long *dma_table;
spinlock_t dma_table_lock;
int tlb_refresh;
spinlock_t iommu_bitmap_lock;
@ -220,7 +221,7 @@ void zpci_device_reserved(struct zpci_dev *zdev);
bool zpci_is_device_configured(struct zpci_dev *zdev);
int zpci_hot_reset_device(struct zpci_dev *zdev);
int zpci_register_ioat(struct zpci_dev *, u8, u64, u64, u64);
int zpci_register_ioat(struct zpci_dev *, u8, u64, u64, u64, u8 *);
int zpci_unregister_ioat(struct zpci_dev *, u8);
void zpci_remove_reserved_devices(void);
void zpci_update_fh(struct zpci_dev *zdev, u32 fh);

6
arch/s390/kvm/pci.c
View File

@ -434,6 +434,7 @@ static void kvm_s390_pci_dev_release(struct zpci_dev *zdev)
static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm)
{
struct zpci_dev *zdev = opaque;
u8 status;
int rc;
if (!zdev)
@ -486,7 +487,7 @@ static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm)
/* Re-register the IOMMU that was already created */
rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table));
virt_to_phys(zdev->dma_table), &status);
if (rc)
goto clear_gisa;
@ -516,6 +517,7 @@ static void kvm_s390_pci_unregister_kvm(void *opaque)
{
struct zpci_dev *zdev = opaque;
struct kvm *kvm;
u8 status;
if (!zdev)
return;
@ -554,7 +556,7 @@ static void kvm_s390_pci_unregister_kvm(void *opaque)
/* Re-register the IOMMU that was already created */
zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table));
virt_to_phys(zdev->dma_table), &status);
out:
spin_lock(&kvm->arch.kzdev_list_lock);

13
arch/s390/pci/pci.c
View File

@ -116,20 +116,20 @@ EXPORT_SYMBOL_GPL(pci_proc_domain);
/* Modify PCI: Register I/O address translation parameters */
int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
u64 base, u64 limit, u64 iota)
u64 base, u64 limit, u64 iota, u8 *status)
{
u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT);
struct zpci_fib fib = {0};
u8 cc, status;
u8 cc;
WARN_ON_ONCE(iota & 0x3fff);
fib.pba = base;
fib.pal = limit;
fib.iota = iota | ZPCI_IOTA_RTTO_FLAG;
fib.gd = zdev->gisa;
cc = zpci_mod_fc(req, &fib, &status);
cc = zpci_mod_fc(req, &fib, status);
if (cc)
zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status);
zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, *status);
return cc;
}
EXPORT_SYMBOL_GPL(zpci_register_ioat);
@ -764,6 +764,7 @@ EXPORT_SYMBOL_GPL(zpci_disable_device);
*/
int zpci_hot_reset_device(struct zpci_dev *zdev)
{
u8 status;
int rc;
zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh);
@ -787,7 +788,7 @@ int zpci_hot_reset_device(struct zpci_dev *zdev)
if (zdev->dma_table)
rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table));
virt_to_phys(zdev->dma_table), &status);
else
rc = zpci_dma_init_device(zdev);
if (rc) {
@ -995,7 +996,7 @@ void zpci_release_device(struct kref *kref)
break;
}
zpci_dbg(3, "rem fid:%x\n", zdev->fid);
kfree(zdev);
kfree_rcu(zdev, rcu);
}
int zpci_report_error(struct pci_dev *pdev,

77
arch/s390/pci/pci_dma.c
View File

@ -63,37 +63,55 @@ static void dma_free_page_table(void *table)
kmem_cache_free(dma_page_table_cache, table);
}
static unsigned long *dma_get_seg_table_origin(unsigned long *entry)
static unsigned long *dma_get_seg_table_origin(unsigned long *rtep)
{
unsigned long old_rte, rte;
unsigned long *sto;
if (reg_entry_isvalid(*entry))
sto = get_rt_sto(*entry);
else {
rte = READ_ONCE(*rtep);
if (reg_entry_isvalid(rte)) {
sto = get_rt_sto(rte);
} else {
sto = dma_alloc_cpu_table();
if (!sto)
return NULL;
set_rt_sto(entry, virt_to_phys(sto));
validate_rt_entry(entry);
entry_clr_protected(entry);
set_rt_sto(&rte, virt_to_phys(sto));
validate_rt_entry(&rte);
entry_clr_protected(&rte);
old_rte = cmpxchg(rtep, ZPCI_TABLE_INVALID, rte);
if (old_rte != ZPCI_TABLE_INVALID) {
/* Somone else was faster, use theirs */
dma_free_cpu_table(sto);
sto = get_rt_sto(old_rte);
}
}
return sto;
}
static unsigned long *dma_get_page_table_origin(unsigned long *entry)
static unsigned long *dma_get_page_table_origin(unsigned long *step)
{
unsigned long old_ste, ste;
unsigned long *pto;
if (reg_entry_isvalid(*entry))
pto = get_st_pto(*entry);
else {
ste = READ_ONCE(*step);
if (reg_entry_isvalid(ste)) {
pto = get_st_pto(ste);
} else {
pto = dma_alloc_page_table();
if (!pto)
return NULL;
set_st_pto(entry, virt_to_phys(pto));
validate_st_entry(entry);
entry_clr_protected(entry);
set_st_pto(&ste, virt_to_phys(pto));
validate_st_entry(&ste);
entry_clr_protected(&ste);
old_ste = cmpxchg(step, ZPCI_TABLE_INVALID, ste);
if (old_ste != ZPCI_TABLE_INVALID) {
/* Somone else was faster, use theirs */
dma_free_page_table(pto);
pto = get_st_pto(old_ste);
}
}
return pto;
}
@ -117,19 +135,24 @@ unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr)
return &pto[px];
}
void dma_update_cpu_trans(unsigned long *entry, phys_addr_t page_addr, int flags)
void dma_update_cpu_trans(unsigned long *ptep, phys_addr_t page_addr, int flags)
{
unsigned long pte;
pte = READ_ONCE(*ptep);
if (flags & ZPCI_PTE_INVALID) {
invalidate_pt_entry(entry);
invalidate_pt_entry(&pte);
} else {
set_pt_pfaa(entry, page_addr);
validate_pt_entry(entry);
set_pt_pfaa(&pte, page_addr);
validate_pt_entry(&pte);
}
if (flags & ZPCI_TABLE_PROTECTED)
entry_set_protected(entry);
entry_set_protected(&pte);
else
entry_clr_protected(entry);
entry_clr_protected(&pte);
xchg(ptep, pte);
}
static int __dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa,
@ -137,18 +160,14 @@ static int __dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa,
{
unsigned int nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
phys_addr_t page_addr = (pa & PAGE_MASK);
unsigned long irq_flags;
unsigned long *entry;
int i, rc = 0;
if (!nr_pages)
return -EINVAL;
spin_lock_irqsave(&zdev->dma_table_lock, irq_flags);
if (!zdev->dma_table) {
rc = -EINVAL;
goto out_unlock;
}
if (!zdev->dma_table)
return -EINVAL;
for (i = 0; i < nr_pages; i++) {
entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr);
@ -173,8 +192,6 @@ static int __dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa,
dma_update_cpu_trans(entry, page_addr, flags);
}
}
out_unlock:
spin_unlock_irqrestore(&zdev->dma_table_lock, irq_flags);
return rc;
}
@ -547,6 +564,7 @@ static void s390_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
int zpci_dma_init_device(struct zpci_dev *zdev)
{
u8 status;
int rc;
/*
@ -557,7 +575,6 @@ int zpci_dma_init_device(struct zpci_dev *zdev)
WARN_ON(zdev->s390_domain);
spin_lock_init(&zdev->iommu_bitmap_lock);
spin_lock_init(&zdev->dma_table_lock);
zdev->dma_table = dma_alloc_cpu_table();
if (!zdev->dma_table) {
@ -598,7 +615,7 @@ int zpci_dma_init_device(struct zpci_dev *zdev)
}
if (zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table))) {
virt_to_phys(zdev->dma_table), &status)) {
rc = -EIO;
goto free_bitmap;
}

3
drivers/dma/idxd/cdev.c
View File

@ -6,7 +6,6 @@
#include <linux/pci.h>
#include <linux/device.h>
#include <linux/sched/task.h>
#include <linux/intel-svm.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/cdev.h>
#include <linux/fs.h>
@ -100,7 +99,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp)
filp->private_data = ctx;
if (device_user_pasid_enabled(idxd)) {
sva = iommu_sva_bind_device(dev, current->mm, NULL);
sva = iommu_sva_bind_device(dev, current->mm);
if (IS_ERR(sva)) {
rc = PTR_ERR(sva);
dev_err(dev, "pasid allocation failed: %d\n", rc);

25
drivers/dma/idxd/init.c
View File

@ -14,7 +14,6 @@
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/device.h>
#include <linux/idr.h>
#include <linux/intel-svm.h>
#include <linux/iommu.h>
#include <uapi/linux/idxd.h>
#include <linux/dmaengine.h>
@ -502,29 +501,7 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_d
static int idxd_enable_system_pasid(struct idxd_device *idxd)
{
int flags;
unsigned int pasid;
struct iommu_sva *sva;
flags = SVM_FLAG_SUPERVISOR_MODE;
sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags);
if (IS_ERR(sva)) {
dev_warn(&idxd->pdev->dev,
"iommu sva bind failed: %ld\n", PTR_ERR(sva));
return PTR_ERR(sva);
}
pasid = iommu_sva_get_pasid(sva);
if (pasid == IOMMU_PASID_INVALID) {
iommu_sva_unbind_device(sva);
return -ENODEV;
}
idxd->sva = sva;
idxd->pasid = pasid;
dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid);
return 0;
return -EOPNOTSUPP;
}
static void idxd_disable_system_pasid(struct idxd_device *idxd)

2
drivers/iommu/Makefile
View File

@ -28,6 +28,6 @@ obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o
obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o
obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o
obj-$(CONFIG_IOMMU_SVA) += iommu-sva-lib.o io-pgfault.o
obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o io-pgfault.o
obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
obj-$(CONFIG_APPLE_DART) += apple-dart.o

86
drivers/iommu/amd/init.c
View File

@ -85,7 +85,7 @@
#define LOOP_TIMEOUT 2000000
#define IVRS_GET_SBDF_ID(seg, bus, dev, fd) (((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \
#define IVRS_GET_SBDF_ID(seg, bus, dev, fn) (((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \
| ((dev & 0x1f) << 3) | (fn & 0x7))
/*
@ -3402,18 +3402,24 @@ static int __init parse_amd_iommu_options(char *str)
static int __init parse_ivrs_ioapic(char *str)
{
u32 seg = 0, bus, dev, fn;
int ret, id, i;
int id, i;
u32 devid;
ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);
if (ret != 4) {
ret = sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn);
if (ret != 5) {
pr_err("Invalid command line: ivrs_ioapic%s\n", str);
return 1;
}
if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
goto found;
if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
pr_warn("ivrs_ioapic%s option format deprecated; use ivrs_ioapic=%d@%04x:%02x:%02x.%d instead\n",
str, id, seg, bus, dev, fn);
goto found;
}
pr_err("Invalid command line: ivrs_ioapic%s\n", str);
return 1;
found:
if (early_ioapic_map_size == EARLY_MAP_SIZE) {
pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",
str);
@ -3434,18 +3440,24 @@ static int __init parse_ivrs_ioapic(char *str)
static int __init parse_ivrs_hpet(char *str)
{
u32 seg = 0, bus, dev, fn;
int ret, id, i;
int id, i;
u32 devid;
ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);
if (ret != 4) {
ret = sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn);
if (ret != 5) {
pr_err("Invalid command line: ivrs_hpet%s\n", str);
return 1;
}
if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
goto found;
if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
pr_warn("ivrs_hpet%s option format deprecated; use ivrs_hpet=%d@%04x:%02x:%02x.%d instead\n",
str, id, seg, bus, dev, fn);
goto found;
}
pr_err("Invalid command line: ivrs_hpet%s\n", str);
return 1;
found:
if (early_hpet_map_size == EARLY_MAP_SIZE) {
pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n",
str);
@ -3466,19 +3478,36 @@ static int __init parse_ivrs_hpet(char *str)
static int __init parse_ivrs_acpihid(char *str)
{
u32 seg = 0, bus, dev, fn;
char *hid, *uid, *p;
char *hid, *uid, *p, *addr;
char acpiid[ACPIHID_UID_LEN + ACPIHID_HID_LEN] = {0};
int ret, i;
int i;
ret = sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid);
if (ret != 4) {
ret = sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid);
if (ret != 5) {
pr_err("Invalid command line: ivrs_acpihid(%s)\n", str);
return 1;
addr = strchr(str, '@');
if (!addr) {
if (sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid) == 4 ||
sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid) == 5) {
pr_warn("ivrs_acpihid%s option format deprecated; use ivrs_acpihid=%s@%04x:%02x:%02x.%d instead\n",
str, acpiid, seg, bus, dev, fn);
goto found;
}
goto not_found;
}
/* We have the '@', make it the terminator to get just the acpiid */
*addr++ = 0;
if (sscanf(str, "=%s", acpiid) != 1)
goto not_found;
if (sscanf(addr, "%x:%x.%x", &bus, &dev, &fn) == 3 ||
sscanf(addr, "%x:%x:%x.%x", &seg, &bus, &dev, &fn) == 4)
goto found;
not_found:
pr_err("Invalid command line: ivrs_acpihid%s\n", str);
return 1;
found:
p = acpiid;
hid = strsep(&p, ":");
uid = p;
@ -3488,6 +3517,13 @@ static int __init parse_ivrs_acpihid(char *str)
return 1;
}
/*
* Ignore leading zeroes after ':', so e.g., AMDI0095:00
* will match AMDI0095:0 in the second strcmp in acpi_dev_hid_uid_match
*/
while (*uid == '0' && *(uid + 1))
uid++;
i = early_acpihid_map_size++;
memcpy(early_acpihid_map[i].hid, hid, strlen(hid));
memcpy(early_acpihid_map[i].uid, uid, strlen(uid));

15
drivers/iommu/amd/iommu.c
View File

@ -767,7 +767,7 @@ EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier);
static void iommu_poll_ga_log(struct amd_iommu *iommu)
{
u32 head, tail, cnt = 0;
u32 head, tail;
if (iommu->ga_log == NULL)
return;
@ -780,7 +780,6 @@ static void iommu_poll_ga_log(struct amd_iommu *iommu)
u64 log_entry;
raw = (u64 *)(iommu->ga_log + head);
cnt++;
/* Avoid memcpy function-call overhead */
log_entry = *raw;
@ -2155,21 +2154,13 @@ static void amd_iommu_detach_device(struct iommu_domain *dom,
static int amd_iommu_attach_device(struct iommu_domain *dom,
struct device *dev)
{
struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
struct protection_domain *domain = to_pdomain(dom);
struct iommu_dev_data *dev_data;
struct amd_iommu *iommu;
struct amd_iommu *iommu = rlookup_amd_iommu(dev);
int ret;
if (!check_device(dev))
return -EINVAL;
dev_data = dev_iommu_priv_get(dev);
dev_data->defer_attach = false;
iommu = rlookup_amd_iommu(dev);
if (!iommu)
return -EINVAL;
if (dev_data->domain)
detach_device(dev);

5
drivers/iommu/amd/iommu_v2.c
View File

@ -587,6 +587,7 @@ static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
put_device_state(dev_state);
out:
pci_dev_put(pdev);
return ret;
}
@ -639,7 +640,9 @@ int amd_iommu_bind_pasid(struct pci_dev *pdev, u32 pasid,
if (pasid_state->mm == NULL)
goto out_free;
mmu_notifier_register(&pasid_state->mn, mm);
ret = mmu_notifier_register(&pasid_state->mn, mm);
if (ret)
goto out_free;
ret = set_pasid_state(dev_state, pasid_state, pasid);
if (ret)

104
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
View File

@ -10,7 +10,7 @@
#include <linux/slab.h>
#include "arm-smmu-v3.h"
#include "../../iommu-sva-lib.h"
#include "../../iommu-sva.h"
#include "../../io-pgtable-arm.h"
struct arm_smmu_mmu_notifier {
@ -344,11 +344,6 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
if (!bond)
return ERR_PTR(-ENOMEM);
/* Allocate a PASID for this mm if necessary */
ret = iommu_sva_alloc_pasid(mm, 1, (1U << master->ssid_bits) - 1);
if (ret)
goto err_free_bond;
bond->mm = mm;
bond->sva.dev = dev;
refcount_set(&bond->refs, 1);
@ -367,42 +362,6 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
return ERR_PTR(ret);
}
struct iommu_sva *
arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
{
struct iommu_sva *handle;
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
return ERR_PTR(-EINVAL);
mutex_lock(&sva_lock);
handle = __arm_smmu_sva_bind(dev, mm);
mutex_unlock(&sva_lock);
return handle;
}
void arm_smmu_sva_unbind(struct iommu_sva *handle)
{
struct arm_smmu_bond *bond = sva_to_bond(handle);
mutex_lock(&sva_lock);
if (refcount_dec_and_test(&bond->refs)) {
list_del(&bond->list);
arm_smmu_mmu_notifier_put(bond->smmu_mn);
kfree(bond);
}
mutex_unlock(&sva_lock);
}
u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle)
{
struct arm_smmu_bond *bond = sva_to_bond(handle);
return bond->mm->pasid;
}
bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
{
unsigned long reg, fld;
@ -550,3 +509,64 @@ void arm_smmu_sva_notifier_synchronize(void)
*/
mmu_notifier_synchronize();
}
void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t id)
{
struct mm_struct *mm = domain->mm;
struct arm_smmu_bond *bond = NULL, *t;
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
mutex_lock(&sva_lock);
list_for_each_entry(t, &master->bonds, list) {
if (t->mm == mm) {
bond = t;
break;
}
}
if (!WARN_ON(!bond) && refcount_dec_and_test(&bond->refs)) {
list_del(&bond->list);
arm_smmu_mmu_notifier_put(bond->smmu_mn);
kfree(bond);
}
mutex_unlock(&sva_lock);
}
static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t id)
{
int ret = 0;
struct iommu_sva *handle;
struct mm_struct *mm = domain->mm;
mutex_lock(&sva_lock);
handle = __arm_smmu_sva_bind(dev, mm);
if (IS_ERR(handle))
ret = PTR_ERR(handle);
mutex_unlock(&sva_lock);
return ret;
}
static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
{
kfree(domain);
}
static const struct iommu_domain_ops arm_smmu_sva_domain_ops = {
.set_dev_pasid = arm_smmu_sva_set_dev_pasid,
.free = arm_smmu_sva_domain_free
};
struct iommu_domain *arm_smmu_sva_domain_alloc(void)
{
struct iommu_domain *domain;
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
if (!domain)
return NULL;
domain->ops = &arm_smmu_sva_domain_ops;
return domain;
}

32
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
View File

@ -29,7 +29,7 @@
#include "arm-smmu-v3.h"
#include "../../dma-iommu.h"
#include "../../iommu-sva-lib.h"
#include "../../iommu-sva.h"
static bool disable_bypass = true;
module_param(disable_bypass, bool, 0444);
@ -2009,6 +2009,9 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
{
struct arm_smmu_domain *smmu_domain;
if (type == IOMMU_DOMAIN_SVA)
return arm_smmu_sva_domain_alloc();
if (type != IOMMU_DOMAIN_UNMANAGED &&
type != IOMMU_DOMAIN_DMA &&
type != IOMMU_DOMAIN_DMA_FQ &&
@ -2430,23 +2433,14 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
goto out_unlock;
}
} else if (smmu_domain->smmu != smmu) {
dev_err(dev,
"cannot attach to SMMU %s (upstream of %s)\n",
dev_name(smmu_domain->smmu->dev),
dev_name(smmu->dev));
ret = -ENXIO;
ret = -EINVAL;
goto out_unlock;
} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
dev_err(dev,
"cannot attach to incompatible domain (%u SSID bits != %u)\n",
smmu_domain->s1_cfg.s1cdmax, master->ssid_bits);
ret = -EINVAL;
goto out_unlock;
} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
smmu_domain->stall_enabled != master->stall_enabled) {
dev_err(dev, "cannot attach to stall-%s domain\n",
smmu_domain->stall_enabled ? "enabled" : "disabled");
ret = -EINVAL;
goto out_unlock;
}
@ -2838,6 +2832,17 @@ static int arm_smmu_def_domain_type(struct device *dev)
return 0;
}
static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
{
struct iommu_domain *domain;
domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA);
if (WARN_ON(IS_ERR(domain)) || !domain)
return;
arm_smmu_sva_remove_dev_pasid(domain, dev, pasid);
}
static struct iommu_ops arm_smmu_ops = {
.capable = arm_smmu_capable,
.domain_alloc = arm_smmu_domain_alloc,
@ -2846,11 +2851,9 @@ static struct iommu_ops arm_smmu_ops = {
.device_group = arm_smmu_device_group,
.of_xlate = arm_smmu_of_xlate,
.get_resv_regions = arm_smmu_get_resv_regions,
.remove_dev_pasid = arm_smmu_remove_dev_pasid,
.dev_enable_feat = arm_smmu_dev_enable_feature,
.dev_disable_feat = arm_smmu_dev_disable_feature,
.sva_bind = arm_smmu_sva_bind,
.sva_unbind = arm_smmu_sva_unbind,
.sva_get_pasid = arm_smmu_sva_get_pasid,
.page_response = arm_smmu_page_response,
.def_domain_type = arm_smmu_def_domain_type,
.pgsize_bitmap = -1UL, /* Restricted during device attach */
@ -3543,6 +3546,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
/* SID/SSID sizes */
smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
/*
* If the SMMU supports fewer bits than would fill a single L2 stream

31
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
View File

@ -754,11 +754,10 @@ bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);
int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master);
struct iommu_sva *arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm,
void *drvdata);
void arm_smmu_sva_unbind(struct iommu_sva *handle);
u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle);
void arm_smmu_sva_notifier_synchronize(void);
struct iommu_domain *arm_smmu_sva_domain_alloc(void);
void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t id);
#else /* CONFIG_ARM_SMMU_V3_SVA */
static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
{
@ -790,19 +789,17 @@ static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master
return false;
}
static inline struct iommu_sva *
arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
{
return ERR_PTR(-ENODEV);
}
static inline void arm_smmu_sva_unbind(struct iommu_sva *handle) {}
static inline u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle)
{
return IOMMU_PASID_INVALID;
}
static inline void arm_smmu_sva_notifier_synchronize(void) {}
static inline struct iommu_domain *arm_smmu_sva_domain_alloc(void)
{
return NULL;
}
static inline void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
struct device *dev,
ioasid_t id)
{
}
#endif /* CONFIG_ARM_SMMU_V3_SVA */
#endif /* _ARM_SMMU_V3_H */

3
drivers/iommu/arm/arm-smmu/arm-smmu-impl.c
View File

@ -136,6 +136,9 @@ int arm_mmu500_reset(struct arm_smmu_device *smmu)
reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
reg &= ~ARM_MMU500_ACTLR_CPRE;
arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_ACTLR, reg);
reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
if (reg & ARM_MMU500_ACTLR_CPRE)
dev_warn_once(smmu->dev, "Failed to disable prefetcher [errata #841119 and #826419], check ACR.CACHE_LOCK\n");
}
return 0;

91
drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c
View File

@ -10,16 +10,6 @@
#include "arm-smmu.h"
#include "arm-smmu-qcom.h"
enum qcom_smmu_impl_reg_offset {
QCOM_SMMU_TBU_PWR_STATUS,
QCOM_SMMU_STATS_SYNC_INV_TBU_ACK,
QCOM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR,
};
struct qcom_smmu_config {
const u32 *reg_offset;
};
void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu)
{
int ret;
@ -59,84 +49,3 @@ void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu)
tbu_pwr_status, sync_inv_ack, sync_inv_progress);
}
}
/* Implementation Defined Register Space 0 register offsets */
static const u32 qcom_smmu_impl0_reg_offset[] = {
[QCOM_SMMU_TBU_PWR_STATUS] = 0x2204,
[QCOM_SMMU_STATS_SYNC_INV_TBU_ACK] = 0x25dc,
[QCOM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR] = 0x2670,
};
static const struct qcom_smmu_config qcm2290_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sc7180_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sc7280_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sc8180x_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sc8280xp_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm6125_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm6350_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm8150_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm8250_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm8350_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm8450_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct of_device_id __maybe_unused qcom_smmu_impl_debug_match[] = {
{ .compatible = "qcom,msm8998-smmu-v2" },
{ .compatible = "qcom,qcm2290-smmu-500", .data = &qcm2290_smmu_cfg },
{ .compatible = "qcom,sc7180-smmu-500", .data = &sc7180_smmu_cfg },
{ .compatible = "qcom,sc7280-smmu-500", .data = &sc7280_smmu_cfg},
{ .compatible = "qcom,sc8180x-smmu-500", .data = &sc8180x_smmu_cfg },
{ .compatible = "qcom,sc8280xp-smmu-500", .data = &sc8280xp_smmu_cfg },
{ .compatible = "qcom,sdm630-smmu-v2" },
{ .compatible = "qcom,sdm845-smmu-500" },
{ .compatible = "qcom,sm6125-smmu-500", .data = &sm6125_smmu_cfg},
{ .compatible = "qcom,sm6350-smmu-500", .data = &sm6350_smmu_cfg},
{ .compatible = "qcom,sm8150-smmu-500", .data = &sm8150_smmu_cfg },
{ .compatible = "qcom,sm8250-smmu-500", .data = &sm8250_smmu_cfg },
{ .compatible = "qcom,sm8350-smmu-500", .data = &sm8350_smmu_cfg },
{ .compatible = "qcom,sm8450-smmu-500", .data = &sm8450_smmu_cfg },
{ }
};
const void *qcom_smmu_impl_data(struct arm_smmu_device *smmu)
{
const struct of_device_id *match;
const struct device_node *np = smmu->dev->of_node;
match = of_match_node(qcom_smmu_impl_debug_match, np);
if (!match)
return NULL;
return match->data;
}

161
drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
View File

@ -361,6 +361,8 @@ static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
{
int ret;
arm_mmu500_reset(smmu);
/*
* To address performance degradation in non-real time clients,
* such as USB and UFS, turn off wait-for-safe on sdm845 based boards,
@ -374,41 +376,67 @@ static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
return ret;
}
static int qcom_smmu500_reset(struct arm_smmu_device *smmu)
{
const struct device_node *np = smmu->dev->of_node;
arm_mmu500_reset(smmu);
if (of_device_is_compatible(np, "qcom,sdm845-smmu-500"))
return qcom_sdm845_smmu500_reset(smmu);
return 0;
}
static const struct arm_smmu_impl qcom_smmu_impl = {
static const struct arm_smmu_impl qcom_smmu_v2_impl = {
.init_context = qcom_smmu_init_context,
.cfg_probe = qcom_smmu_cfg_probe,
.def_domain_type = qcom_smmu_def_domain_type,
.reset = qcom_smmu500_reset,
.write_s2cr = qcom_smmu_write_s2cr,
.tlb_sync = qcom_smmu_tlb_sync,
};
static const struct arm_smmu_impl qcom_adreno_smmu_impl = {
static const struct arm_smmu_impl qcom_smmu_500_impl = {
.init_context = qcom_smmu_init_context,
.cfg_probe = qcom_smmu_cfg_probe,
.def_domain_type = qcom_smmu_def_domain_type,
.reset = arm_mmu500_reset,
.write_s2cr = qcom_smmu_write_s2cr,
.tlb_sync = qcom_smmu_tlb_sync,
};
static const struct arm_smmu_impl sdm845_smmu_500_impl = {
.init_context = qcom_smmu_init_context,
.cfg_probe = qcom_smmu_cfg_probe,
.def_domain_type = qcom_smmu_def_domain_type,
.reset = qcom_sdm845_smmu500_reset,
.write_s2cr = qcom_smmu_write_s2cr,
.tlb_sync = qcom_smmu_tlb_sync,
};
static const struct arm_smmu_impl qcom_adreno_smmu_v2_impl = {
.init_context = qcom_adreno_smmu_init_context,
.def_domain_type = qcom_smmu_def_domain_type,
.reset = qcom_smmu500_reset,
.alloc_context_bank = qcom_adreno_smmu_alloc_context_bank,
.write_sctlr = qcom_adreno_smmu_write_sctlr,
.tlb_sync = qcom_smmu_tlb_sync,
};
static const struct arm_smmu_impl qcom_adreno_smmu_500_impl = {
.init_context = qcom_adreno_smmu_init_context,
.def_domain_type = qcom_smmu_def_domain_type,
.reset = arm_mmu500_reset,
.alloc_context_bank = qcom_adreno_smmu_alloc_context_bank,
.write_sctlr = qcom_adreno_smmu_write_sctlr,
.tlb_sync = qcom_smmu_tlb_sync,
};
static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu,
const struct arm_smmu_impl *impl)
const struct qcom_smmu_match_data *data)
{
const struct device_node *np = smmu->dev->of_node;
const struct arm_smmu_impl *impl;
struct qcom_smmu *qsmmu;
if (!data)
return ERR_PTR(-EINVAL);
if (np && of_device_is_compatible(np, "qcom,adreno-smmu"))
impl = data->adreno_impl;
else
impl = data->impl;
if (!impl)
return smmu;
/* Check to make sure qcom_scm has finished probing */
if (!qcom_scm_is_available())
return ERR_PTR(-EPROBE_DEFER);
@ -418,27 +446,77 @@ static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu,
return ERR_PTR(-ENOMEM);
qsmmu->smmu.impl = impl;
qsmmu->cfg = qcom_smmu_impl_data(smmu);
qsmmu->cfg = data->cfg;
return &qsmmu->smmu;
}
/* Implementation Defined Register Space 0 register offsets */
static const u32 qcom_smmu_impl0_reg_offset[] = {
[QCOM_SMMU_TBU_PWR_STATUS] = 0x2204,
[QCOM_SMMU_STATS_SYNC_INV_TBU_ACK] = 0x25dc,
[QCOM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR] = 0x2670,
};
static const struct qcom_smmu_config qcom_smmu_impl0_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
/*
* It is not yet possible to use MDP SMMU with the bypass quirk on the msm8996,
* there are not enough context banks.
*/
static const struct qcom_smmu_match_data msm8996_smmu_data = {
.impl = NULL,
.adreno_impl = &qcom_adreno_smmu_v2_impl,
};
static const struct qcom_smmu_match_data qcom_smmu_v2_data = {
.impl = &qcom_smmu_v2_impl,
.adreno_impl = &qcom_adreno_smmu_v2_impl,
};
static const struct qcom_smmu_match_data sdm845_smmu_500_data = {
.impl = &sdm845_smmu_500_impl,
/*
* No need for adreno impl here. On sdm845 the Adreno SMMU is handled
* by the separate sdm845-smmu-v2 device.
*/
/* Also no debug configuration. */
};
static const struct qcom_smmu_match_data qcom_smmu_500_impl0_data = {
.impl = &qcom_smmu_500_impl,
.adreno_impl = &qcom_adreno_smmu_500_impl,
.cfg = &qcom_smmu_impl0_cfg,
};
/*
* Do not add any more qcom,SOC-smmu-500 entries to this list, unless they need
* special handling and can not be covered by the qcom,smmu-500 entry.
*/
static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = {
{ .compatible = "qcom,msm8998-smmu-v2" },
{ .compatible = "qcom,qcm2290-smmu-500" },
{ .compatible = "qcom,sc7180-smmu-500" },
{ .compatible = "qcom,sc7280-smmu-500" },
{ .compatible = "qcom,sc8180x-smmu-500" },
{ .compatible = "qcom,sc8280xp-smmu-500" },
{ .compatible = "qcom,sdm630-smmu-v2" },
{ .compatible = "qcom,sdm845-smmu-500" },
{ .compatible = "qcom,sm6125-smmu-500" },
{ .compatible = "qcom,sm6350-smmu-500" },
{ .compatible = "qcom,sm6375-smmu-500" },
{ .compatible = "qcom,sm8150-smmu-500" },
{ .compatible = "qcom,sm8250-smmu-500" },
{ .compatible = "qcom,sm8350-smmu-500" },
{ .compatible = "qcom,sm8450-smmu-500" },
{ .compatible = "qcom,msm8996-smmu-v2", .data = &msm8996_smmu_data },
{ .compatible = "qcom,msm8998-smmu-v2", .data = &qcom_smmu_v2_data },
{ .compatible = "qcom,qcm2290-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,qdu1000-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sc7180-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sc7280-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sc8180x-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sc8280xp-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sdm630-smmu-v2", .data = &qcom_smmu_v2_data },
{ .compatible = "qcom,sdm845-smmu-v2", .data = &qcom_smmu_v2_data },
{ .compatible = "qcom,sdm845-smmu-500", .data = &sdm845_smmu_500_data },
{ .compatible = "qcom,sm6115-smmu-500", .data = &qcom_smmu_500_impl0_data},
{ .compatible = "qcom,sm6125-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm6350-smmu-v2", .data = &qcom_smmu_v2_data },
{ .compatible = "qcom,sm6350-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm6375-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm8150-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm8250-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm8350-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm8450-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,smmu-500", .data = &qcom_smmu_500_impl0_data },
{ }
};
@ -453,26 +531,19 @@ static struct acpi_platform_list qcom_acpi_platlist[] = {
struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu)
{
const struct device_node *np = smmu->dev->of_node;
const struct of_device_id *match;
#ifdef CONFIG_ACPI
if (np == NULL) {
/* Match platform for ACPI boot */
if (acpi_match_platform_list(qcom_acpi_platlist) >= 0)
return qcom_smmu_create(smmu, &qcom_smmu_impl);
return qcom_smmu_create(smmu, &qcom_smmu_500_impl0_data);
}
#endif
/*
* Do not change this order of implementation, i.e., first adreno
* smmu impl and then apss smmu since we can have both implementing
* arm,mmu-500 in which case we will miss setting adreno smmu specific
* features if the order is changed.
*/
if (of_device_is_compatible(np, "qcom,adreno-smmu"))
return qcom_smmu_create(smmu, &qcom_adreno_smmu_impl);
if (of_match_node(qcom_smmu_impl_of_match, np))
return qcom_smmu_create(smmu, &qcom_smmu_impl);
match = of_match_node(qcom_smmu_impl_of_match, np);
if (match)
return qcom_smmu_create(smmu, match->data);
return smmu;
}

21
drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
View File

@ -14,15 +14,26 @@ struct qcom_smmu {
u32 stall_enabled;
};
enum qcom_smmu_impl_reg_offset {
QCOM_SMMU_TBU_PWR_STATUS,
QCOM_SMMU_STATS_SYNC_INV_TBU_ACK,
QCOM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR,
};
struct qcom_smmu_config {
const u32 *reg_offset;
};
struct qcom_smmu_match_data {
const struct qcom_smmu_config *cfg;
const struct arm_smmu_impl *impl;
const struct arm_smmu_impl *adreno_impl;
};
#ifdef CONFIG_ARM_SMMU_QCOM_DEBUG
void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu);
const void *qcom_smmu_impl_data(struct arm_smmu_device *smmu);
#else
static inline void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu) { }
static inline const void *qcom_smmu_impl_data(struct arm_smmu_device *smmu)
{
return NULL;
}
#endif
#endif /* _ARM_SMMU_QCOM_H */

3
drivers/iommu/arm/arm-smmu/arm-smmu.c
View File

@ -1150,9 +1150,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
* different SMMUs.
*/
if (smmu_domain->smmu != smmu) {
dev_err(dev,
"cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
ret = -EINVAL;
goto rpm_put;
}

21
drivers/iommu/arm/arm-smmu/qcom_iommu.c
View File

@ -381,13 +381,8 @@ static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev
* Sanity check the domain. We don't support domains across
* different IOMMUs.
*/
if (qcom_domain->iommu != qcom_iommu) {
dev_err(dev, "cannot attach to IOMMU %s while already "
"attached to domain on IOMMU %s\n",
dev_name(qcom_domain->iommu->dev),
dev_name(qcom_iommu->dev));
if (qcom_domain->iommu != qcom_iommu)
return -EINVAL;
}
return 0;
}
@ -415,7 +410,8 @@ static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *de
}
static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
int ret;
unsigned long flags;
@ -426,13 +422,14 @@ static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
return -ENODEV;
spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
ret = ops->map(ops, iova, paddr, size, prot, GFP_ATOMIC);
ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, GFP_ATOMIC, mapped);
spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
return ret;
}
static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
size_t ret;
unsigned long flags;
@ -449,7 +446,7 @@ static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
*/
pm_runtime_get_sync(qcom_domain->iommu->dev);
spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
ret = ops->unmap(ops, iova, size, gather);
ret = ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
pm_runtime_put_sync(qcom_domain->iommu->dev);
@ -587,8 +584,8 @@ static const struct iommu_ops qcom_iommu_ops = {
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = qcom_iommu_attach_dev,
.detach_dev = qcom_iommu_detach_dev,
.map = qcom_iommu_map,
.unmap = qcom_iommu_unmap,
.map_pages = qcom_iommu_map,
.unmap_pages = qcom_iommu_unmap,
.flush_iotlb_all = qcom_iommu_flush_iotlb_all,
.iotlb_sync = qcom_iommu_iotlb_sync,
.iova_to_phys = qcom_iommu_iova_to_phys,

26
drivers/iommu/exynos-iommu.c
View File

@ -708,10 +708,6 @@ static int exynos_sysmmu_probe(struct platform_device *pdev)
if (ret)
return ret;
ret = iommu_device_register(&data->iommu, &exynos_iommu_ops, dev);
if (ret)
goto err_iommu_register;
platform_set_drvdata(pdev, data);
if (PG_ENT_SHIFT < 0) {
@ -743,11 +739,13 @@ static int exynos_sysmmu_probe(struct platform_device *pdev)
pm_runtime_enable(dev);
ret = iommu_device_register(&data->iommu, &exynos_iommu_ops, dev);
if (ret)
goto err_dma_set_mask;
return 0;
err_dma_set_mask:
iommu_device_unregister(&data->iommu);
err_iommu_register:
iommu_device_sysfs_remove(&data->iommu);
return ret;
}
@ -1432,12 +1430,6 @@ static int __init exynos_iommu_init(void)
return -ENOMEM;
}
ret = platform_driver_register(&exynos_sysmmu_driver);
if (ret) {
pr_err("%s: Failed to register driver\n", __func__);
goto err_reg_driver;
}
zero_lv2_table = kmem_cache_zalloc(lv2table_kmem_cache, GFP_KERNEL);
if (zero_lv2_table == NULL) {
pr_err("%s: Failed to allocate zero level2 page table\n",
@ -1446,10 +1438,16 @@ static int __init exynos_iommu_init(void)
goto err_zero_lv2;
}
ret = platform_driver_register(&exynos_sysmmu_driver);
if (ret) {
pr_err("%s: Failed to register driver\n", __func__);
goto err_reg_driver;
}
return 0;
err_zero_lv2:
platform_driver_unregister(&exynos_sysmmu_driver);
err_reg_driver:
platform_driver_unregister(&exynos_sysmmu_driver);
err_zero_lv2:
kmem_cache_destroy(lv2table_kmem_cache);
return ret;
}

8
drivers/iommu/fsl_pamu.c
View File

@ -211,7 +211,7 @@ int pamu_config_ppaace(int liodn, u32 omi, u32 stashid, int prot)
ppaace->op_encode.index_ot.omi = omi;
} else if (~omi != 0) {
pr_debug("bad operation mapping index: %d\n", omi);
return -EINVAL;
return -ENODEV;
}
/* configure stash id */
@ -779,7 +779,7 @@ static int fsl_pamu_probe(struct platform_device *pdev)
of_get_address(dev->of_node, 0, &size, NULL);
irq = irq_of_parse_and_map(dev->of_node, 0);
if (irq == NO_IRQ) {
if (!irq) {
dev_warn(dev, "no interrupts listed in PAMU node\n");
goto error;
}
@ -868,7 +868,7 @@ static int fsl_pamu_probe(struct platform_device *pdev)
ret = create_csd(ppaact_phys, mem_size, csd_port_id);
if (ret) {
dev_err(dev, "could not create coherence subdomain\n");
return ret;
goto error;
}
}
@ -903,7 +903,7 @@ static int fsl_pamu_probe(struct platform_device *pdev)
return 0;
error:
if (irq != NO_IRQ)
if (irq)
free_irq(irq, data);
kfree_sensitive(data);

4
drivers/iommu/fsl_pamu_domain.c
View File

@ -258,7 +258,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain,
liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
if (!liodn) {
pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
return -EINVAL;
return -ENODEV;
}
spin_lock_irqsave(&dma_domain->domain_lock, flags);
@ -267,7 +267,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain,
if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
liodn[i], dev->of_node);
ret = -EINVAL;
ret = -ENODEV;
break;
}

7
drivers/iommu/intel/dmar.c
View File

@ -1105,6 +1105,13 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
raw_spin_lock_init(&iommu->register_lock);
/*
* A value of N in PSS field of eCap register indicates hardware
* supports PASID field of N+1 bits.
*/
if (pasid_supported(iommu))
iommu->iommu.max_pasids = 2UL << ecap_pss(iommu->ecap);
/*
* This is only for hotplug; at boot time intel_iommu_enabled won't
* be set yet. When intel_iommu_init() runs, it registers the units

209
drivers/iommu/intel/iommu.c
View File

@ -27,7 +27,7 @@
#include "iommu.h"
#include "../dma-iommu.h"
#include "../irq_remapping.h"
#include "../iommu-sva-lib.h"
#include "../iommu-sva.h"
#include "pasid.h"
#include "cap_audit.h"
@ -277,7 +277,8 @@ static LIST_HEAD(dmar_satc_units);
#define for_each_rmrr_units(rmrr) \
list_for_each_entry(rmrr, &dmar_rmrr_units, list)
static void dmar_remove_one_dev_info(struct device *dev);
static void device_block_translation(struct device *dev);
static void intel_iommu_domain_free(struct iommu_domain *domain);
int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);
int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON);
@ -382,11 +383,6 @@ static inline int domain_type_is_si(struct dmar_domain *domain)
return domain->domain.type == IOMMU_DOMAIN_IDENTITY;
}
static inline bool domain_use_first_level(struct dmar_domain *domain)
{
return domain->flags & DOMAIN_FLAG_USE_FIRST_LEVEL;
}
static inline int domain_pfn_supported(struct dmar_domain *domain,
unsigned long pfn)
{
@ -500,7 +496,7 @@ static int domain_update_iommu_superpage(struct dmar_domain *domain,
rcu_read_lock();
for_each_active_iommu(iommu, drhd) {
if (iommu != skip) {
if (domain && domain_use_first_level(domain)) {
if (domain && domain->use_first_level) {
if (!cap_fl1gp_support(iommu->cap))
mask = 0x1;
} else {
@ -578,7 +574,7 @@ static void domain_update_iommu_cap(struct dmar_domain *domain)
* paging and 57-bits with 5-level paging). Hence, skip bit
* [N-1].
*/
if (domain_use_first_level(domain))
if (domain->use_first_level)
domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1);
else
domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw);
@ -779,19 +775,6 @@ static void domain_flush_cache(struct dmar_domain *domain,
clflush_cache_range(addr, size);
}
static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
{
struct context_entry *context;
int ret = 0;
spin_lock(&iommu->lock);
context = iommu_context_addr(iommu, bus, devfn, 0);
if (context)
ret = context_present(context);
spin_unlock(&iommu->lock);
return ret;
}
static void free_context_table(struct intel_iommu *iommu)
{
struct context_entry *context;
@ -959,7 +942,7 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
if (domain_use_first_level(domain))
if (domain->use_first_level)
pteval |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
if (cmpxchg64(&pte->val, 0ULL, pteval))
@ -1418,7 +1401,7 @@ static void iommu_enable_pci_caps(struct device_domain_info *info)
{
struct pci_dev *pdev;
if (!info || !dev_is_pci(info->dev))
if (!dev_is_pci(info->dev))
return;
pdev = to_pci_dev(info->dev);
@ -1458,7 +1441,7 @@ static void iommu_enable_pci_caps(struct device_domain_info *info)
}
}
static void iommu_disable_dev_iotlb(struct device_domain_info *info)
static void iommu_disable_pci_caps(struct device_domain_info *info)
{
struct pci_dev *pdev;
@ -1529,7 +1512,7 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
if (ih)
ih = 1 << 6;
if (domain_use_first_level(domain)) {
if (domain->use_first_level) {
qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, pages, ih);
} else {
unsigned long bitmask = aligned_pages - 1;
@ -1583,7 +1566,7 @@ static inline void __mapping_notify_one(struct intel_iommu *iommu,
* It's a non-present to present mapping. Only flush if caching mode
* and second level.
*/
if (cap_caching_mode(iommu->cap) && !domain_use_first_level(domain))
if (cap_caching_mode(iommu->cap) && !domain->use_first_level)
iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
else
iommu_flush_write_buffer(iommu);
@ -1599,7 +1582,7 @@ static void intel_flush_iotlb_all(struct iommu_domain *domain)
struct intel_iommu *iommu = info->iommu;
u16 did = domain_id_iommu(dmar_domain, iommu);
if (domain_use_first_level(dmar_domain))
if (dmar_domain->use_first_level)
qi_flush_piotlb(iommu, did, PASID_RID2PASID, 0, -1, 0);
else
iommu->flush.flush_iotlb(iommu, did, 0, 0,
@ -1772,7 +1755,7 @@ static struct dmar_domain *alloc_domain(unsigned int type)
domain->nid = NUMA_NO_NODE;
if (first_level_by_default(type))
domain->flags |= DOMAIN_FLAG_USE_FIRST_LEVEL;
domain->use_first_level = true;
domain->has_iotlb_device = false;
INIT_LIST_HEAD(&domain->devices);
spin_lock_init(&domain->lock);
@ -2064,7 +2047,6 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
} else {
iommu_flush_write_buffer(iommu);
}
iommu_enable_pci_caps(info);
ret = 0;
@ -2116,30 +2098,6 @@ domain_context_mapping(struct dmar_domain *domain, struct device *dev)
&domain_context_mapping_cb, &data);
}
static int domain_context_mapped_cb(struct pci_dev *pdev,
u16 alias, void *opaque)
{
struct intel_iommu *iommu = opaque;
return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
}
static int domain_context_mapped(struct device *dev)
{
struct intel_iommu *iommu;
u8 bus, devfn;
iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu)
return -ENODEV;
if (!dev_is_pci(dev))
return device_context_mapped(iommu, bus, devfn);
return !pci_for_each_dma_alias(to_pci_dev(dev),
domain_context_mapped_cb, iommu);
}
/* Returns a number of VTD pages, but aligned to MM page size */
static inline unsigned long aligned_nrpages(unsigned long host_addr,
size_t size)
@ -2229,7 +2187,7 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP);
attr |= DMA_FL_PTE_PRESENT;
if (domain_use_first_level(domain)) {
if (domain->use_first_level) {
attr |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
if (prot & DMA_PTE_WRITE)
attr |= DMA_FL_PTE_DIRTY;
@ -2472,7 +2430,8 @@ static int __init si_domain_init(int hw)
return 0;
}
static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
static int dmar_domain_attach_device(struct dmar_domain *domain,
struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct intel_iommu *iommu;
@ -2494,18 +2453,11 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
/* PASID table is mandatory for a PCI device in scalable mode. */
if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
ret = intel_pasid_alloc_table(dev);
if (ret) {
dev_err(dev, "PASID table allocation failed\n");
dmar_remove_one_dev_info(dev);
return ret;
}
/* Setup the PASID entry for requests without PASID: */
if (hw_pass_through && domain_type_is_si(domain))
ret = intel_pasid_setup_pass_through(iommu, domain,
dev, PASID_RID2PASID);
else if (domain_use_first_level(domain))
else if (domain->use_first_level)
ret = domain_setup_first_level(iommu, domain, dev,
PASID_RID2PASID);
else
@ -2513,7 +2465,7 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
dev, PASID_RID2PASID);
if (ret) {
dev_err(dev, "Setup RID2PASID failed\n");
dmar_remove_one_dev_info(dev);
device_block_translation(dev);
return ret;
}
}
@ -2521,10 +2473,12 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
ret = domain_context_mapping(domain, dev);
if (ret) {
dev_err(dev, "Domain context map failed\n");
dmar_remove_one_dev_info(dev);
device_block_translation(dev);
return ret;
}
iommu_enable_pci_caps(info);
return 0;
}
@ -4125,9 +4079,8 @@ static void dmar_remove_one_dev_info(struct device *dev)
intel_pasid_tear_down_entry(iommu, info->dev,
PASID_RID2PASID, false);
iommu_disable_dev_iotlb(info);
iommu_disable_pci_caps(info);
domain_context_clear(info);
intel_pasid_free_table(info->dev);
}
spin_lock_irqsave(&domain->lock, flags);
@ -4138,6 +4091,37 @@ static void dmar_remove_one_dev_info(struct device *dev)
info->domain = NULL;
}
/*
* Clear the page table pointer in context or pasid table entries so that
* all DMA requests without PASID from the device are blocked. If the page
* table has been set, clean up the data structures.
*/
static void device_block_translation(struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct intel_iommu *iommu = info->iommu;
unsigned long flags;
iommu_disable_pci_caps(info);
if (!dev_is_real_dma_subdevice(dev)) {
if (sm_supported(iommu))
intel_pasid_tear_down_entry(iommu, dev,
PASID_RID2PASID, false);
else
domain_context_clear(info);
}
if (!info->domain)
return;
spin_lock_irqsave(&info->domain->lock, flags);
list_del(&info->link);
spin_unlock_irqrestore(&info->domain->lock, flags);
domain_detach_iommu(info->domain, iommu);
info->domain = NULL;
}
static int md_domain_init(struct dmar_domain *domain, int guest_width)
{
int adjust_width;
@ -4159,12 +4143,28 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
return 0;
}
static int blocking_domain_attach_dev(struct iommu_domain *domain,
struct device *dev)
{
device_block_translation(dev);
return 0;
}
static struct iommu_domain blocking_domain = {
.ops = &(const struct iommu_domain_ops) {
.attach_dev = blocking_domain_attach_dev,
.free = intel_iommu_domain_free
}
};
static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
{
struct dmar_domain *dmar_domain;
struct iommu_domain *domain;
switch (type) {
case IOMMU_DOMAIN_BLOCKED:
return &blocking_domain;
case IOMMU_DOMAIN_DMA:
case IOMMU_DOMAIN_DMA_FQ:
case IOMMU_DOMAIN_UNMANAGED:
@ -4188,6 +4188,8 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
return domain;
case IOMMU_DOMAIN_IDENTITY:
return &si_domain->domain;
case IOMMU_DOMAIN_SVA:
return intel_svm_domain_alloc();
default:
return NULL;
}
@ -4197,7 +4199,7 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
static void intel_iommu_domain_free(struct iommu_domain *domain)
{
if (domain != &si_domain->domain)
if (domain != &si_domain->domain && domain != &blocking_domain)
domain_exit(to_dmar_domain(domain));
}
@ -4213,19 +4215,15 @@ static int prepare_domain_attach_device(struct iommu_domain *domain,
return -ENODEV;
if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
return -EOPNOTSUPP;
return -EINVAL;
/* check if this iommu agaw is sufficient for max mapped address */
addr_width = agaw_to_width(iommu->agaw);
if (addr_width > cap_mgaw(iommu->cap))
addr_width = cap_mgaw(iommu->cap);
if (dmar_domain->max_addr > (1LL << addr_width)) {
dev_err(dev, "%s: iommu width (%d) is not "
"sufficient for the mapped address (%llx)\n",
__func__, addr_width, dmar_domain->max_addr);
return -EFAULT;
}
if (dmar_domain->max_addr > (1LL << addr_width))
return -EINVAL;
dmar_domain->gaw = addr_width;
/*
@ -4248,6 +4246,7 @@ static int prepare_domain_attach_device(struct iommu_domain *domain,
static int intel_iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
int ret;
if (domain->type == IOMMU_DOMAIN_UNMANAGED &&
@ -4256,25 +4255,14 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
return -EPERM;
}
/* normally dev is not mapped */
if (unlikely(domain_context_mapped(dev))) {
struct device_domain_info *info = dev_iommu_priv_get(dev);
if (info->domain)
dmar_remove_one_dev_info(dev);
}
if (info->domain)
device_block_translation(dev);
ret = prepare_domain_attach_device(domain, dev);
if (ret)
return ret;
return domain_add_dev_info(to_dmar_domain(domain), dev);
}
static void intel_iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
dmar_remove_one_dev_info(dev);
return dmar_domain_attach_device(to_dmar_domain(domain), dev);
}
static int intel_iommu_map(struct iommu_domain *domain,
@ -4438,7 +4426,7 @@ static void domain_set_force_snooping(struct dmar_domain *domain)
* Second level page table supports per-PTE snoop control. The
* iommu_map() interface will handle this by setting SNP bit.
*/
if (!domain_use_first_level(domain)) {
if (!domain->use_first_level) {
domain->set_pte_snp = true;
return;
}
@ -4487,6 +4475,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
struct device_domain_info *info;
struct intel_iommu *iommu;
u8 bus, devfn;
int ret;
iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu || !iommu->iommu.ops)
@ -4531,6 +4520,16 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
dev_iommu_priv_set(dev, info);
if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
ret = intel_pasid_alloc_table(dev);
if (ret) {
dev_err(dev, "PASID table allocation failed\n");
dev_iommu_priv_set(dev, NULL);
kfree(info);
return ERR_PTR(ret);
}
}
return &iommu->iommu;
}
@ -4539,6 +4538,7 @@ static void intel_iommu_release_device(struct device *dev)
struct device_domain_info *info = dev_iommu_priv_get(dev);
dmar_remove_one_dev_info(dev);
intel_pasid_free_table(dev);
dev_iommu_priv_set(dev, NULL);
kfree(info);
set_dma_ops(dev, NULL);
@ -4732,6 +4732,28 @@ static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
__mapping_notify_one(info->iommu, dmar_domain, pfn, pages);
}
static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
{
struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
struct iommu_domain *domain;
/* Domain type specific cleanup: */
domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0);
if (domain) {
switch (domain->type) {
case IOMMU_DOMAIN_SVA:
intel_svm_remove_dev_pasid(dev, pasid);
break;
default:
/* should never reach here */
WARN_ON(1);
break;
}
}
intel_pasid_tear_down_entry(iommu, dev, pasid, false);
}
const struct iommu_ops intel_iommu_ops = {
.capable = intel_iommu_capable,
.domain_alloc = intel_iommu_domain_alloc,
@ -4744,16 +4766,13 @@ const struct iommu_ops intel_iommu_ops = {
.dev_disable_feat = intel_iommu_dev_disable_feat,
.is_attach_deferred = intel_iommu_is_attach_deferred,
.def_domain_type = device_def_domain_type,
.remove_dev_pasid = intel_iommu_remove_dev_pasid,
.pgsize_bitmap = SZ_4K,
#ifdef CONFIG_INTEL_IOMMU_SVM
.sva_bind = intel_svm_bind,
.sva_unbind = intel_svm_unbind,
.sva_get_pasid = intel_svm_get_pasid,
.page_response = intel_svm_page_response,
#endif
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = intel_iommu_attach_device,
.detach_dev = intel_iommu_detach_device,
.map_pages = intel_iommu_map_pages,
.unmap_pages = intel_iommu_unmap_pages,
.iotlb_sync_map = intel_iommu_iotlb_sync_map,

33
drivers/iommu/intel/iommu.h
View File

@ -480,8 +480,6 @@ enum {
#define VTD_FLAG_IRQ_REMAP_PRE_ENABLED (1 << 1)
#define VTD_FLAG_SVM_CAPABLE (1 << 2)
extern int intel_iommu_sm;
#define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap))
#define pasid_supported(iommu) (sm_supported(iommu) && \
ecap_pasid((iommu)->ecap))
@ -517,14 +515,6 @@ struct context_entry {
u64 hi;
};
/*
* When VT-d works in the scalable mode, it allows DMA translation to
* happen through either first level or second level page table. This
* bit marks that the DMA translation for the domain goes through the
* first level page table, otherwise, it goes through the second level.
*/
#define DOMAIN_FLAG_USE_FIRST_LEVEL BIT(1)
struct iommu_domain_info {
struct intel_iommu *iommu;
unsigned int refcnt; /* Refcount of devices per iommu */
@ -541,6 +531,11 @@ struct dmar_domain {
u8 iommu_coherency: 1; /* indicate coherency of iommu access */
u8 force_snooping : 1; /* Create IOPTEs with snoop control */
u8 set_pte_snp:1;
u8 use_first_level:1; /* DMA translation for the domain goes
* through the first level page table,
* otherwise, goes through the second
* level.
*/
spinlock_t lock; /* Protect device tracking lists */
struct list_head devices; /* all devices' list */
@ -550,8 +545,6 @@ struct dmar_domain {
/* adjusted guest address width, 0 is level 2 30-bit */
int agaw;
int flags; /* flags to find out type of domain */
int iommu_superpage;/* Level of superpages supported:
0 == 4KiB (no superpages), 1 == 2MiB,
2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
@ -754,12 +747,10 @@ struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn);
extern void intel_svm_check(struct intel_iommu *iommu);
extern int intel_svm_enable_prq(struct intel_iommu *iommu);
extern int intel_svm_finish_prq(struct intel_iommu *iommu);
struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm,
void *drvdata);
void intel_svm_unbind(struct iommu_sva *handle);
u32 intel_svm_get_pasid(struct iommu_sva *handle);
int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt,
struct iommu_page_response *msg);
struct iommu_domain *intel_svm_domain_alloc(void);
void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid);
struct intel_svm_dev {
struct list_head list;
@ -784,6 +775,14 @@ struct intel_svm {
};
#else
static inline void intel_svm_check(struct intel_iommu *iommu) {}
static inline struct iommu_domain *intel_svm_domain_alloc(void)
{
return NULL;
}
static inline void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid)
{
}
#endif
#ifdef CONFIG_INTEL_IOMMU_DEBUGFS
@ -799,6 +798,7 @@ struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
extern const struct iommu_ops intel_iommu_ops;
#ifdef CONFIG_INTEL_IOMMU
extern int intel_iommu_sm;
extern int iommu_calculate_agaw(struct intel_iommu *iommu);
extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu);
extern int dmar_disabled;
@ -814,6 +814,7 @@ static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
}
#define dmar_disabled (1)
#define intel_iommu_enabled (0)
#define intel_iommu_sm (0)
#endif
static inline const char *decode_prq_descriptor(char *str, size_t size,

6
drivers/iommu/intel/pasid.c
View File

@ -101,8 +101,10 @@ int intel_pasid_alloc_table(struct device *dev)
might_sleep();
info = dev_iommu_priv_get(dev);
if (WARN_ON(!info || !dev_is_pci(dev) || info->pasid_table))
return -EINVAL;
if (WARN_ON(!info || !dev_is_pci(dev)))
return -ENODEV;
if (WARN_ON(info->pasid_table))
return -EEXIST;
pasid_table = kzalloc(sizeof(*pasid_table), GFP_KERNEL);
if (!pasid_table)

145
drivers/iommu/intel/svm.c
View File

@ -24,7 +24,7 @@
#include "iommu.h"
#include "pasid.h"
#include "perf.h"
#include "../iommu-sva-lib.h"
#include "../iommu-sva.h"
#include "trace.h"
static irqreturn_t prq_event_thread(int irq, void *d);
@ -299,19 +299,9 @@ static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid,
return 0;
}
static int intel_svm_alloc_pasid(struct device *dev, struct mm_struct *mm,
unsigned int flags)
{
ioasid_t max_pasid = dev_is_pci(dev) ?
pci_max_pasids(to_pci_dev(dev)) : intel_pasid_max_id;
return iommu_sva_alloc_pasid(mm, PASID_MIN, max_pasid - 1);
}
static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu,
struct device *dev,
struct mm_struct *mm,
unsigned int flags)
struct mm_struct *mm)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct intel_svm_dev *sdev;
@ -327,22 +317,18 @@ static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu,
svm->pasid = mm->pasid;
svm->mm = mm;
svm->flags = flags;
INIT_LIST_HEAD_RCU(&svm->devs);
if (!(flags & SVM_FLAG_SUPERVISOR_MODE)) {
svm->notifier.ops = &intel_mmuops;
ret = mmu_notifier_register(&svm->notifier, mm);
if (ret) {
kfree(svm);
return ERR_PTR(ret);
}
svm->notifier.ops = &intel_mmuops;
ret = mmu_notifier_register(&svm->notifier, mm);
if (ret) {
kfree(svm);
return ERR_PTR(ret);
}
ret = pasid_private_add(svm->pasid, svm);
if (ret) {
if (svm->notifier.ops)
mmu_notifier_unregister(&svm->notifier, mm);
mmu_notifier_unregister(&svm->notifier, mm);
kfree(svm);
return ERR_PTR(ret);
}
@ -377,9 +363,7 @@ static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu,
}
/* Setup the pasid table: */
sflags = (flags & SVM_FLAG_SUPERVISOR_MODE) ?
PASID_FLAG_SUPERVISOR_MODE : 0;
sflags |= cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
ret = intel_pasid_setup_first_level(iommu, dev, mm->pgd, mm->pasid,
FLPT_DEFAULT_DID, sflags);
if (ret)
@ -393,8 +377,7 @@ static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu,
kfree(sdev);
free_svm:
if (list_empty(&svm->devs)) {
if (svm->notifier.ops)
mmu_notifier_unregister(&svm->notifier, mm);
mmu_notifier_unregister(&svm->notifier, mm);
pasid_private_remove(mm->pasid);
kfree(svm);
}
@ -787,67 +770,6 @@ static irqreturn_t prq_event_thread(int irq, void *d)
return IRQ_RETVAL(handled);
}
struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
{
struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
unsigned int flags = 0;
struct iommu_sva *sva;
int ret;
if (drvdata)
flags = *(unsigned int *)drvdata;
if (flags & SVM_FLAG_SUPERVISOR_MODE) {
if (!ecap_srs(iommu->ecap)) {
dev_err(dev, "%s: Supervisor PASID not supported\n",
iommu->name);
return ERR_PTR(-EOPNOTSUPP);
}
if (mm) {
dev_err(dev, "%s: Supervisor PASID with user provided mm\n",
iommu->name);
return ERR_PTR(-EINVAL);
}
mm = &init_mm;
}
mutex_lock(&pasid_mutex);
ret = intel_svm_alloc_pasid(dev, mm, flags);
if (ret) {
mutex_unlock(&pasid_mutex);
return ERR_PTR(ret);
}
sva = intel_svm_bind_mm(iommu, dev, mm, flags);
mutex_unlock(&pasid_mutex);
return sva;
}
void intel_svm_unbind(struct iommu_sva *sva)
{
struct intel_svm_dev *sdev = to_intel_svm_dev(sva);
mutex_lock(&pasid_mutex);
intel_svm_unbind_mm(sdev->dev, sdev->pasid);
mutex_unlock(&pasid_mutex);
}
u32 intel_svm_get_pasid(struct iommu_sva *sva)
{
struct intel_svm_dev *sdev;
u32 pasid;
mutex_lock(&pasid_mutex);
sdev = to_intel_svm_dev(sva);
pasid = sdev->pasid;
mutex_unlock(&pasid_mutex);
return pasid;
}
int intel_svm_page_response(struct device *dev,
struct iommu_fault_event *evt,
struct iommu_page_response *msg)
@ -918,3 +840,50 @@ int intel_svm_page_response(struct device *dev,
out:
return ret;
}
void intel_svm_remove_dev_pasid(struct device *dev, ioasid_t pasid)
{
mutex_lock(&pasid_mutex);
intel_svm_unbind_mm(dev, pasid);
mutex_unlock(&pasid_mutex);
}
static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct intel_iommu *iommu = info->iommu;
struct mm_struct *mm = domain->mm;
struct iommu_sva *sva;
int ret = 0;
mutex_lock(&pasid_mutex);
sva = intel_svm_bind_mm(iommu, dev, mm);
if (IS_ERR(sva))
ret = PTR_ERR(sva);
mutex_unlock(&pasid_mutex);
return ret;
}
static void intel_svm_domain_free(struct iommu_domain *domain)
{
kfree(to_dmar_domain(domain));
}
static const struct iommu_domain_ops intel_svm_domain_ops = {
.set_dev_pasid = intel_svm_set_dev_pasid,
.free = intel_svm_domain_free
};
struct iommu_domain *intel_svm_domain_alloc(void)
{
struct dmar_domain *domain;
domain = kzalloc(sizeof(*domain), GFP_KERNEL);
if (!domain)
return NULL;
domain->domain.ops = &intel_svm_domain_ops;
return &domain->domain;
}

77
drivers/iommu/io-pgfault.c
View File

@ -11,7 +11,7 @@
#include <linux/slab.h>
#include <linux/workqueue.h>
#include "iommu-sva-lib.h"
#include "iommu-sva.h"
/**
* struct iopf_queue - IO Page Fault queue
@ -69,69 +69,18 @@ static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf,
return iommu_page_response(dev, &resp);
}
static enum iommu_page_response_code
iopf_handle_single(struct iopf_fault *iopf)
{
vm_fault_t ret;
struct mm_struct *mm;
struct vm_area_struct *vma;
unsigned int access_flags = 0;
unsigned int fault_flags = FAULT_FLAG_REMOTE;
struct iommu_fault_page_request *prm = &iopf->fault.prm;
enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID;
if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID))
return status;
mm = iommu_sva_find(prm->pasid);
if (IS_ERR_OR_NULL(mm))
return status;
mmap_read_lock(mm);
vma = find_extend_vma(mm, prm->addr);
if (!vma)
/* Unmapped area */
goto out_put_mm;
if (prm->perm & IOMMU_FAULT_PERM_READ)
access_flags |= VM_READ;
if (prm->perm & IOMMU_FAULT_PERM_WRITE) {
access_flags |= VM_WRITE;
fault_flags |= FAULT_FLAG_WRITE;
}
if (prm->perm & IOMMU_FAULT_PERM_EXEC) {
access_flags |= VM_EXEC;
fault_flags |= FAULT_FLAG_INSTRUCTION;
}
if (!(prm->perm & IOMMU_FAULT_PERM_PRIV))
fault_flags |= FAULT_FLAG_USER;
if (access_flags & ~vma->vm_flags)
/* Access fault */
goto out_put_mm;
ret = handle_mm_fault(vma, prm->addr, fault_flags, NULL);
status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID :
IOMMU_PAGE_RESP_SUCCESS;
out_put_mm:
mmap_read_unlock(mm);
mmput(mm);
return status;
}
static void iopf_handle_group(struct work_struct *work)
static void iopf_handler(struct work_struct *work)
{
struct iopf_group *group;
struct iommu_domain *domain;
struct iopf_fault *iopf, *next;
enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS;
group = container_of(work, struct iopf_group, work);
domain = iommu_get_domain_for_dev_pasid(group->dev,
group->last_fault.fault.prm.pasid, 0);
if (!domain || !domain->iopf_handler)
status = IOMMU_PAGE_RESP_INVALID;
list_for_each_entry_safe(iopf, next, &group->faults, list) {
/*
@ -139,7 +88,8 @@ static void iopf_handle_group(struct work_struct *work)
* faults in the group if there is an error.
*/
if (status == IOMMU_PAGE_RESP_SUCCESS)
status = iopf_handle_single(iopf);
status = domain->iopf_handler(&iopf->fault,
domain->fault_data);
if (!(iopf->fault.prm.flags &
IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE))
@ -181,6 +131,13 @@ static void iopf_handle_group(struct work_struct *work)
* request completes, outstanding faults will have been dealt with by the time
* the PASID is freed.
*
* Any valid page fault will be eventually routed to an iommu domain and the
* page fault handler installed there will get called. The users of this
* handling framework should guarantee that the iommu domain could only be
* freed after the device has stopped generating page faults (or the iommu
* hardware has been set to block the page faults) and the pending page faults
* have been flushed.
*
* Return: 0 on success and <0 on error.
*/
int iommu_queue_iopf(struct iommu_fault *fault, void *cookie)
@ -235,7 +192,7 @@ int iommu_queue_iopf(struct iommu_fault *fault, void *cookie)
group->last_fault.fault = *fault;
INIT_LIST_HEAD(&group->faults);
list_add(&group->last_fault.list, &group->faults);
INIT_WORK(&group->work, iopf_handle_group);
INIT_WORK(&group->work, iopf_handler);
/* See if we have partial faults for this group */
list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) {

41
drivers/iommu/io-pgtable-arm-v7s.c
View File

@ -564,8 +564,7 @@ static int arm_v7s_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
iova += pgsize;
paddr += pgsize;
if (mapped)
*mapped += pgsize;
*mapped += pgsize;
}
/*
* Synchronise all PTE updates for the new mapping before there's
@ -576,12 +575,6 @@ static int arm_v7s_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
return ret;
}
static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
return arm_v7s_map_pages(ops, iova, paddr, size, 1, prot, gfp, NULL);
}
static void arm_v7s_free_pgtable(struct io_pgtable *iop)
{
struct arm_v7s_io_pgtable *data = io_pgtable_to_data(iop);
@ -764,12 +757,6 @@ static size_t arm_v7s_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova
return unmapped;
}
static size_t arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
{
return arm_v7s_unmap_pages(ops, iova, size, 1, gather);
}
static phys_addr_t arm_v7s_iova_to_phys(struct io_pgtable_ops *ops,
unsigned long iova)
{
@ -842,9 +829,7 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg,
goto out_free_data;
data->iop.ops = (struct io_pgtable_ops) {
.map = arm_v7s_map,
.map_pages = arm_v7s_map_pages,
.unmap = arm_v7s_unmap,
.unmap_pages = arm_v7s_unmap_pages,
.iova_to_phys = arm_v7s_iova_to_phys,
};
@ -954,6 +939,7 @@ static int __init arm_v7s_do_selftests(void)
};
unsigned int iova, size, iova_start;
unsigned int i, loopnr = 0;
size_t mapped;
selftest_running = true;
@ -984,15 +970,16 @@ static int __init arm_v7s_do_selftests(void)
iova = 0;
for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) {
size = 1UL << i;
if (ops->map(ops, iova, iova, size, IOMMU_READ |
IOMMU_WRITE |
IOMMU_NOEXEC |
IOMMU_CACHE, GFP_KERNEL))
if (ops->map_pages(ops, iova, iova, size, 1,
IOMMU_READ | IOMMU_WRITE |
IOMMU_NOEXEC | IOMMU_CACHE,
GFP_KERNEL, &mapped))
return __FAIL(ops);
/* Overlapping mappings */
if (!ops->map(ops, iova, iova + size, size,
IOMMU_READ | IOMMU_NOEXEC, GFP_KERNEL))
if (!ops->map_pages(ops, iova, iova + size, size, 1,
IOMMU_READ | IOMMU_NOEXEC, GFP_KERNEL,
&mapped))
return __FAIL(ops);
if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
@ -1007,11 +994,12 @@ static int __init arm_v7s_do_selftests(void)
size = 1UL << __ffs(cfg.pgsize_bitmap);
while (i < loopnr) {
iova_start = i * SZ_16M;
if (ops->unmap(ops, iova_start + size, size, NULL) != size)
if (ops->unmap_pages(ops, iova_start + size, size, 1, NULL) != size)
return __FAIL(ops);
/* Remap of partial unmap */
if (ops->map(ops, iova_start + size, size, size, IOMMU_READ, GFP_KERNEL))
if (ops->map_pages(ops, iova_start + size, size, size, 1,
IOMMU_READ, GFP_KERNEL, &mapped))
return __FAIL(ops);
if (ops->iova_to_phys(ops, iova_start + size + 42)
@ -1025,14 +1013,15 @@ static int __init arm_v7s_do_selftests(void)
for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) {
size = 1UL << i;
if (ops->unmap(ops, iova, size, NULL) != size)
if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
return __FAIL(ops);
if (ops->iova_to_phys(ops, iova + 42))
return __FAIL(ops);
/* Remap full block */
if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL))
if (ops->map_pages(ops, iova, iova, size, 1, IOMMU_WRITE,
GFP_KERNEL, &mapped))
return __FAIL(ops);
if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))

42
drivers/iommu/io-pgtable-arm.c
View File

@ -360,7 +360,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
max_entries = ARM_LPAE_PTES_PER_TABLE(data) - map_idx_start;
num_entries = min_t(int, pgcount, max_entries);
ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);
if (!ret && mapped)
if (!ret)
*mapped += num_entries * size;
return ret;
@ -496,13 +496,6 @@ static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
return ret;
}
static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,
phys_addr_t paddr, size_t size, int iommu_prot, gfp_t gfp)
{
return arm_lpae_map_pages(ops, iova, paddr, size, 1, iommu_prot, gfp,
NULL);
}
static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
arm_lpae_iopte *ptep)
{
@ -682,12 +675,6 @@ static size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iov
data->start_level, ptep);
}
static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
{
return arm_lpae_unmap_pages(ops, iova, size, 1, gather);
}
static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
unsigned long iova)
{
@ -799,9 +786,7 @@ arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)
data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1));
data->iop.ops = (struct io_pgtable_ops) {
.map = arm_lpae_map,
.map_pages = arm_lpae_map_pages,
.unmap = arm_lpae_unmap,
.unmap_pages = arm_lpae_unmap_pages,
.iova_to_phys = arm_lpae_iova_to_phys,
};
@ -1176,7 +1161,7 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
int i, j;
unsigned long iova;
size_t size;
size_t size, mapped;
struct io_pgtable_ops *ops;
selftest_running = true;
@ -1209,15 +1194,16 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
size = 1UL << j;
if (ops->map(ops, iova, iova, size, IOMMU_READ |
IOMMU_WRITE |
IOMMU_NOEXEC |
IOMMU_CACHE, GFP_KERNEL))
if (ops->map_pages(ops, iova, iova, size, 1,
IOMMU_READ | IOMMU_WRITE |
IOMMU_NOEXEC | IOMMU_CACHE,
GFP_KERNEL, &mapped))
return __FAIL(ops, i);
/* Overlapping mappings */
if (!ops->map(ops, iova, iova + size, size,
IOMMU_READ | IOMMU_NOEXEC, GFP_KERNEL))
if (!ops->map_pages(ops, iova, iova + size, size, 1,
IOMMU_READ | IOMMU_NOEXEC,
GFP_KERNEL, &mapped))
return __FAIL(ops, i);
if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
@ -1228,11 +1214,12 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
/* Partial unmap */
size = 1UL << __ffs(cfg->pgsize_bitmap);
if (ops->unmap(ops, SZ_1G + size, size, NULL) != size)
if (ops->unmap_pages(ops, SZ_1G + size, size, 1, NULL) != size)
return __FAIL(ops, i);
/* Remap of partial unmap */
if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ, GFP_KERNEL))
if (ops->map_pages(ops, SZ_1G + size, size, size, 1,
IOMMU_READ, GFP_KERNEL, &mapped))
return __FAIL(ops, i);
if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42))
@ -1243,14 +1230,15 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
size = 1UL << j;
if (ops->unmap(ops, iova, size, NULL) != size)
if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
return __FAIL(ops, i);
if (ops->iova_to_phys(ops, iova + 42))
return __FAIL(ops, i);
/* Remap full block */
if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL))
if (ops->map_pages(ops, iova, iova, size, 1,
IOMMU_WRITE, GFP_KERNEL, &mapped))
return __FAIL(ops, i);
if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))

71
drivers/iommu/iommu-sva-lib.c
View File

@ -1,71 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Helpers for IOMMU drivers implementing SVA
*/
#include <linux/mutex.h>
#include <linux/sched/mm.h>
#include "iommu-sva-lib.h"
static DEFINE_MUTEX(iommu_sva_lock);
static DECLARE_IOASID_SET(iommu_sva_pasid);
/**
* iommu_sva_alloc_pasid - Allocate a PASID for the mm
* @mm: the mm
* @min: minimum PASID value (inclusive)
* @max: maximum PASID value (inclusive)
*
* Try to allocate a PASID for this mm, or take a reference to the existing one
* provided it fits within the [@min, @max] range. On success the PASID is
* available in mm->pasid and will be available for the lifetime of the mm.
*
* Returns 0 on success and < 0 on error.
*/
int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max)
{
int ret = 0;
ioasid_t pasid;
if (min == INVALID_IOASID || max == INVALID_IOASID ||
min == 0 || max < min)
return -EINVAL;
mutex_lock(&iommu_sva_lock);
/* Is a PASID already associated with this mm? */
if (pasid_valid(mm->pasid)) {
if (mm->pasid < min || mm->pasid >= max)
ret = -EOVERFLOW;
goto out;
}
pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm);
if (!pasid_valid(pasid))
ret = -ENOMEM;
else
mm_pasid_set(mm, pasid);
out:
mutex_unlock(&iommu_sva_lock);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_sva_alloc_pasid);
/* ioasid_find getter() requires a void * argument */
static bool __mmget_not_zero(void *mm)
{
return mmget_not_zero(mm);
}
/**
* iommu_sva_find() - Find mm associated to the given PASID
* @pasid: Process Address Space ID assigned to the mm
*
* On success a reference to the mm is taken, and must be released with mmput().
*
* Returns the mm corresponding to this PASID, or an error if not found.
*/
struct mm_struct *iommu_sva_find(ioasid_t pasid)
{
return ioasid_find(&iommu_sva_pasid, pasid, __mmget_not_zero);
}
EXPORT_SYMBOL_GPL(iommu_sva_find);

240
drivers/iommu/iommu-sva.c Normal file
View File

@ -0,0 +1,240 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Helpers for IOMMU drivers implementing SVA
*/
#include <linux/mutex.h>
#include <linux/sched/mm.h>
#include <linux/iommu.h>
#include "iommu-sva.h"
static DEFINE_MUTEX(iommu_sva_lock);
static DECLARE_IOASID_SET(iommu_sva_pasid);
/**
* iommu_sva_alloc_pasid - Allocate a PASID for the mm
* @mm: the mm
* @min: minimum PASID value (inclusive)
* @max: maximum PASID value (inclusive)
*
* Try to allocate a PASID for this mm, or take a reference to the existing one
* provided it fits within the [@min, @max] range. On success the PASID is
* available in mm->pasid and will be available for the lifetime of the mm.
*
* Returns 0 on success and < 0 on error.
*/
int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max)
{
int ret = 0;
ioasid_t pasid;
if (min == INVALID_IOASID || max == INVALID_IOASID ||
min == 0 || max < min)
return -EINVAL;
mutex_lock(&iommu_sva_lock);
/* Is a PASID already associated with this mm? */
if (pasid_valid(mm->pasid)) {
if (mm->pasid < min || mm->pasid >= max)
ret = -EOVERFLOW;
goto out;
}
pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm);
if (!pasid_valid(pasid))
ret = -ENOMEM;
else
mm_pasid_set(mm, pasid);
out:
mutex_unlock(&iommu_sva_lock);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_sva_alloc_pasid);
/* ioasid_find getter() requires a void * argument */
static bool __mmget_not_zero(void *mm)
{
return mmget_not_zero(mm);
}
/**
* iommu_sva_find() - Find mm associated to the given PASID
* @pasid: Process Address Space ID assigned to the mm
*
* On success a reference to the mm is taken, and must be released with mmput().
*
* Returns the mm corresponding to this PASID, or an error if not found.
*/
struct mm_struct *iommu_sva_find(ioasid_t pasid)
{
return ioasid_find(&iommu_sva_pasid, pasid, __mmget_not_zero);
}
EXPORT_SYMBOL_GPL(iommu_sva_find);
/**
* iommu_sva_bind_device() - Bind a process address space to a device
* @dev: the device
* @mm: the mm to bind, caller must hold a reference to mm_users
*
* Create a bond between device and address space, allowing the device to
* access the mm using the PASID returned by iommu_sva_get_pasid(). If a
* bond already exists between @device and @mm, an additional internal
* reference is taken. Caller must call iommu_sva_unbind_device()
* to release each reference.
*
* iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
* initialize the required SVA features.
*
* On error, returns an ERR_PTR value.
*/
struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
{
struct iommu_domain *domain;
struct iommu_sva *handle;
ioasid_t max_pasids;
int ret;
max_pasids = dev->iommu->max_pasids;
if (!max_pasids)
return ERR_PTR(-EOPNOTSUPP);
/* Allocate mm->pasid if necessary. */
ret = iommu_sva_alloc_pasid(mm, 1, max_pasids - 1);
if (ret)
return ERR_PTR(ret);
handle = kzalloc(sizeof(*handle), GFP_KERNEL);
if (!handle)
return ERR_PTR(-ENOMEM);
mutex_lock(&iommu_sva_lock);
/* Search for an existing domain. */
domain = iommu_get_domain_for_dev_pasid(dev, mm->pasid,
IOMMU_DOMAIN_SVA);
if (IS_ERR(domain)) {
ret = PTR_ERR(domain);
goto out_unlock;
}
if (domain) {
domain->users++;
goto out;
}
/* Allocate a new domain and set it on device pasid. */
domain = iommu_sva_domain_alloc(dev, mm);
if (!domain) {
ret = -ENOMEM;
goto out_unlock;
}
ret = iommu_attach_device_pasid(domain, dev, mm->pasid);
if (ret)
goto out_free_domain;
domain->users = 1;
out:
mutex_unlock(&iommu_sva_lock);
handle->dev = dev;
handle->domain = domain;
return handle;
out_free_domain:
iommu_domain_free(domain);
out_unlock:
mutex_unlock(&iommu_sva_lock);
kfree(handle);
return ERR_PTR(ret);
}
EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
/**
* iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
* @handle: the handle returned by iommu_sva_bind_device()
*
* Put reference to a bond between device and address space. The device should
* not be issuing any more transaction for this PASID. All outstanding page
* requests for this PASID must have been flushed to the IOMMU.
*/
void iommu_sva_unbind_device(struct iommu_sva *handle)
{
struct iommu_domain *domain = handle->domain;
ioasid_t pasid = domain->mm->pasid;
struct device *dev = handle->dev;
mutex_lock(&iommu_sva_lock);
if (--domain->users == 0) {
iommu_detach_device_pasid(domain, dev, pasid);
iommu_domain_free(domain);
}
mutex_unlock(&iommu_sva_lock);
kfree(handle);
}
EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
u32 iommu_sva_get_pasid(struct iommu_sva *handle)
{
struct iommu_domain *domain = handle->domain;
return domain->mm->pasid;
}
EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);
/*
* I/O page fault handler for SVA
*/
enum iommu_page_response_code
iommu_sva_handle_iopf(struct iommu_fault *fault, void *data)
{
vm_fault_t ret;
struct vm_area_struct *vma;
struct mm_struct *mm = data;
unsigned int access_flags = 0;
unsigned int fault_flags = FAULT_FLAG_REMOTE;
struct iommu_fault_page_request *prm = &fault->prm;
enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID;
if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID))
return status;
if (!mmget_not_zero(mm))
return status;
mmap_read_lock(mm);
vma = find_extend_vma(mm, prm->addr);
if (!vma)
/* Unmapped area */
goto out_put_mm;
if (prm->perm & IOMMU_FAULT_PERM_READ)
access_flags |= VM_READ;
if (prm->perm & IOMMU_FAULT_PERM_WRITE) {
access_flags |= VM_WRITE;
fault_flags |= FAULT_FLAG_WRITE;
}
if (prm->perm & IOMMU_FAULT_PERM_EXEC) {
access_flags |= VM_EXEC;
fault_flags |= FAULT_FLAG_INSTRUCTION;
}
if (!(prm->perm & IOMMU_FAULT_PERM_PRIV))
fault_flags |= FAULT_FLAG_USER;
if (access_flags & ~vma->vm_flags)
/* Access fault */
goto out_put_mm;
ret = handle_mm_fault(vma, prm->addr, fault_flags, NULL);
status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID :
IOMMU_PAGE_RESP_SUCCESS;
out_put_mm:
mmap_read_unlock(mm);
mmput(mm);
return status;
}

14
drivers/iommu/iommu-sva-lib.h → drivers/iommu/iommu-sva.h
View File

@ -2,8 +2,8 @@
/*
* SVA library for IOMMU drivers
*/
#ifndef _IOMMU_SVA_LIB_H
#define _IOMMU_SVA_LIB_H
#ifndef _IOMMU_SVA_H
#define _IOMMU_SVA_H
#include <linux/ioasid.h>
#include <linux/mm_types.h>
@ -26,6 +26,8 @@ int iopf_queue_flush_dev(struct device *dev);
struct iopf_queue *iopf_queue_alloc(const char *name);
void iopf_queue_free(struct iopf_queue *queue);
int iopf_queue_discard_partial(struct iopf_queue *queue);
enum iommu_page_response_code
iommu_sva_handle_iopf(struct iommu_fault *fault, void *data);
#else /* CONFIG_IOMMU_SVA */
static inline int iommu_queue_iopf(struct iommu_fault *fault, void *cookie)
@ -63,5 +65,11 @@ static inline int iopf_queue_discard_partial(struct iopf_queue *queue)
{
return -ENODEV;
}
static inline enum iommu_page_response_code
iommu_sva_handle_iopf(struct iommu_fault *fault, void *data)
{
return IOMMU_PAGE_RESP_INVALID;
}
#endif /* CONFIG_IOMMU_SVA */
#endif /* _IOMMU_SVA_LIB_H */
#endif /* _IOMMU_SVA_H */

329
drivers/iommu/iommu.c
View File

@ -21,6 +21,7 @@
#include <linux/idr.h>
#include <linux/err.h>
#include <linux/pci.h>
#include <linux/pci-ats.h>
#include <linux/bitops.h>
#include <linux/platform_device.h>
#include <linux/property.h>
@ -28,9 +29,12 @@
#include <linux/module.h>
#include <linux/cc_platform.h>
#include <trace/events/iommu.h>
#include <linux/sched/mm.h>
#include "dma-iommu.h"
#include "iommu-sva.h"
static struct kset *iommu_group_kset;
static DEFINE_IDA(iommu_group_ida);
@ -42,6 +46,7 @@ struct iommu_group {
struct kobject kobj;
struct kobject *devices_kobj;
struct list_head devices;
struct xarray pasid_array;
struct mutex mutex;
void *iommu_data;
void (*iommu_data_release)(void *iommu_data);
@ -278,18 +283,46 @@ static void dev_iommu_free(struct device *dev)
kfree(param);
}
static u32 dev_iommu_get_max_pasids(struct device *dev)
{
u32 max_pasids = 0, bits = 0;
int ret;
if (dev_is_pci(dev)) {
ret = pci_max_pasids(to_pci_dev(dev));
if (ret > 0)
max_pasids = ret;
} else {
ret = device_property_read_u32(dev, "pasid-num-bits", &bits);
if (!ret)
max_pasids = 1UL << bits;
}
return min_t(u32, max_pasids, dev->iommu->iommu_dev->max_pasids);
}
static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
{
const struct iommu_ops *ops = dev->bus->iommu_ops;
struct iommu_device *iommu_dev;
struct iommu_group *group;
static DEFINE_MUTEX(iommu_probe_device_lock);
int ret;
if (!ops)
return -ENODEV;
if (!dev_iommu_get(dev))
return -ENOMEM;
/*
* Serialise to avoid races between IOMMU drivers registering in
* parallel and/or the "replay" calls from ACPI/OF code via client
* driver probe. Once the latter have been cleaned up we should
* probably be able to use device_lock() here to minimise the scope,
* but for now enforcing a simple global ordering is fine.
*/
mutex_lock(&iommu_probe_device_lock);
if (!dev_iommu_get(dev)) {
ret = -ENOMEM;
goto err_unlock;
}
if (!try_module_get(ops->owner)) {
ret = -EINVAL;
@ -303,17 +336,21 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
}
dev->iommu->iommu_dev = iommu_dev;
dev->iommu->max_pasids = dev_iommu_get_max_pasids(dev);
group = iommu_group_get_for_dev(dev);
if (IS_ERR(group)) {
ret = PTR_ERR(group);
goto out_release;
}
iommu_group_put(group);
mutex_lock(&group->mutex);
if (group_list && !group->default_domain && list_empty(&group->entry))
list_add_tail(&group->entry, group_list);
mutex_unlock(&group->mutex);
iommu_group_put(group);
mutex_unlock(&iommu_probe_device_lock);
iommu_device_link(iommu_dev, dev);
return 0;
@ -328,6 +365,9 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
err_free:
dev_iommu_free(dev);
err_unlock:
mutex_unlock(&iommu_probe_device_lock);
return ret;
}
@ -703,6 +743,7 @@ struct iommu_group *iommu_group_alloc(void)
mutex_init(&group->mutex);
INIT_LIST_HEAD(&group->devices);
INIT_LIST_HEAD(&group->entry);
xa_init(&group->pasid_array);
ret = ida_alloc(&iommu_group_ida, GFP_KERNEL);
if (ret < 0) {
@ -1799,11 +1840,11 @@ int bus_iommu_probe(struct bus_type *bus)
return ret;
list_for_each_entry_safe(group, next, &group_list, entry) {
mutex_lock(&group->mutex);
/* Remove item from the list */
list_del_init(&group->entry);
mutex_lock(&group->mutex);
/* Try to allocate default domain */
probe_alloc_default_domain(bus, group);
@ -1912,6 +1953,8 @@ EXPORT_SYMBOL_GPL(iommu_domain_alloc);
void iommu_domain_free(struct iommu_domain *domain)
{
if (domain->type == IOMMU_DOMAIN_SVA)
mmdrop(domain->mm);
iommu_put_dma_cookie(domain);
domain->ops->free(domain);
}
@ -1949,6 +1992,18 @@ static int __iommu_attach_device(struct iommu_domain *domain,
return ret;
}
/**
* iommu_attach_device - Attach an IOMMU domain to a device
* @domain: IOMMU domain to attach
* @dev: Device that will be attached
*
* Returns 0 on success and error code on failure
*
* Note that EINVAL can be treated as a soft failure, indicating
* that certain configuration of the domain is incompatible with
* the device. In this case attaching a different domain to the
* device may succeed.
*/
int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
{
struct iommu_group *group;
@ -2075,6 +2130,18 @@ static int __iommu_attach_group(struct iommu_domain *domain,
return ret;
}
/**
* iommu_attach_group - Attach an IOMMU domain to an IOMMU group
* @domain: IOMMU domain to attach
* @group: IOMMU group that will be attached
*
* Returns 0 on success and error code on failure
*
* Note that EINVAL can be treated as a soft failure, indicating
* that certain configuration of the domain is incompatible with
* the group. In this case attaching a different domain to the
* group may succeed.
*/
int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)
{
int ret;
@ -2726,98 +2793,6 @@ int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
}
EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);
/**
* iommu_sva_bind_device() - Bind a process address space to a device
* @dev: the device
* @mm: the mm to bind, caller must hold a reference to it
* @drvdata: opaque data pointer to pass to bind callback
*
* Create a bond between device and address space, allowing the device to access
* the mm using the returned PASID. If a bond already exists between @device and
* @mm, it is returned and an additional reference is taken. Caller must call
* iommu_sva_unbind_device() to release each reference.
*
* iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to
* initialize the required SVA features.
*
* On error, returns an ERR_PTR value.
*/
struct iommu_sva *
iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)
{
struct iommu_group *group;
struct iommu_sva *handle = ERR_PTR(-EINVAL);
const struct iommu_ops *ops = dev_iommu_ops(dev);
if (!ops->sva_bind)
return ERR_PTR(-ENODEV);
group = iommu_group_get(dev);
if (!group)
return ERR_PTR(-ENODEV);
/* Ensure device count and domain don't change while we're binding */
mutex_lock(&group->mutex);
/*
* To keep things simple, SVA currently doesn't support IOMMU groups
* with more than one device. Existing SVA-capable systems are not
* affected by the problems that required IOMMU groups (lack of ACS
* isolation, device ID aliasing and other hardware issues).
*/
if (iommu_group_device_count(group) != 1)
goto out_unlock;
handle = ops->sva_bind(dev, mm, drvdata);
out_unlock:
mutex_unlock(&group->mutex);
iommu_group_put(group);
return handle;
}
EXPORT_SYMBOL_GPL(iommu_sva_bind_device);
/**
* iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device
* @handle: the handle returned by iommu_sva_bind_device()
*
* Put reference to a bond between device and address space. The device should
* not be issuing any more transaction for this PASID. All outstanding page
* requests for this PASID must have been flushed to the IOMMU.
*/
void iommu_sva_unbind_device(struct iommu_sva *handle)
{
struct iommu_group *group;
struct device *dev = handle->dev;
const struct iommu_ops *ops = dev_iommu_ops(dev);
if (!ops->sva_unbind)
return;
group = iommu_group_get(dev);
if (!group)
return;
mutex_lock(&group->mutex);
ops->sva_unbind(handle);
mutex_unlock(&group->mutex);
iommu_group_put(group);
}
EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
u32 iommu_sva_get_pasid(struct iommu_sva *handle)
{
const struct iommu_ops *ops = dev_iommu_ops(handle->dev);
if (!ops->sva_get_pasid)
return IOMMU_PASID_INVALID;
return ops->sva_get_pasid(handle);
}
EXPORT_SYMBOL_GPL(iommu_sva_get_pasid);
/*
* Changes the default domain of an iommu group that has *only* one device
*
@ -3087,7 +3062,8 @@ int iommu_device_use_default_domain(struct device *dev)
mutex_lock(&group->mutex);
if (group->owner_cnt) {
if (group->owner || !iommu_is_default_domain(group)) {
if (group->owner || !iommu_is_default_domain(group) ||
!xa_empty(&group->pasid_array)) {
ret = -EBUSY;
goto unlock_out;
}
@ -3118,7 +3094,7 @@ void iommu_device_unuse_default_domain(struct device *dev)
return;
mutex_lock(&group->mutex);
if (!WARN_ON(!group->owner_cnt))
if (!WARN_ON(!group->owner_cnt || !xa_empty(&group->pasid_array)))
group->owner_cnt--;
mutex_unlock(&group->mutex);
@ -3166,7 +3142,8 @@ int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner)
ret = -EPERM;
goto unlock_out;
} else {
if (group->domain && group->domain != group->default_domain) {
if ((group->domain && group->domain != group->default_domain) ||
!xa_empty(&group->pasid_array)) {
ret = -EBUSY;
goto unlock_out;
}
@ -3200,7 +3177,8 @@ void iommu_group_release_dma_owner(struct iommu_group *group)
int ret;
mutex_lock(&group->mutex);
if (WARN_ON(!group->owner_cnt || !group->owner))
if (WARN_ON(!group->owner_cnt || !group->owner ||
!xa_empty(&group->pasid_array)))
goto unlock_out;
group->owner_cnt = 0;
@ -3231,3 +3209,150 @@ bool iommu_group_dma_owner_claimed(struct iommu_group *group)
return user;
}
EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
static int __iommu_set_group_pasid(struct iommu_domain *domain,
struct iommu_group *group, ioasid_t pasid)
{
struct group_device *device;
int ret = 0;
list_for_each_entry(device, &group->devices, list) {
ret = domain->ops->set_dev_pasid(domain, device->dev, pasid);
if (ret)
break;
}
return ret;
}
static void __iommu_remove_group_pasid(struct iommu_group *group,
ioasid_t pasid)
{
struct group_device *device;
const struct iommu_ops *ops;
list_for_each_entry(device, &group->devices, list) {
ops = dev_iommu_ops(device->dev);
ops->remove_dev_pasid(device->dev, pasid);
}
}
/*
* iommu_attach_device_pasid() - Attach a domain to pasid of device
* @domain: the iommu domain.
* @dev: the attached device.
* @pasid: the pasid of the device.
*
* Return: 0 on success, or an error.
*/
int iommu_attach_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid)
{
struct iommu_group *group;
void *curr;
int ret;
if (!domain->ops->set_dev_pasid)
return -EOPNOTSUPP;
group = iommu_group_get(dev);
if (!group)
return -ENODEV;
mutex_lock(&group->mutex);
curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);
if (curr) {
ret = xa_err(curr) ? : -EBUSY;
goto out_unlock;
}
ret = __iommu_set_group_pasid(domain, group, pasid);
if (ret) {
__iommu_remove_group_pasid(group, pasid);
xa_erase(&group->pasid_array, pasid);
}
out_unlock:
mutex_unlock(&group->mutex);
iommu_group_put(group);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
/*
* iommu_detach_device_pasid() - Detach the domain from pasid of device
* @domain: the iommu domain.
* @dev: the attached device.
* @pasid: the pasid of the device.
*
* The @domain must have been attached to @pasid of the @dev with
* iommu_attach_device_pasid().
*/
void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
ioasid_t pasid)
{
struct iommu_group *group = iommu_group_get(dev);
mutex_lock(&group->mutex);
__iommu_remove_group_pasid(group, pasid);
WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
mutex_unlock(&group->mutex);
iommu_group_put(group);
}
EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
/*
* iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
* @dev: the queried device
* @pasid: the pasid of the device
* @type: matched domain type, 0 for any match
*
* This is a variant of iommu_get_domain_for_dev(). It returns the existing
* domain attached to pasid of a device. Callers must hold a lock around this
* function, and both iommu_attach/detach_dev_pasid() whenever a domain of
* type is being manipulated. This API does not internally resolve races with
* attach/detach.
*
* Return: attached domain on success, NULL otherwise.
*/
struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,
ioasid_t pasid,
unsigned int type)
{
struct iommu_domain *domain;
struct iommu_group *group;
group = iommu_group_get(dev);
if (!group)
return NULL;
xa_lock(&group->pasid_array);
domain = xa_load(&group->pasid_array, pasid);
if (type && domain && domain->type != type)
domain = ERR_PTR(-EBUSY);
xa_unlock(&group->pasid_array);
iommu_group_put(group);
return domain;
}
EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
struct mm_struct *mm)
{
const struct iommu_ops *ops = dev_iommu_ops(dev);
struct iommu_domain *domain;
domain = ops->domain_alloc(IOMMU_DOMAIN_SVA);
if (!domain)
return NULL;
domain->type = IOMMU_DOMAIN_SVA;
mmgrab(mm);
domain->mm = mm;
domain->iopf_handler = iommu_sva_handle_iopf;
domain->fault_data = mm;
return domain;
}

20
drivers/iommu/ipmmu-vmsa.c
View File

@ -628,8 +628,6 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
* Something is wrong, we can't attach two devices using
* different IOMMUs to the same domain.
*/
dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n",
dev_name(mmu->dev), dev_name(domain->mmu->dev));
ret = -EINVAL;
} else
dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id);
@ -661,22 +659,22 @@ static void ipmmu_detach_device(struct iommu_domain *io_domain,
}
static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
if (!domain)
return -ENODEV;
return domain->iop->map(domain->iop, iova, paddr, size, prot, gfp);
return domain->iop->map_pages(domain->iop, iova, paddr, pgsize, pgcount,
prot, gfp, mapped);
}
static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
return domain->iop->unmap(domain->iop, iova, size, gather);
return domain->iop->unmap_pages(domain->iop, iova, pgsize, pgcount, gather);
}
static void ipmmu_flush_iotlb_all(struct iommu_domain *io_domain)
@ -879,8 +877,8 @@ static const struct iommu_ops ipmmu_ops = {
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = ipmmu_attach_device,
.detach_dev = ipmmu_detach_device,
.map = ipmmu_map,
.unmap = ipmmu_unmap,
.map_pages = ipmmu_map,
.unmap_pages = ipmmu_unmap,
.flush_iotlb_all = ipmmu_flush_iotlb_all,
.iotlb_sync = ipmmu_iotlb_sync,
.iova_to_phys = ipmmu_iova_to_phys,

18
drivers/iommu/msm_iommu.c
View File

@ -471,14 +471,16 @@ static void msm_iommu_detach_dev(struct iommu_domain *domain,
}
static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t pa, size_t len, int prot, gfp_t gfp)
phys_addr_t pa, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct msm_priv *priv = to_msm_priv(domain);
unsigned long flags;
int ret;
spin_lock_irqsave(&priv->pgtlock, flags);
ret = priv->iop->map(priv->iop, iova, pa, len, prot, GFP_ATOMIC);
ret = priv->iop->map_pages(priv->iop, iova, pa, pgsize, pgcount, prot,
GFP_ATOMIC, mapped);
spin_unlock_irqrestore(&priv->pgtlock, flags);
return ret;
@ -493,16 +495,18 @@ static void msm_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t len, struct iommu_iotlb_gather *gather)
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
struct msm_priv *priv = to_msm_priv(domain);
unsigned long flags;
size_t ret;
spin_lock_irqsave(&priv->pgtlock, flags);
len = priv->iop->unmap(priv->iop, iova, len, gather);
ret = priv->iop->unmap_pages(priv->iop, iova, pgsize, pgcount, gather);
spin_unlock_irqrestore(&priv->pgtlock, flags);
return len;
return ret;
}
static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
@ -679,8 +683,8 @@ static struct iommu_ops msm_iommu_ops = {
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = msm_iommu_attach_dev,
.detach_dev = msm_iommu_detach_dev,
.map = msm_iommu_map,
.unmap = msm_iommu_unmap,
.map_pages = msm_iommu_map,
.unmap_pages = msm_iommu_unmap,
/*
* Nothing is needed here, the barrier to guarantee
* completion of the tlb sync operation is implicitly

153
drivers/iommu/mtk_iommu.c
View File

@ -108,8 +108,12 @@
#define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3)
#define F_MMU_INT_ID_COMM_ID_EXT(a) (((a) >> 10) & 0x7)
#define F_MMU_INT_ID_SUB_COMM_ID_EXT(a) (((a) >> 7) & 0x7)
/* Macro for 5 bits length port ID field (default) */
#define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7)
#define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f)
/* Macro for 6 bits length port ID field */
#define F_MMU_INT_ID_LARB_ID_WID_6(a) (((a) >> 8) & 0x7)
#define F_MMU_INT_ID_PORT_ID_WID_6(a) (((a) >> 2) & 0x3f)
#define MTK_PROTECT_PA_ALIGN 256
#define MTK_IOMMU_BANK_SZ 0x1000
@ -139,6 +143,7 @@
#define IFA_IOMMU_PCIE_SUPPORT BIT(16)
#define PGTABLE_PA_35_EN BIT(17)
#define TF_PORT_TO_ADDR_MT8173 BIT(18)
#define INT_ID_PORT_WIDTH_6 BIT(19)
#define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) \
((((pdata)->flags) & (mask)) == (_x))
@ -165,6 +170,7 @@ enum mtk_iommu_plat {
M4U_MT8186,
M4U_MT8192,
M4U_MT8195,
M4U_MT8365,
};
struct mtk_iommu_iova_region {
@ -223,10 +229,7 @@ struct mtk_iommu_data {
struct device *smicomm_dev;
struct mtk_iommu_bank_data *bank;
struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */
struct regmap *pericfg;
struct mutex mutex; /* Protect m4u_group/m4u_dom above */
/*
@ -441,20 +444,25 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
fault_pa |= (u64)pa34_32 << 32;
if (MTK_IOMMU_IS_TYPE(plat_data, MTK_IOMMU_TYPE_MM)) {
fault_port = F_MMU_INT_ID_PORT_ID(regval);
if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_2BITS)) {
fault_larb = F_MMU_INT_ID_COMM_ID(regval);
sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval);
fault_port = F_MMU_INT_ID_PORT_ID(regval);
} else if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_3BITS)) {
fault_larb = F_MMU_INT_ID_COMM_ID_EXT(regval);
sub_comm = F_MMU_INT_ID_SUB_COMM_ID_EXT(regval);
fault_port = F_MMU_INT_ID_PORT_ID(regval);
} else if (MTK_IOMMU_HAS_FLAG(plat_data, INT_ID_PORT_WIDTH_6)) {
fault_port = F_MMU_INT_ID_PORT_ID_WID_6(regval);
fault_larb = F_MMU_INT_ID_LARB_ID_WID_6(regval);
} else {
fault_port = F_MMU_INT_ID_PORT_ID(regval);
fault_larb = F_MMU_INT_ID_LARB_ID(regval);
}
fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm];
}
if (report_iommu_fault(&dom->domain, bank->parent_dev, fault_iova,
if (!dom || report_iommu_fault(&dom->domain, bank->parent_dev, fault_iova,
write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) {
dev_err_ratelimited(
bank->parent_dev,
@ -609,7 +617,7 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom,
dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data);
if (!dom->iop) {
dev_err(data->dev, "Failed to alloc io pgtable\n");
return -EINVAL;
return -ENOMEM;
}
/* Update our support page sizes bitmap */
@ -668,7 +676,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain,
ret = mtk_iommu_domain_finalise(dom, frstdata, region_id);
if (ret) {
mutex_unlock(&dom->mutex);
return -ENODEV;
return ret;
}
dom->bank = &data->bank[bankid];
}
@ -711,7 +719,8 @@ static void mtk_iommu_detach_device(struct iommu_domain *domain,
}
static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
@ -720,17 +729,17 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
paddr |= BIT_ULL(32);
/* Synchronize with the tlb_lock */
return dom->iop->map(dom->iop, iova, paddr, size, prot, gfp);
return dom->iop->map_pages(dom->iop, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
}
static size_t mtk_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
unsigned long iova, size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
iommu_iotlb_gather_add_range(gather, iova, size);
return dom->iop->unmap(dom->iop, iova, size, gather);
iommu_iotlb_gather_add_range(gather, iova, pgsize * pgcount);
return dom->iop->unmap_pages(dom->iop, iova, pgsize, pgcount, gather);
}
static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain)
@ -938,8 +947,8 @@ static const struct iommu_ops mtk_iommu_ops = {
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = mtk_iommu_attach_device,
.detach_dev = mtk_iommu_detach_device,
.map = mtk_iommu_map,
.unmap = mtk_iommu_unmap,
.map_pages = mtk_iommu_map,
.unmap_pages = mtk_iommu_unmap,
.flush_iotlb_all = mtk_iommu_flush_iotlb_all,
.iotlb_sync = mtk_iommu_iotlb_sync,
.iotlb_sync_map = mtk_iommu_sync_map,
@ -1043,21 +1052,26 @@ static const struct component_master_ops mtk_iommu_com_ops = {
static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **match,
struct mtk_iommu_data *data)
{
struct device_node *larbnode, *smicomm_node, *smi_subcomm_node;
struct platform_device *plarbdev;
struct device_node *larbnode, *frst_avail_smicomm_node = NULL;
struct platform_device *plarbdev, *pcommdev;
struct device_link *link;
int i, larb_nr, ret;
larb_nr = of_count_phandle_with_args(dev->of_node, "mediatek,larbs", NULL);
if (larb_nr < 0)
return larb_nr;
if (larb_nr == 0 || larb_nr > MTK_LARB_NR_MAX)
return -EINVAL;
for (i = 0; i < larb_nr; i++) {
struct device_node *smicomm_node, *smi_subcomm_node;
u32 id;
larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i);
if (!larbnode)
return -EINVAL;
if (!larbnode) {
ret = -EINVAL;
goto err_larbdev_put;
}
if (!of_device_is_available(larbnode)) {
of_node_put(larbnode);
@ -1067,48 +1081,91 @@ static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **m
ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id);
if (ret)/* The id is consecutive if there is no this property */
id = i;
if (id >= MTK_LARB_NR_MAX) {
of_node_put(larbnode);
ret = -EINVAL;
goto err_larbdev_put;
}
plarbdev = of_find_device_by_node(larbnode);
of_node_put(larbnode);
if (!plarbdev) {
of_node_put(larbnode);
return -ENODEV;
ret = -ENODEV;
goto err_larbdev_put;
}
if (!plarbdev->dev.driver) {
of_node_put(larbnode);
return -EPROBE_DEFER;
if (data->larb_imu[id].dev) {
platform_device_put(plarbdev);
ret = -EEXIST;
goto err_larbdev_put;
}
data->larb_imu[id].dev = &plarbdev->dev;
component_match_add_release(dev, match, component_release_of,
component_compare_of, larbnode);
if (!plarbdev->dev.driver) {
ret = -EPROBE_DEFER;
goto err_larbdev_put;
}
/* Get smi-(sub)-common dev from the last larb. */
smi_subcomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0);
if (!smi_subcomm_node) {
ret = -EINVAL;
goto err_larbdev_put;
}
/*
* It may have two level smi-common. the node is smi-sub-common if it
* has a new mediatek,smi property. otherwise it is smi-commmon.
*/
smicomm_node = of_parse_phandle(smi_subcomm_node, "mediatek,smi", 0);
if (smicomm_node)
of_node_put(smi_subcomm_node);
else
smicomm_node = smi_subcomm_node;
/*
* All the larbs that connect to one IOMMU must connect with the same
* smi-common.
*/
if (!frst_avail_smicomm_node) {
frst_avail_smicomm_node = smicomm_node;
} else if (frst_avail_smicomm_node != smicomm_node) {
dev_err(dev, "mediatek,smi property is not right @larb%d.", id);
of_node_put(smicomm_node);
ret = -EINVAL;
goto err_larbdev_put;
} else {
of_node_put(smicomm_node);
}
component_match_add(dev, match, component_compare_dev, &plarbdev->dev);
platform_device_put(plarbdev);
}
/* Get smi-(sub)-common dev from the last larb. */
smi_subcomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0);
if (!smi_subcomm_node)
if (!frst_avail_smicomm_node)
return -EINVAL;
/*
* It may have two level smi-common. the node is smi-sub-common if it
* has a new mediatek,smi property. otherwise it is smi-commmon.
*/
smicomm_node = of_parse_phandle(smi_subcomm_node, "mediatek,smi", 0);
if (smicomm_node)
of_node_put(smi_subcomm_node);
else
smicomm_node = smi_subcomm_node;
plarbdev = of_find_device_by_node(smicomm_node);
of_node_put(smicomm_node);
data->smicomm_dev = &plarbdev->dev;
pcommdev = of_find_device_by_node(frst_avail_smicomm_node);
of_node_put(frst_avail_smicomm_node);
if (!pcommdev)
return -ENODEV;
data->smicomm_dev = &pcommdev->dev;
link = device_link_add(data->smicomm_dev, dev,
DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
platform_device_put(pcommdev);
if (!link) {
dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev));
return -EINVAL;
}
return 0;
err_larbdev_put:
for (i = MTK_LARB_NR_MAX - 1; i >= 0; i--) {
if (!data->larb_imu[i].dev)
continue;
put_device(data->larb_imu[i].dev);
}
return ret;
}
static int mtk_iommu_probe(struct platform_device *pdev)
@ -1173,6 +1230,8 @@ static int mtk_iommu_probe(struct platform_device *pdev)
banks_num = data->plat_data->banks_num;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!res)
return -EINVAL;
if (resource_size(res) < banks_num * MTK_IOMMU_BANK_SZ) {
dev_err(dev, "banknr %d. res %pR is not enough.\n", banks_num, res);
return -EINVAL;
@ -1516,6 +1575,17 @@ static const struct mtk_iommu_plat_data mt8195_data_vpp = {
{4, MTK_INVALID_LARBID, MTK_INVALID_LARBID, MTK_INVALID_LARBID, 6}},
};
static const struct mtk_iommu_plat_data mt8365_data = {
.m4u_plat = M4U_MT8365,
.flags = RESET_AXI | INT_ID_PORT_WIDTH_6,
.inv_sel_reg = REG_MMU_INV_SEL_GEN1,
.banks_num = 1,
.banks_enable = {true},
.iova_region = single_domain,
.iova_region_nr = ARRAY_SIZE(single_domain),
.larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */
};
static const struct of_device_id mtk_iommu_of_ids[] = {
{ .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data},
{ .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data},
@ -1528,6 +1598,7 @@ static const struct of_device_id mtk_iommu_of_ids[] = {
{ .compatible = "mediatek,mt8195-iommu-infra", .data = &mt8195_data_infra},
{ .compatible = "mediatek,mt8195-iommu-vdo", .data = &mt8195_data_vdo},
{ .compatible = "mediatek,mt8195-iommu-vpp", .data = &mt8195_data_vpp},
{ .compatible = "mediatek,mt8365-m4u", .data = &mt8365_data},
{}
};

30
drivers/iommu/mtk_iommu_v1.c
View File

@ -327,44 +327,42 @@ static void mtk_iommu_v1_detach_device(struct iommu_domain *domain, struct devic
}
static int mtk_iommu_v1_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct mtk_iommu_v1_domain *dom = to_mtk_domain(domain);
unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT;
unsigned long flags;
unsigned int i;
u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT);
u32 pabase = (u32)paddr;
int map_size = 0;
spin_lock_irqsave(&dom->pgtlock, flags);
for (i = 0; i < page_num; i++) {
if (pgt_base_iova[i]) {
memset(pgt_base_iova, 0, i * sizeof(u32));
for (i = 0; i < pgcount; i++) {
if (pgt_base_iova[i])
break;
}
pgt_base_iova[i] = pabase | F_DESC_VALID | F_DESC_NONSEC;
pabase += MT2701_IOMMU_PAGE_SIZE;
map_size += MT2701_IOMMU_PAGE_SIZE;
}
spin_unlock_irqrestore(&dom->pgtlock, flags);
mtk_iommu_v1_tlb_flush_range(dom->data, iova, size);
*mapped = i * MT2701_IOMMU_PAGE_SIZE;
mtk_iommu_v1_tlb_flush_range(dom->data, iova, *mapped);
return map_size == size ? 0 : -EEXIST;
return i == pgcount ? 0 : -EEXIST;
}
static size_t mtk_iommu_v1_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
struct mtk_iommu_v1_domain *dom = to_mtk_domain(domain);
unsigned long flags;
u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT);
unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT;
size_t size = pgcount * MT2701_IOMMU_PAGE_SIZE;
spin_lock_irqsave(&dom->pgtlock, flags);
memset(pgt_base_iova, 0, page_num * sizeof(u32));
memset(pgt_base_iova, 0, pgcount * sizeof(u32));
spin_unlock_irqrestore(&dom->pgtlock, flags);
mtk_iommu_v1_tlb_flush_range(dom->data, iova, size);
@ -586,13 +584,13 @@ static const struct iommu_ops mtk_iommu_v1_ops = {
.release_device = mtk_iommu_v1_release_device,
.def_domain_type = mtk_iommu_v1_def_domain_type,
.device_group = generic_device_group,
.pgsize_bitmap = ~0UL << MT2701_IOMMU_PAGE_SHIFT,
.pgsize_bitmap = MT2701_IOMMU_PAGE_SIZE,
.owner = THIS_MODULE,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = mtk_iommu_v1_attach_device,
.detach_dev = mtk_iommu_v1_detach_device,
.map = mtk_iommu_v1_map,
.unmap = mtk_iommu_v1_unmap,
.map_pages = mtk_iommu_v1_map,
.unmap_pages = mtk_iommu_v1_unmap,
.iova_to_phys = mtk_iommu_v1_iova_to_phys,
.free = mtk_iommu_v1_domain_free,
}

6
drivers/iommu/omap-iommu.c
View File

@ -1414,7 +1414,7 @@ static int omap_iommu_attach_init(struct device *dev,
odomain->num_iommus = omap_iommu_count(dev);
if (!odomain->num_iommus)
return -EINVAL;
return -ENODEV;
odomain->iommus = kcalloc(odomain->num_iommus, sizeof(*iommu),
GFP_ATOMIC);
@ -1464,7 +1464,7 @@ omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
if (!arch_data || !arch_data->iommu_dev) {
dev_err(dev, "device doesn't have an associated iommu\n");
return -EINVAL;
return -ENODEV;
}
spin_lock(&omap_domain->lock);
@ -1472,7 +1472,7 @@ omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
/* only a single client device can be attached to a domain */
if (omap_domain->dev) {
dev_err(dev, "iommu domain is already attached\n");
ret = -EBUSY;
ret = -EINVAL;
goto out;
}

10
drivers/iommu/rockchip-iommu.c
View File

@ -280,19 +280,17 @@ static u32 rk_mk_pte(phys_addr_t page, int prot)
* 11:9 - Page address bit 34:32
* 8:4 - Page address bit 39:35
* 3 - Security
* 2 - Readable
* 1 - Writable
* 2 - Writable
* 1 - Readable
* 0 - 1 if Page @ Page address is valid
*/
#define RK_PTE_PAGE_READABLE_V2 BIT(2)
#define RK_PTE_PAGE_WRITABLE_V2 BIT(1)
static u32 rk_mk_pte_v2(phys_addr_t page, int prot)
{
u32 flags = 0;
flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE_V2 : 0;
flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE_V2 : 0;
flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0;
flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0;
return rk_mk_dte_v2(page) | flags;
}

383
drivers/iommu/s390-iommu.c
View File

@ -10,28 +10,18 @@
#include <linux/iommu.h>
#include <linux/iommu-helper.h>
#include <linux/sizes.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <asm/pci_dma.h>
/*
* Physically contiguous memory regions can be mapped with 4 KiB alignment,
* we allow all page sizes that are an order of 4KiB (no special large page
* support so far).
*/
#define S390_IOMMU_PGSIZES (~0xFFFUL)
static const struct iommu_ops s390_iommu_ops;
struct s390_domain {
struct iommu_domain domain;
struct list_head devices;
unsigned long *dma_table;
spinlock_t dma_table_lock;
spinlock_t list_lock;
};
struct s390_domain_device {
struct list_head list;
struct zpci_dev *zdev;
struct rcu_head rcu;
};
static struct s390_domain *to_s390_domain(struct iommu_domain *dom)
@ -67,20 +57,50 @@ static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
kfree(s390_domain);
return NULL;
}
s390_domain->domain.geometry.force_aperture = true;
s390_domain->domain.geometry.aperture_start = 0;
s390_domain->domain.geometry.aperture_end = ZPCI_TABLE_SIZE_RT - 1;
spin_lock_init(&s390_domain->dma_table_lock);
spin_lock_init(&s390_domain->list_lock);
INIT_LIST_HEAD(&s390_domain->devices);
INIT_LIST_HEAD_RCU(&s390_domain->devices);
return &s390_domain->domain;
}
static void s390_iommu_rcu_free_domain(struct rcu_head *head)
{
struct s390_domain *s390_domain = container_of(head, struct s390_domain, rcu);
dma_cleanup_tables(s390_domain->dma_table);
kfree(s390_domain);
}
static void s390_domain_free(struct iommu_domain *domain)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
dma_cleanup_tables(s390_domain->dma_table);
kfree(s390_domain);
rcu_read_lock();
WARN_ON(!list_empty(&s390_domain->devices));
rcu_read_unlock();
call_rcu(&s390_domain->rcu, s390_iommu_rcu_free_domain);
}
static void __s390_iommu_detach_device(struct zpci_dev *zdev)
{
struct s390_domain *s390_domain = zdev->s390_domain;
unsigned long flags;
if (!s390_domain)
return;
spin_lock_irqsave(&s390_domain->list_lock, flags);
list_del_rcu(&zdev->iommu_list);
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
zpci_unregister_ioat(zdev, 0);
zdev->s390_domain = NULL;
zdev->dma_table = NULL;
}
static int s390_iommu_attach_device(struct iommu_domain *domain,
@ -88,98 +108,74 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
{
struct s390_domain *s390_domain = to_s390_domain(domain);
struct zpci_dev *zdev = to_zpci_dev(dev);
struct s390_domain_device *domain_device;
unsigned long flags;
int cc, rc;
u8 status;
int cc;
if (!zdev)
return -ENODEV;
domain_device = kzalloc(sizeof(*domain_device), GFP_KERNEL);
if (!domain_device)
return -ENOMEM;
if (zdev->dma_table && !zdev->s390_domain) {
cc = zpci_dma_exit_device(zdev);
if (cc) {
rc = -EIO;
goto out_free;
}
}
if (WARN_ON(domain->geometry.aperture_start > zdev->end_dma ||
domain->geometry.aperture_end < zdev->start_dma))
return -EINVAL;
if (zdev->s390_domain)
zpci_unregister_ioat(zdev, 0);
__s390_iommu_detach_device(zdev);
else if (zdev->dma_table)
zpci_dma_exit_device(zdev);
cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(s390_domain->dma_table), &status);
/*
* If the device is undergoing error recovery the reset code
* will re-establish the new domain.
*/
if (cc && status != ZPCI_PCI_ST_FUNC_NOT_AVAIL)
return -EIO;
zdev->dma_table = s390_domain->dma_table;
zdev->dma_table = s390_domain->dma_table;
cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table));
if (cc) {
rc = -EIO;
goto out_restore;
}
zdev->s390_domain = s390_domain;
spin_lock_irqsave(&s390_domain->list_lock, flags);
/* First device defines the DMA range limits */
if (list_empty(&s390_domain->devices)) {
domain->geometry.aperture_start = zdev->start_dma;
domain->geometry.aperture_end = zdev->end_dma;
domain->geometry.force_aperture = true;
/* Allow only devices with identical DMA range limits */
} else if (domain->geometry.aperture_start != zdev->start_dma ||
domain->geometry.aperture_end != zdev->end_dma) {
rc = -EINVAL;
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
goto out_restore;
}
domain_device->zdev = zdev;
zdev->s390_domain = s390_domain;
list_add(&domain_device->list, &s390_domain->devices);
list_add_rcu(&zdev->iommu_list, &s390_domain->devices);
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
return 0;
out_restore:
if (!zdev->s390_domain) {
zpci_dma_init_device(zdev);
} else {
zdev->dma_table = zdev->s390_domain->dma_table;
zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table));
}
out_free:
kfree(domain_device);
return rc;
}
static void s390_iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
struct zpci_dev *zdev = to_zpci_dev(dev);
struct s390_domain_device *domain_device, *tmp;
unsigned long flags;
int found = 0;
if (!zdev)
return;
WARN_ON(zdev->s390_domain != to_s390_domain(domain));
spin_lock_irqsave(&s390_domain->list_lock, flags);
list_for_each_entry_safe(domain_device, tmp, &s390_domain->devices,
list) {
if (domain_device->zdev == zdev) {
list_del(&domain_device->list);
kfree(domain_device);
found = 1;
break;
}
__s390_iommu_detach_device(zdev);
zpci_dma_init_device(zdev);
}
static void s390_iommu_get_resv_regions(struct device *dev,
struct list_head *list)
{
struct zpci_dev *zdev = to_zpci_dev(dev);
struct iommu_resv_region *region;
if (zdev->start_dma) {
region = iommu_alloc_resv_region(0, zdev->start_dma, 0,
IOMMU_RESV_RESERVED, GFP_KERNEL);
if (!region)
return;
list_add_tail(&region->list, list);
}
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
if (found && (zdev->s390_domain == s390_domain)) {
zdev->s390_domain = NULL;
zpci_unregister_ioat(zdev, 0);
zpci_dma_init_device(zdev);
if (zdev->end_dma < ZPCI_TABLE_SIZE_RT - 1) {
region = iommu_alloc_resv_region(zdev->end_dma + 1,
ZPCI_TABLE_SIZE_RT - zdev->end_dma - 1,
0, IOMMU_RESV_RESERVED, GFP_KERNEL);
if (!region)
return;
list_add_tail(&region->list, list);
}
}
@ -192,55 +188,88 @@ static struct iommu_device *s390_iommu_probe_device(struct device *dev)
zdev = to_zpci_dev(dev);
if (zdev->start_dma > zdev->end_dma ||
zdev->start_dma > ZPCI_TABLE_SIZE_RT - 1)
return ERR_PTR(-EINVAL);
if (zdev->end_dma > ZPCI_TABLE_SIZE_RT - 1)
zdev->end_dma = ZPCI_TABLE_SIZE_RT - 1;
return &zdev->iommu_dev;
}
static void s390_iommu_release_device(struct device *dev)
{
struct zpci_dev *zdev = to_zpci_dev(dev);
struct iommu_domain *domain;
/*
* This is a workaround for a scenario where the IOMMU API common code
* "forgets" to call the detach_dev callback: After binding a device
* to vfio-pci and completing the VFIO_SET_IOMMU ioctl (which triggers
* the attach_dev), removing the device via
* "echo 1 > /sys/bus/pci/devices/.../remove" won't trigger detach_dev,
* only release_device will be called via the BUS_NOTIFY_REMOVED_DEVICE
* notifier.
*
* So let's call detach_dev from here if it hasn't been called before.
* release_device is expected to detach any domain currently attached
* to the device, but keep it attached to other devices in the group.
*/
if (zdev && zdev->s390_domain) {
domain = iommu_get_domain_for_dev(dev);
if (domain)
s390_iommu_detach_device(domain, dev);
}
if (zdev)
__s390_iommu_detach_device(zdev);
}
static int s390_iommu_update_trans(struct s390_domain *s390_domain,
phys_addr_t pa, dma_addr_t dma_addr,
size_t size, int flags)
static void s390_iommu_flush_iotlb_all(struct iommu_domain *domain)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
struct zpci_dev *zdev;
rcu_read_lock();
list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) {
zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma,
zdev->end_dma - zdev->start_dma + 1);
}
rcu_read_unlock();
}
static void s390_iommu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
size_t size = gather->end - gather->start + 1;
struct zpci_dev *zdev;
/* If gather was never added to there is nothing to flush */
if (!gather->end)
return;
rcu_read_lock();
list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) {
zpci_refresh_trans((u64)zdev->fh << 32, gather->start,
size);
}
rcu_read_unlock();
}
static void s390_iommu_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
struct zpci_dev *zdev;
rcu_read_lock();
list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) {
if (!zdev->tlb_refresh)
continue;
zpci_refresh_trans((u64)zdev->fh << 32,
iova, size);
}
rcu_read_unlock();
}
static int s390_iommu_validate_trans(struct s390_domain *s390_domain,
phys_addr_t pa, dma_addr_t dma_addr,
unsigned long nr_pages, int flags)
{
struct s390_domain_device *domain_device;
phys_addr_t page_addr = pa & PAGE_MASK;
dma_addr_t start_dma_addr = dma_addr;
unsigned long irq_flags, nr_pages, i;
unsigned long *entry;
int rc = 0;
unsigned long i;
int rc;
if (dma_addr < s390_domain->domain.geometry.aperture_start ||
dma_addr + size > s390_domain->domain.geometry.aperture_end)
return -EINVAL;
nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
if (!nr_pages)
return 0;
spin_lock_irqsave(&s390_domain->dma_table_lock, irq_flags);
for (i = 0; i < nr_pages; i++) {
entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr);
if (!entry) {
if (unlikely(!entry)) {
rc = -ENOMEM;
goto undo_cpu_trans;
}
@ -249,47 +278,70 @@ static int s390_iommu_update_trans(struct s390_domain *s390_domain,
dma_addr += PAGE_SIZE;
}
spin_lock(&s390_domain->list_lock);
list_for_each_entry(domain_device, &s390_domain->devices, list) {
rc = zpci_refresh_trans((u64) domain_device->zdev->fh << 32,
start_dma_addr, nr_pages * PAGE_SIZE);
if (rc)
break;
}
spin_unlock(&s390_domain->list_lock);
return 0;
undo_cpu_trans:
if (rc && ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID)) {
flags = ZPCI_PTE_INVALID;
while (i-- > 0) {
page_addr -= PAGE_SIZE;
dma_addr -= PAGE_SIZE;
entry = dma_walk_cpu_trans(s390_domain->dma_table,
dma_addr);
if (!entry)
break;
dma_update_cpu_trans(entry, page_addr, flags);
}
while (i-- > 0) {
dma_addr -= PAGE_SIZE;
entry = dma_walk_cpu_trans(s390_domain->dma_table,
dma_addr);
if (!entry)
break;
dma_update_cpu_trans(entry, 0, ZPCI_PTE_INVALID);
}
spin_unlock_irqrestore(&s390_domain->dma_table_lock, irq_flags);
return rc;
}
static int s390_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
static int s390_iommu_invalidate_trans(struct s390_domain *s390_domain,
dma_addr_t dma_addr, unsigned long nr_pages)
{
unsigned long *entry;
unsigned long i;
int rc = 0;
for (i = 0; i < nr_pages; i++) {
entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr);
if (unlikely(!entry)) {
rc = -EINVAL;
break;
}
dma_update_cpu_trans(entry, 0, ZPCI_PTE_INVALID);
dma_addr += PAGE_SIZE;
}
return rc;
}
static int s390_iommu_map_pages(struct iommu_domain *domain,
unsigned long iova, phys_addr_t paddr,
size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
size_t size = pgcount << __ffs(pgsize);
int flags = ZPCI_PTE_VALID, rc = 0;
if (pgsize != SZ_4K)
return -EINVAL;
if (iova < s390_domain->domain.geometry.aperture_start ||
(iova + size - 1) > s390_domain->domain.geometry.aperture_end)
return -EINVAL;
if (!IS_ALIGNED(iova | paddr, pgsize))
return -EINVAL;
if (!(prot & IOMMU_READ))
return -EINVAL;
if (!(prot & IOMMU_WRITE))
flags |= ZPCI_TABLE_PROTECTED;
rc = s390_iommu_update_trans(s390_domain, paddr, iova,
size, flags);
rc = s390_iommu_validate_trans(s390_domain, paddr, iova,
pgcount, flags);
if (!rc)
*mapped = size;
return rc;
}
@ -298,7 +350,8 @@ static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
unsigned long *sto, *pto, *rto, flags;
unsigned long *rto, *sto, *pto;
unsigned long ste, pte, rte;
unsigned int rtx, sx, px;
phys_addr_t phys = 0;
@ -311,38 +364,40 @@ static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
px = calc_px(iova);
rto = s390_domain->dma_table;
spin_lock_irqsave(&s390_domain->dma_table_lock, flags);
if (rto && reg_entry_isvalid(rto[rtx])) {
sto = get_rt_sto(rto[rtx]);
if (sto && reg_entry_isvalid(sto[sx])) {
pto = get_st_pto(sto[sx]);
if (pto && pt_entry_isvalid(pto[px]))
phys = pto[px] & ZPCI_PTE_ADDR_MASK;
rte = READ_ONCE(rto[rtx]);
if (reg_entry_isvalid(rte)) {
sto = get_rt_sto(rte);
ste = READ_ONCE(sto[sx]);
if (reg_entry_isvalid(ste)) {
pto = get_st_pto(ste);
pte = READ_ONCE(pto[px]);
if (pt_entry_isvalid(pte))
phys = pte & ZPCI_PTE_ADDR_MASK;
}
}
spin_unlock_irqrestore(&s390_domain->dma_table_lock, flags);
return phys;
}
static size_t s390_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
struct iommu_iotlb_gather *gather)
static size_t s390_iommu_unmap_pages(struct iommu_domain *domain,
unsigned long iova,
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
int flags = ZPCI_PTE_INVALID;
phys_addr_t paddr;
size_t size = pgcount << __ffs(pgsize);
int rc;
paddr = s390_iommu_iova_to_phys(domain, iova);
if (!paddr)
if (WARN_ON(iova < s390_domain->domain.geometry.aperture_start ||
(iova + size - 1) > s390_domain->domain.geometry.aperture_end))
return 0;
rc = s390_iommu_update_trans(s390_domain, paddr, iova,
size, flags);
rc = s390_iommu_invalidate_trans(s390_domain, iova, pgcount);
if (rc)
return 0;
iommu_iotlb_gather_add_range(gather, iova, size);
return size;
}
@ -380,12 +435,16 @@ static const struct iommu_ops s390_iommu_ops = {
.probe_device = s390_iommu_probe_device,
.release_device = s390_iommu_release_device,
.device_group = generic_device_group,
.pgsize_bitmap = S390_IOMMU_PGSIZES,
.pgsize_bitmap = SZ_4K,
.get_resv_regions = s390_iommu_get_resv_regions,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = s390_iommu_attach_device,
.detach_dev = s390_iommu_detach_device,
.map = s390_iommu_map,
.unmap = s390_iommu_unmap,
.map_pages = s390_iommu_map_pages,
.unmap_pages = s390_iommu_unmap_pages,
.flush_iotlb_all = s390_iommu_flush_iotlb_all,
.iotlb_sync = s390_iommu_iotlb_sync,
.iotlb_sync_map = s390_iommu_iotlb_sync_map,
.iova_to_phys = s390_iommu_iova_to_phys,
.free = s390_domain_free,
}

29
drivers/iommu/sprd-iommu.c
View File

@ -237,10 +237,8 @@ static int sprd_iommu_attach_device(struct iommu_domain *domain,
struct sprd_iommu_domain *dom = to_sprd_domain(domain);
size_t pgt_size = sprd_iommu_pgt_size(domain);
if (dom->sdev) {
pr_err("There's already a device attached to this domain.\n");
if (dom->sdev)
return -EINVAL;
}
dom->pgt_va = dma_alloc_coherent(sdev->dev, pgt_size, &dom->pgt_pa, GFP_KERNEL);
if (!dom->pgt_va)
@ -273,10 +271,11 @@ static void sprd_iommu_detach_device(struct iommu_domain *domain,
}
static int sprd_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct sprd_iommu_domain *dom = to_sprd_domain(domain);
unsigned int page_num = size >> SPRD_IOMMU_PAGE_SHIFT;
size_t size = pgcount * SPRD_IOMMU_PAGE_SIZE;
unsigned long flags;
unsigned int i;
u32 *pgt_base_iova;
@ -298,35 +297,37 @@ static int sprd_iommu_map(struct iommu_domain *domain, unsigned long iova,
pgt_base_iova = dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT);
spin_lock_irqsave(&dom->pgtlock, flags);
for (i = 0; i < page_num; i++) {
for (i = 0; i < pgcount; i++) {
pgt_base_iova[i] = pabase >> SPRD_IOMMU_PAGE_SHIFT;
pabase += SPRD_IOMMU_PAGE_SIZE;
}
spin_unlock_irqrestore(&dom->pgtlock, flags);
*mapped = size;
return 0;
}
static size_t sprd_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *iotlb_gather)
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *iotlb_gather)
{
struct sprd_iommu_domain *dom = to_sprd_domain(domain);
unsigned long flags;
u32 *pgt_base_iova;
unsigned int page_num = size >> SPRD_IOMMU_PAGE_SHIFT;
size_t size = pgcount * SPRD_IOMMU_PAGE_SIZE;
unsigned long start = domain->geometry.aperture_start;
unsigned long end = domain->geometry.aperture_end;
if (iova < start || (iova + size) > (end + 1))
return -EINVAL;
return 0;
pgt_base_iova = dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT);
spin_lock_irqsave(&dom->pgtlock, flags);
memset(pgt_base_iova, 0, page_num * sizeof(u32));
memset(pgt_base_iova, 0, pgcount * sizeof(u32));
spin_unlock_irqrestore(&dom->pgtlock, flags);
return 0;
return size;
}
static void sprd_iommu_sync_map(struct iommu_domain *domain,
@ -409,13 +410,13 @@ static const struct iommu_ops sprd_iommu_ops = {
.probe_device = sprd_iommu_probe_device,
.device_group = sprd_iommu_device_group,
.of_xlate = sprd_iommu_of_xlate,
.pgsize_bitmap = ~0UL << SPRD_IOMMU_PAGE_SHIFT,
.pgsize_bitmap = SPRD_IOMMU_PAGE_SIZE,
.owner = THIS_MODULE,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = sprd_iommu_attach_device,
.detach_dev = sprd_iommu_detach_device,
.map = sprd_iommu_map,
.unmap = sprd_iommu_unmap,
.map_pages = sprd_iommu_map,
.unmap_pages = sprd_iommu_unmap,
.iotlb_sync_map = sprd_iommu_sync_map,
.iotlb_sync = sprd_iommu_sync,
.iova_to_phys = sprd_iommu_iova_to_phys,

89
drivers/iommu/sun50i-iommu.c
View File

@ -27,6 +27,7 @@
#include <linux/types.h>
#define IOMMU_RESET_REG 0x010
#define IOMMU_RESET_RELEASE_ALL 0xffffffff
#define IOMMU_ENABLE_REG 0x020
#define IOMMU_ENABLE_ENABLE BIT(0)
@ -92,6 +93,8 @@
#define NUM_PT_ENTRIES 256
#define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE)
#define SPAGE_SIZE 4096
struct sun50i_iommu {
struct iommu_device iommu;
@ -270,7 +273,7 @@ static u32 sun50i_mk_pte(phys_addr_t page, int prot)
enum sun50i_iommu_aci aci;
u32 flags = 0;
if (prot & (IOMMU_READ | IOMMU_WRITE))
if ((prot & (IOMMU_READ | IOMMU_WRITE)) == (IOMMU_READ | IOMMU_WRITE))
aci = SUN50I_IOMMU_ACI_RD_WR;
else if (prot & IOMMU_READ)
aci = SUN50I_IOMMU_ACI_RD;
@ -294,6 +297,62 @@ static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain,
dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);
}
static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu,
unsigned long iova)
{
u32 reg;
int ret;
iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova);
iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12));
iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG,
IOMMU_TLB_IVLD_ENABLE_ENABLE);
ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG,
reg, !reg, 1, 2000);
if (ret)
dev_warn(iommu->dev, "TLB invalidation timed out!\n");
}
static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu,
unsigned long iova)
{
u32 reg;
int ret;
iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova);
iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG,
IOMMU_PC_IVLD_ENABLE_ENABLE);
ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG,
reg, !reg, 1, 2000);
if (ret)
dev_warn(iommu->dev, "PTW cache invalidation timed out!\n");
}
static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu,
unsigned long iova, size_t size)
{
assert_spin_locked(&iommu->iommu_lock);
iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0);
sun50i_iommu_zap_iova(iommu, iova);
sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE);
if (size > SPAGE_SIZE) {
sun50i_iommu_zap_iova(iommu, iova + size);
sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE);
}
sun50i_iommu_zap_ptw_cache(iommu, iova);
sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M);
if (size > SZ_1M) {
sun50i_iommu_zap_ptw_cache(iommu, iova + size);
sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M);
}
iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
}
static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)
{
u32 reg;
@ -343,6 +402,18 @@ static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
spin_unlock_irqrestore(&iommu->iommu_lock, flags);
}
static void sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
struct sun50i_iommu *iommu = sun50i_domain->iommu;
unsigned long flags;
spin_lock_irqsave(&iommu->iommu_lock, flags);
sun50i_iommu_zap_range(iommu, iova, size);
spin_unlock_irqrestore(&iommu->iommu_lock, flags);
}
static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
@ -511,7 +582,7 @@ static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain,
sun50i_iommu_free_page_table(iommu, drop_pt);
}
sun50i_table_flush(sun50i_domain, page_table, PT_SIZE);
sun50i_table_flush(sun50i_domain, page_table, NUM_PT_ENTRIES);
sun50i_table_flush(sun50i_domain, dte_addr, 1);
return page_table;
@ -601,7 +672,6 @@ static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type)
struct sun50i_iommu_domain *sun50i_domain;
if (type != IOMMU_DOMAIN_DMA &&
type != IOMMU_DOMAIN_IDENTITY &&
type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
@ -766,6 +836,7 @@ static const struct iommu_ops sun50i_iommu_ops = {
.attach_dev = sun50i_iommu_attach_device,
.detach_dev = sun50i_iommu_detach_device,
.flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
.iotlb_sync_map = sun50i_iommu_iotlb_sync_map,
.iotlb_sync = sun50i_iommu_iotlb_sync,
.iova_to_phys = sun50i_iommu_iova_to_phys,
.map = sun50i_iommu_map,
@ -785,6 +856,8 @@ static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,
report_iommu_fault(iommu->domain, iommu->dev, iova, prot);
else
dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");
sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE);
}
static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,
@ -868,8 +941,8 @@ static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)
static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
{
u32 status, l1_status, l2_status, resets;
struct sun50i_iommu *iommu = dev_id;
u32 status;
spin_lock(&iommu->iommu_lock);
@ -879,6 +952,9 @@ static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
return IRQ_NONE;
}
l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG);
l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG);
if (status & IOMMU_INT_INVALID_L2PG)
sun50i_iommu_handle_pt_irq(iommu,
IOMMU_INT_ERR_ADDR_L2_REG,
@ -892,8 +968,9 @@ static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
iommu_write(iommu, IOMMU_INT_CLR_REG, status);
iommu_write(iommu, IOMMU_RESET_REG, ~status);
iommu_write(iommu, IOMMU_RESET_REG, status);
resets = (status | l1_status | l2_status) & IOMMU_INT_MASTER_MASK;
iommu_write(iommu, IOMMU_RESET_REG, ~resets);
iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL);
spin_unlock(&iommu->iommu_lock);

2
drivers/iommu/tegra-gart.c
View File

@ -112,7 +112,7 @@ static int gart_iommu_attach_dev(struct iommu_domain *domain,
spin_lock(&gart->dom_lock);
if (gart->active_domain && gart->active_domain != domain) {
ret = -EBUSY;
ret = -EINVAL;
} else if (dev_iommu_priv_get(dev) != domain) {
dev_iommu_priv_set(dev, domain);
gart->active_domain = domain;

7
drivers/iommu/virtio-iommu.c
View File

@ -670,7 +670,7 @@ static int viommu_domain_finalise(struct viommu_endpoint *vdev,
dev_err(vdev->dev,
"granule 0x%lx larger than system page size 0x%lx\n",
viommu_page_size, PAGE_SIZE);
return -EINVAL;
return -ENODEV;
}
ret = ida_alloc_range(&viommu->domain_ids, viommu->first_domain,
@ -697,7 +697,7 @@ static int viommu_domain_finalise(struct viommu_endpoint *vdev,
if (ret) {
ida_free(&viommu->domain_ids, vdomain->id);
vdomain->viommu = NULL;
return -EOPNOTSUPP;
return ret;
}
}
@ -734,8 +734,7 @@ static int viommu_attach_dev(struct iommu_domain *domain, struct device *dev)
*/
ret = viommu_domain_finalise(vdev, domain);
} else if (vdomain->viommu != vdev->viommu) {
dev_err(dev, "cannot attach to foreign vIOMMU\n");
ret = -EXDEV;
ret = -EINVAL;
}
mutex_unlock(&vdomain->mutex);

2
drivers/misc/uacce/uacce.c
View File

@ -108,7 +108,7 @@ static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)
if (!(uacce->flags & UACCE_DEV_SVA))
return 0;
handle = iommu_sva_bind_device(uacce->parent, current->mm, NULL);
handle = iommu_sva_bind_device(uacce->parent, current->mm);
if (IS_ERR(handle))
return PTR_ERR(handle);

3
drivers/pci/ats.c
View File

@ -382,6 +382,9 @@ int pci_enable_pasid(struct pci_dev *pdev, int features)
if (!pasid)
return -EINVAL;
if (!pci_acs_path_enabled(pdev, NULL, PCI_ACS_RR | PCI_ACS_UF))
return -EINVAL;
pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;

90
include/dt-bindings/memory/mediatek,mt8365-larb-port.h Normal file
View File

@ -0,0 +1,90 @@
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
/*
* Copyright (c) 2022 MediaTek Inc.
* Author: Yong Wu <yong.wu@mediatek.com>
*/
#ifndef _DT_BINDINGS_MEMORY_MT8365_LARB_PORT_H_
#define _DT_BINDINGS_MEMORY_MT8365_LARB_PORT_H_
#include <dt-bindings/memory/mtk-memory-port.h>
#define M4U_LARB0_ID 0
#define M4U_LARB1_ID 1
#define M4U_LARB2_ID 2
#define M4U_LARB3_ID 3
/* larb0 */
#define M4U_PORT_DISP_OVL0 MTK_M4U_ID(M4U_LARB0_ID, 0)
#define M4U_PORT_DISP_OVL0_2L MTK_M4U_ID(M4U_LARB0_ID, 1)
#define M4U_PORT_DISP_RDMA0 MTK_M4U_ID(M4U_LARB0_ID, 2)
#define M4U_PORT_DISP_WDMA0 MTK_M4U_ID(M4U_LARB0_ID, 3)
#define M4U_PORT_DISP_RDMA1 MTK_M4U_ID(M4U_LARB0_ID, 4)
#define M4U_PORT_MDP_RDMA0 MTK_M4U_ID(M4U_LARB0_ID, 5)
#define M4U_PORT_MDP_WROT1 MTK_M4U_ID(M4U_LARB0_ID, 6)
#define M4U_PORT_MDP_WROT0 MTK_M4U_ID(M4U_LARB0_ID, 7)
#define M4U_PORT_MDP_RDMA1 MTK_M4U_ID(M4U_LARB0_ID, 8)
#define M4U_PORT_DISP_FAKE0 MTK_M4U_ID(M4U_LARB0_ID, 9)
#define M4U_PORT_APU_READ MTK_M4U_ID(M4U_LARB0_ID, 10)
#define M4U_PORT_APU_WRITE MTK_M4U_ID(M4U_LARB0_ID, 11)
/* larb1 */
#define M4U_PORT_VENC_RCPU MTK_M4U_ID(M4U_LARB1_ID, 0)
#define M4U_PORT_VENC_REC MTK_M4U_ID(M4U_LARB1_ID, 1)
#define M4U_PORT_VENC_BSDMA MTK_M4U_ID(M4U_LARB1_ID, 2)
#define M4U_PORT_VENC_SV_COMV MTK_M4U_ID(M4U_LARB1_ID, 3)
#define M4U_PORT_VENC_RD_COMV MTK_M4U_ID(M4U_LARB1_ID, 4)
#define M4U_PORT_VENC_NBM_RDMA MTK_M4U_ID(M4U_LARB1_ID, 5)
#define M4U_PORT_VENC_NBM_RDMA_LITE MTK_M4U_ID(M4U_LARB1_ID, 6)
#define M4U_PORT_JPGENC_Y_RDMA MTK_M4U_ID(M4U_LARB1_ID, 7)
#define M4U_PORT_JPGENC_C_RDMA MTK_M4U_ID(M4U_LARB1_ID, 8)
#define M4U_PORT_JPGENC_Q_TABLE MTK_M4U_ID(M4U_LARB1_ID, 9)
#define M4U_PORT_JPGENC_BSDMA MTK_M4U_ID(M4U_LARB1_ID, 10)
#define M4U_PORT_JPGDEC_WDMA MTK_M4U_ID(M4U_LARB1_ID, 11)
#define M4U_PORT_JPGDEC_BSDMA MTK_M4U_ID(M4U_LARB1_ID, 12)
#define M4U_PORT_VENC_NBM_WDMA MTK_M4U_ID(M4U_LARB1_ID, 13)
#define M4U_PORT_VENC_NBM_WDMA_LITE MTK_M4U_ID(M4U_LARB1_ID, 14)
#define M4U_PORT_VENC_CUR_LUMA MTK_M4U_ID(M4U_LARB1_ID, 15)
#define M4U_PORT_VENC_CUR_CHROMA MTK_M4U_ID(M4U_LARB1_ID, 16)
#define M4U_PORT_VENC_REF_LUMA MTK_M4U_ID(M4U_LARB1_ID, 17)
#define M4U_PORT_VENC_REF_CHROMA MTK_M4U_ID(M4U_LARB1_ID, 18)
/* larb2 */
#define M4U_PORT_CAM_IMGO MTK_M4U_ID(M4U_LARB2_ID, 0)
#define M4U_PORT_CAM_RRZO MTK_M4U_ID(M4U_LARB2_ID, 1)
#define M4U_PORT_CAM_AAO MTK_M4U_ID(M4U_LARB2_ID, 2)
#define M4U_PORT_CAM_LCS MTK_M4U_ID(M4U_LARB2_ID, 3)
#define M4U_PORT_CAM_ESFKO MTK_M4U_ID(M4U_LARB2_ID, 4)
#define M4U_PORT_CAM_CAM_SV0 MTK_M4U_ID(M4U_LARB2_ID, 5)
#define M4U_PORT_CAM_CAM_SV1 MTK_M4U_ID(M4U_LARB2_ID, 6)
#define M4U_PORT_CAM_LSCI MTK_M4U_ID(M4U_LARB2_ID, 7)
#define M4U_PORT_CAM_LSCI_D MTK_M4U_ID(M4U_LARB2_ID, 8)
#define M4U_PORT_CAM_AFO MTK_M4U_ID(M4U_LARB2_ID, 9)
#define M4U_PORT_CAM_SPARE MTK_M4U_ID(M4U_LARB2_ID, 10)
#define M4U_PORT_CAM_BPCI MTK_M4U_ID(M4U_LARB2_ID, 11)
#define M4U_PORT_CAM_BPCI_D MTK_M4U_ID(M4U_LARB2_ID, 12)
#define M4U_PORT_CAM_UFDI MTK_M4U_ID(M4U_LARB2_ID, 13)
#define M4U_PORT_CAM_IMGI MTK_M4U_ID(M4U_LARB2_ID, 14)
#define M4U_PORT_CAM_IMG2O MTK_M4U_ID(M4U_LARB2_ID, 15)
#define M4U_PORT_CAM_IMG3O MTK_M4U_ID(M4U_LARB2_ID, 16)
#define M4U_PORT_CAM_WPE0_I MTK_M4U_ID(M4U_LARB2_ID, 17)
#define M4U_PORT_CAM_WPE1_I MTK_M4U_ID(M4U_LARB2_ID, 18)
#define M4U_PORT_CAM_WPE_O MTK_M4U_ID(M4U_LARB2_ID, 19)
#define M4U_PORT_CAM_FD0_I MTK_M4U_ID(M4U_LARB2_ID, 20)
#define M4U_PORT_CAM_FD1_I MTK_M4U_ID(M4U_LARB2_ID, 21)
#define M4U_PORT_CAM_FD0_O MTK_M4U_ID(M4U_LARB2_ID, 22)
#define M4U_PORT_CAM_FD1_O MTK_M4U_ID(M4U_LARB2_ID, 23)
/* larb3 */
#define M4U_PORT_HW_VDEC_MC_EXT MTK_M4U_ID(M4U_LARB3_ID, 0)
#define M4U_PORT_HW_VDEC_UFO_EXT MTK_M4U_ID(M4U_LARB3_ID, 1)
#define M4U_PORT_HW_VDEC_PP_EXT MTK_M4U_ID(M4U_LARB3_ID, 2)
#define M4U_PORT_HW_VDEC_PRED_RD_EXT MTK_M4U_ID(M4U_LARB3_ID, 3)
#define M4U_PORT_HW_VDEC_PRED_WR_EXT MTK_M4U_ID(M4U_LARB3_ID, 4)
#define M4U_PORT_HW_VDEC_PPWRAP_EXT MTK_M4U_ID(M4U_LARB3_ID, 5)
#define M4U_PORT_HW_VDEC_TILE_EXT MTK_M4U_ID(M4U_LARB3_ID, 6)
#define M4U_PORT_HW_VDEC_VLD_EXT MTK_M4U_ID(M4U_LARB3_ID, 7)
#define M4U_PORT_HW_VDEC_VLD2_EXT MTK_M4U_ID(M4U_LARB3_ID, 8)
#define M4U_PORT_HW_VDEC_AVC_MV_EXT MTK_M4U_ID(M4U_LARB3_ID, 9)
#define M4U_PORT_HW_VDEC_RG_CTRL_DMA_EXT MTK_M4U_ID(M4U_LARB3_ID, 10)
#endif

13
include/linux/intel-svm.h
View File

@ -13,17 +13,4 @@
#define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20)
#define PRQ_DEPTH ((0x1000 << PRQ_ORDER) >> 5)
/*
* The SVM_FLAG_SUPERVISOR_MODE flag requests a PASID which can be used only
* for access to kernel addresses. No IOTLB flushes are automatically done
* for kernel mappings; it is valid only for access to the kernel's static
* 1:1 mapping of physical memory — not to vmalloc or even module mappings.
* A future API addition may permit the use of such ranges, by means of an
* explicit IOTLB flush call (akin to the DMA API's unmap method).
*
* It is unlikely that we will ever hook into flush_tlb_kernel_range() to
* do such IOTLB flushes automatically.
*/
#define SVM_FLAG_SUPERVISOR_MODE BIT(0)
#endif /* __INTEL_SVM_H__ */

6
include/linux/io-pgtable.h
View File

@ -150,9 +150,7 @@ struct io_pgtable_cfg {
/**
* struct io_pgtable_ops - Page table manipulation API for IOMMU drivers.
*
* @map: Map a physically contiguous memory region.
* @map_pages: Map a physically contiguous range of pages of the same size.
* @unmap: Unmap a physically contiguous memory region.
* @unmap_pages: Unmap a range of virtually contiguous pages of the same size.
* @iova_to_phys: Translate iova to physical address.
*
@ -160,13 +158,9 @@ struct io_pgtable_cfg {
* the same names.
*/
struct io_pgtable_ops {
int (*map)(struct io_pgtable_ops *ops, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
int (*map_pages)(struct io_pgtable_ops *ops, unsigned long iova,
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped);
size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather);
size_t (*unmap_pages)(struct io_pgtable_ops *ops, unsigned long iova,
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather);

128
include/linux/iommu.h
View File

@ -64,6 +64,8 @@ struct iommu_domain_geometry {
#define __IOMMU_DOMAIN_PT (1U << 2) /* Domain is identity mapped */
#define __IOMMU_DOMAIN_DMA_FQ (1U << 3) /* DMA-API uses flush queue */
#define __IOMMU_DOMAIN_SVA (1U << 4) /* Shared process address space */
/*
* This are the possible domain-types
*
@ -77,6 +79,8 @@ struct iommu_domain_geometry {
* certain optimizations for these domains
* IOMMU_DOMAIN_DMA_FQ - As above, but definitely using batched TLB
* invalidation.
* IOMMU_DOMAIN_SVA - DMA addresses are shared process addresses
* represented by mm_struct's.
*/
#define IOMMU_DOMAIN_BLOCKED (0U)
#define IOMMU_DOMAIN_IDENTITY (__IOMMU_DOMAIN_PT)
@ -86,15 +90,27 @@ struct iommu_domain_geometry {
#define IOMMU_DOMAIN_DMA_FQ (__IOMMU_DOMAIN_PAGING | \
__IOMMU_DOMAIN_DMA_API | \
__IOMMU_DOMAIN_DMA_FQ)
#define IOMMU_DOMAIN_SVA (__IOMMU_DOMAIN_SVA)
struct iommu_domain {
unsigned type;
const struct iommu_domain_ops *ops;
unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */
iommu_fault_handler_t handler;
void *handler_token;
struct iommu_domain_geometry geometry;
struct iommu_dma_cookie *iova_cookie;
enum iommu_page_response_code (*iopf_handler)(struct iommu_fault *fault,
void *data);
void *fault_data;
union {
struct {
iommu_fault_handler_t handler;
void *handler_token;
};
struct { /* IOMMU_DOMAIN_SVA */
struct mm_struct *mm;
int users;
};
};
};
static inline bool iommu_is_dma_domain(struct iommu_domain *domain)
@ -214,15 +230,15 @@ struct iommu_iotlb_gather {
* driver init to device driver init (default no)
* @dev_enable/disable_feat: per device entries to enable/disable
* iommu specific features.
* @sva_bind: Bind process address space to device
* @sva_unbind: Unbind process address space from device
* @sva_get_pasid: Get PASID associated to a SVA handle
* @page_response: handle page request response
* @def_domain_type: device default domain type, return value:
* - IOMMU_DOMAIN_IDENTITY: must use an identity domain
* - IOMMU_DOMAIN_DMA: must use a dma domain
* - 0: use the default setting
* @default_domain_ops: the default ops for domains
* @remove_dev_pasid: Remove any translation configurations of a specific
* pasid, so that any DMA transactions with this pasid
* will be blocked by the hardware.
* @pgsize_bitmap: bitmap of all possible supported page sizes
* @owner: Driver module providing these ops
*/
@ -247,16 +263,12 @@ struct iommu_ops {
int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f);
int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f);
struct iommu_sva *(*sva_bind)(struct device *dev, struct mm_struct *mm,
void *drvdata);
void (*sva_unbind)(struct iommu_sva *handle);
u32 (*sva_get_pasid)(struct iommu_sva *handle);
int (*page_response)(struct device *dev,
struct iommu_fault_event *evt,
struct iommu_page_response *msg);
int (*def_domain_type)(struct device *dev);
void (*remove_dev_pasid)(struct device *dev, ioasid_t pasid);
const struct iommu_domain_ops *default_domain_ops;
unsigned long pgsize_bitmap;
@ -266,7 +278,20 @@ struct iommu_ops {
/**
* struct iommu_domain_ops - domain specific operations
* @attach_dev: attach an iommu domain to a device
* Return:
* * 0 - success
* * EINVAL - can indicate that device and domain are incompatible due to
* some previous configuration of the domain, in which case the
* driver shouldn't log an error, since it is legitimate for a
* caller to test reuse of existing domains. Otherwise, it may
* still represent some other fundamental problem
* * ENOMEM - out of memory
* * ENOSPC - non-ENOMEM type of resource allocation failures
* * EBUSY - device is attached to a domain and cannot be changed
* * ENODEV - device specific errors, not able to be attached
* * <others> - treated as ENODEV by the caller. Use is discouraged
* @detach_dev: detach an iommu domain from a device
* @set_dev_pasid: set an iommu domain to a pasid of device
* @map: map a physically contiguous memory region to an iommu domain
* @map_pages: map a physically contiguous set of pages of the same size to
* an iommu domain.
@ -287,6 +312,8 @@ struct iommu_ops {
struct iommu_domain_ops {
int (*attach_dev)(struct iommu_domain *domain, struct device *dev);
void (*detach_dev)(struct iommu_domain *domain, struct device *dev);
int (*set_dev_pasid)(struct iommu_domain *domain, struct device *dev,
ioasid_t pasid);
int (*map)(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
@ -322,12 +349,14 @@ struct iommu_domain_ops {
* @list: Used by the iommu-core to keep a list of registered iommus
* @ops: iommu-ops for talking to this iommu
* @dev: struct device for sysfs handling
* @max_pasids: number of supported PASIDs
*/
struct iommu_device {
struct list_head list;
const struct iommu_ops *ops;
struct fwnode_handle *fwnode;
struct device *dev;
u32 max_pasids;
};
/**
@ -366,6 +395,7 @@ struct iommu_fault_param {
* @fwspec: IOMMU fwspec data
* @iommu_dev: IOMMU device this device is linked to
* @priv: IOMMU Driver private data
* @max_pasids: number of PASIDs this device can consume
*
* TODO: migrate other per device data pointers under iommu_dev_data, e.g.
* struct iommu_group *iommu_group;
@ -377,6 +407,7 @@ struct dev_iommu {
struct iommu_fwspec *fwspec;
struct iommu_device *iommu_dev;
void *priv;
u32 max_pasids;
};
int iommu_device_register(struct iommu_device *iommu,
@ -622,6 +653,7 @@ struct iommu_fwspec {
*/
struct iommu_sva {
struct device *dev;
struct iommu_domain *domain;
};
int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
@ -663,12 +695,6 @@ void iommu_release_device(struct device *dev);
int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f);
int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f);
struct iommu_sva *iommu_sva_bind_device(struct device *dev,
struct mm_struct *mm,
void *drvdata);
void iommu_sva_unbind_device(struct iommu_sva *handle);
u32 iommu_sva_get_pasid(struct iommu_sva *handle);
int iommu_device_use_default_domain(struct device *dev);
void iommu_device_unuse_default_domain(struct device *dev);
@ -676,6 +702,15 @@ int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner);
void iommu_group_release_dma_owner(struct iommu_group *group);
bool iommu_group_dma_owner_claimed(struct iommu_group *group);
struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
struct mm_struct *mm);
int iommu_attach_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid);
void iommu_detach_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid);
struct iommu_domain *
iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
unsigned int type);
#else /* CONFIG_IOMMU_API */
struct iommu_ops {};
@ -995,21 +1030,6 @@ iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
return -ENODEV;
}
static inline struct iommu_sva *
iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)
{
return NULL;
}
static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
{
}
static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle)
{
return IOMMU_PASID_INVALID;
}
static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev)
{
return NULL;
@ -1038,6 +1058,30 @@ static inline bool iommu_group_dma_owner_claimed(struct iommu_group *group)
{
return false;
}
static inline struct iommu_domain *
iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm)
{
return NULL;
}
static inline int iommu_attach_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid)
{
return -ENODEV;
}
static inline void iommu_detach_device_pasid(struct iommu_domain *domain,
struct device *dev, ioasid_t pasid)
{
}
static inline struct iommu_domain *
iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
unsigned int type)
{
return NULL;
}
#endif /* CONFIG_IOMMU_API */
/**
@ -1099,4 +1143,26 @@ static inline void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_m
#endif /* CONFIG_IOMMU_DMA */
#ifdef CONFIG_IOMMU_SVA
struct iommu_sva *iommu_sva_bind_device(struct device *dev,
struct mm_struct *mm);
void iommu_sva_unbind_device(struct iommu_sva *handle);
u32 iommu_sva_get_pasid(struct iommu_sva *handle);
#else
static inline struct iommu_sva *
iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
{
return NULL;
}
static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
{
}
static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle)
{
return IOMMU_PASID_INVALID;
}
#endif /* CONFIG_IOMMU_SVA */
#endif /* __LINUX_IOMMU_H */