From f5c262b544975e067ea265fc7403aefbbea8563e Mon Sep 17 00:00:00 2001 From: Fangyu Yu Date: Thu, 22 Jan 2026 22:32:24 +0800 Subject: [PATCH 01/52] iommu/riscv: Add IOTINVAL after updating DDT/PDT entries Add riscv_iommu_iodir_iotinval() to perform required TLB and context cache invalidations after updating DDT or PDT entries, as mandated by the RISC-V IOMMU specification (Section 6.3.1 and 6.3.2). Fixes: 488ffbf18171 ("iommu/riscv: Paging domain support") Signed-off-by: Fangyu Yu Reviewed-by: Andrew Jones Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu.c | 70 +++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index fa2ebfd2f912..aadfbc181138 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -996,7 +996,67 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, } #define RISCV_IOMMU_FSC_BARE 0 +/* + * This function sends IOTINVAL commands as required by the RISC-V + * IOMMU specification (Section 6.3.1 and 6.3.2 in 1.0 spec version) + * after modifying DDT or PDT entries + */ +static void riscv_iommu_iodir_iotinval(struct riscv_iommu_device *iommu, + bool inval_pdt, unsigned long iohgatp, + struct riscv_iommu_dc *dc, + struct riscv_iommu_pc *pc) +{ + struct riscv_iommu_command cmd; + riscv_iommu_cmd_inval_vma(&cmd); + + if (FIELD_GET(RISCV_IOMMU_DC_IOHGATP_MODE, iohgatp) == + RISCV_IOMMU_DC_IOHGATP_MODE_BARE) { + if (inval_pdt) { + /* + * IOTINVAL.VMA with GV=AV=0, and PSCV=1, and + * PSCID=PC.PSCID + */ + riscv_iommu_cmd_inval_set_pscid(&cmd, + FIELD_GET(RISCV_IOMMU_PC_TA_PSCID, pc->ta)); + } else { + if (!FIELD_GET(RISCV_IOMMU_DC_TC_PDTV, dc->tc) && + FIELD_GET(RISCV_IOMMU_DC_FSC_MODE, dc->fsc) != + RISCV_IOMMU_DC_FSC_MODE_BARE) { + /* + * DC.tc.PDTV == 0 && DC.fsc.MODE != Bare + * IOTINVAL.VMA with GV=AV=0, and PSCV=1, and + * PSCID=DC.ta.PSCID + */ + riscv_iommu_cmd_inval_set_pscid(&cmd, + FIELD_GET(RISCV_IOMMU_DC_TA_PSCID, dc->ta)); + } + /* else: IOTINVAL.VMA with GV=AV=PSCV=0 */ + } + } else { + riscv_iommu_cmd_inval_set_gscid(&cmd, + FIELD_GET(RISCV_IOMMU_DC_IOHGATP_GSCID, iohgatp)); + + if (inval_pdt) { + /* + * IOTINVAL.VMA with GV=1, AV=0, and PSCV=1, and + * GSCID=DC.iohgatp.GSCID, PSCID=PC.PSCID + */ + riscv_iommu_cmd_inval_set_pscid(&cmd, + FIELD_GET(RISCV_IOMMU_PC_TA_PSCID, pc->ta)); + } + /* + * else: IOTINVAL.VMA with GV=1,AV=PSCV=0,and + * GSCID=DC.iohgatp.GSCID + * + * IOTINVAL.GVMA with GV=1,AV=0,and + * GSCID=DC.iohgatp.GSCID + * TODO: For now, the Second-Stage feature have not yet been merged, + * also issue IOTINVAL.GVMA once second-stage support is merged. + */ + } + riscv_iommu_cmd_send(iommu, &cmd); +} /* * Update IODIR for the device. * @@ -1031,6 +1091,11 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu, riscv_iommu_cmd_iodir_inval_ddt(&cmd); riscv_iommu_cmd_iodir_set_did(&cmd, fwspec->ids[i]); riscv_iommu_cmd_send(iommu, &cmd); + /* + * For now, the SVA and PASID features have not yet been merged, the + * default configuration is inval_pdt=false and pc=NULL. + */ + riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, dc, NULL); sync_required = true; } @@ -1056,6 +1121,11 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu, riscv_iommu_cmd_iodir_inval_ddt(&cmd); riscv_iommu_cmd_iodir_set_did(&cmd, fwspec->ids[i]); riscv_iommu_cmd_send(iommu, &cmd); + /* + * For now, the SVA and PASID features have not yet been merged, the + * default configuration is inval_pdt=false and pc=NULL. + */ + riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, dc, NULL); } riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT); From 7217cee35aadbb07e12673bcf1dcf729e1b2f6c9 Mon Sep 17 00:00:00 2001 From: Yaxing Guo Date: Fri, 30 Jan 2026 14:54:20 +0800 Subject: [PATCH 02/52] iommu/riscv: Skip IRQ count check when using MSI interrupts In RISC-V IOMMU platform devices that use MSI interrupts (indicated by the presence of 'msi-parent' in the device tree), there are no wired interrupt lines, so calling platform_get_irq_count() returns 0 or -ENXIO, causing the driver to fail during probe. However, MSI interrupts are allocated dynamically via the MSI subsystem and do not appear in the device tree 'interrupts' property. Therefore, the driver should not require a non-zero IRQ count when 'msi-parent' is present. This patch fixes the bug where probe fails when using MSI interrupts (which do not have an 'interrupts' property in the device tree).. Fixes: ("iommu/riscv: Add support for platform msi") Signed-off-by: Yaxing Guo Reviewed-by: Andrew Jones Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu-platform.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/riscv/iommu-platform.c b/drivers/iommu/riscv/iommu-platform.c index 83a28c83f991..8f15b06e8499 100644 --- a/drivers/iommu/riscv/iommu-platform.c +++ b/drivers/iommu/riscv/iommu-platform.c @@ -68,12 +68,7 @@ static int riscv_iommu_platform_probe(struct platform_device *pdev) iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES); iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL); - iommu->irqs_count = platform_irq_count(pdev); - if (iommu->irqs_count <= 0) - return dev_err_probe(dev, -ENODEV, - "no IRQ resources provided\n"); - if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT) - iommu->irqs_count = RISCV_IOMMU_INTR_COUNT; + iommu->irqs_count = RISCV_IOMMU_INTR_COUNT; igs = FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps); switch (igs) { @@ -120,6 +115,13 @@ static int riscv_iommu_platform_probe(struct platform_device *pdev) fallthrough; case RISCV_IOMMU_CAPABILITIES_IGS_WSI: + iommu->irqs_count = platform_irq_count(pdev); + if (iommu->irqs_count <= 0) + return dev_err_probe(dev, -ENODEV, + "no IRQ resources provided\n"); + if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT) + iommu->irqs_count = RISCV_IOMMU_INTR_COUNT; + for (vec = 0; vec < iommu->irqs_count; vec++) iommu->irqs[vec] = platform_get_irq(pdev, vec); From e71e00127110dedc6a9e746178282b4dac97ed96 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Feb 2026 11:25:36 -0400 Subject: [PATCH 03/52] iommupt: Add the RISC-V page table format The RISC-V format is a fairly simple 5 level page table not unlike the x86 one. It has optional support for a single contiguous page size of 64k (16 x 4k). The specification describes a 32-bit format, the general code can support it via a #define but the iommu side implementation has been left off until a user comes. Tested-by: Vincent Chen Acked-by: Paul Walmsley # arch/riscv Reviewed-by: Tomasz Jeznach Tested-by: Tomasz Jeznach Signed-off-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/.kunitconfig | 1 + drivers/iommu/generic_pt/Kconfig | 11 + drivers/iommu/generic_pt/fmt/Makefile | 2 + drivers/iommu/generic_pt/fmt/defs_riscv.h | 29 ++ drivers/iommu/generic_pt/fmt/iommu_riscv64.c | 11 + drivers/iommu/generic_pt/fmt/riscv.h | 313 +++++++++++++++++++ include/linux/generic_pt/common.h | 16 + include/linux/generic_pt/iommu.h | 11 + 8 files changed, 394 insertions(+) create mode 100644 drivers/iommu/generic_pt/fmt/defs_riscv.h create mode 100644 drivers/iommu/generic_pt/fmt/iommu_riscv64.c create mode 100644 drivers/iommu/generic_pt/fmt/riscv.h diff --git a/drivers/iommu/generic_pt/.kunitconfig b/drivers/iommu/generic_pt/.kunitconfig index a78b295f264d..0bb98fe581fe 100644 --- a/drivers/iommu/generic_pt/.kunitconfig +++ b/drivers/iommu/generic_pt/.kunitconfig @@ -5,6 +5,7 @@ CONFIG_DEBUG_GENERIC_PT=y CONFIG_IOMMU_PT=y CONFIG_IOMMU_PT_AMDV1=y CONFIG_IOMMU_PT_VTDSS=y +CONFIG_IOMMU_PT_RISCV64=y CONFIG_IOMMU_PT_X86_64=y CONFIG_IOMMU_PT_KUNIT_TEST=y diff --git a/drivers/iommu/generic_pt/Kconfig b/drivers/iommu/generic_pt/Kconfig index ce4fb4786914..f4ed1add58b7 100644 --- a/drivers/iommu/generic_pt/Kconfig +++ b/drivers/iommu/generic_pt/Kconfig @@ -52,6 +52,16 @@ config IOMMU_PT_VTDSS Selected automatically by an IOMMU driver that uses this format. +config IOMMU_PT_RISCV64 + tristate "IOMMU page table for RISC-V 64 bit Sv57/Sv48/Sv39" + depends on !GENERIC_ATOMIC64 # for cmpxchg64 + help + iommu_domain implementation for RISC-V 64 bit 3/4/5 level page table. + It supports 4K/2M/1G/512G/256T page sizes and can decode a sign + extended portion of the 64 bit IOVA space. + + Selected automatically by an IOMMU driver that uses this format. + config IOMMU_PT_X86_64 tristate "IOMMU page table for x86 64-bit, 4/5 levels" depends on !GENERIC_ATOMIC64 # for cmpxchg64 @@ -66,6 +76,7 @@ config IOMMU_PT_KUNIT_TEST tristate "IOMMU Page Table KUnit Test" if !KUNIT_ALL_TESTS depends on KUNIT depends on IOMMU_PT_AMDV1 || !IOMMU_PT_AMDV1 + depends on IOMMU_PT_RISCV64 || !IOMMU_PT_RISCV64 depends on IOMMU_PT_X86_64 || !IOMMU_PT_X86_64 depends on IOMMU_PT_VTDSS || !IOMMU_PT_VTDSS default KUNIT_ALL_TESTS diff --git a/drivers/iommu/generic_pt/fmt/Makefile b/drivers/iommu/generic_pt/fmt/Makefile index 976b49ec97dc..ea024d582594 100644 --- a/drivers/iommu/generic_pt/fmt/Makefile +++ b/drivers/iommu/generic_pt/fmt/Makefile @@ -5,6 +5,8 @@ iommu_pt_fmt-$(CONFIG_IOMMUFD_TEST) += mock iommu_pt_fmt-$(CONFIG_IOMMU_PT_VTDSS) += vtdss +iommu_pt_fmt-$(CONFIG_IOMMU_PT_RISCV64) += riscv64 + iommu_pt_fmt-$(CONFIG_IOMMU_PT_X86_64) += x86_64 IOMMU_PT_KUNIT_TEST := diff --git a/drivers/iommu/generic_pt/fmt/defs_riscv.h b/drivers/iommu/generic_pt/fmt/defs_riscv.h new file mode 100644 index 000000000000..cf67474d5eba --- /dev/null +++ b/drivers/iommu/generic_pt/fmt/defs_riscv.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES + * + */ +#ifndef __GENERIC_PT_FMT_DEFS_RISCV_H +#define __GENERIC_PT_FMT_DEFS_RISCV_H + +#include +#include + +#ifdef PT_RISCV_32BIT +typedef u32 pt_riscv_entry_t; +#define riscvpt_write_attrs riscv32pt_write_attrs +#else +typedef u64 pt_riscv_entry_t; +#define riscvpt_write_attrs riscv64pt_write_attrs +#endif + +typedef pt_riscv_entry_t pt_vaddr_t; +typedef u64 pt_oaddr_t; + +struct riscvpt_write_attrs { + pt_riscv_entry_t descriptor_bits; + gfp_t gfp; +}; +#define pt_write_attrs riscvpt_write_attrs + +#endif diff --git a/drivers/iommu/generic_pt/fmt/iommu_riscv64.c b/drivers/iommu/generic_pt/fmt/iommu_riscv64.c new file mode 100644 index 000000000000..cbf60fffa9bf --- /dev/null +++ b/drivers/iommu/generic_pt/fmt/iommu_riscv64.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES + */ +#define PT_FMT riscv +#define PT_FMT_VARIANT 64 +#define PT_SUPPORTED_FEATURES \ + (BIT(PT_FEAT_SIGN_EXTEND) | BIT(PT_FEAT_FLUSH_RANGE) | \ + BIT(PT_FEAT_RISCV_SVNAPOT_64K)) + +#include "iommu_template.h" diff --git a/drivers/iommu/generic_pt/fmt/riscv.h b/drivers/iommu/generic_pt/fmt/riscv.h new file mode 100644 index 000000000000..a7fef6266a36 --- /dev/null +++ b/drivers/iommu/generic_pt/fmt/riscv.h @@ -0,0 +1,313 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES + * + * RISC-V page table + * + * This is described in Sections: + * 12.3. Sv32: Page-Based 32-bit Virtual-Memory Systems + * 12.4. Sv39: Page-Based 39-bit Virtual-Memory System + * 12.5. Sv48: Page-Based 48-bit Virtual-Memory System + * 12.6. Sv57: Page-Based 57-bit Virtual-Memory System + * of the "The RISC-V Instruction Set Manual: Volume II" + * + * This includes the contiguous page extension from: + * Chapter 13. "Svnapot" Extension for NAPOT Translation Contiguity, + * Version 1.0 + * + * The table format is sign extended and supports leafs in every level. The spec + * doesn't talk a lot about levels, but level here is the same as i=LEVELS-1 in + * the spec. + */ +#ifndef __GENERIC_PT_FMT_RISCV_H +#define __GENERIC_PT_FMT_RISCV_H + +#include "defs_riscv.h" +#include "../pt_defs.h" + +#include +#include +#include +#include + +enum { + PT_ITEM_WORD_SIZE = sizeof(pt_riscv_entry_t), +#ifdef PT_RISCV_32BIT + PT_MAX_VA_ADDRESS_LG2 = 32, + PT_MAX_OUTPUT_ADDRESS_LG2 = 34, + PT_MAX_TOP_LEVEL = 1, +#else + PT_MAX_VA_ADDRESS_LG2 = 57, + PT_MAX_OUTPUT_ADDRESS_LG2 = 56, + PT_MAX_TOP_LEVEL = 4, +#endif + PT_GRANULE_LG2SZ = 12, + PT_TABLEMEM_LG2SZ = 12, + + /* fsc.PPN is 44 bits wide, all PPNs are 4k aligned */ + PT_TOP_PHYS_MASK = GENMASK_ULL(55, 12), +}; + +/* PTE bits */ +enum { + RISCVPT_V = BIT(0), + RISCVPT_R = BIT(1), + RISCVPT_W = BIT(2), + RISCVPT_X = BIT(3), + RISCVPT_U = BIT(4), + RISCVPT_G = BIT(5), + RISCVPT_A = BIT(6), + RISCVPT_D = BIT(7), + RISCVPT_RSW = GENMASK(9, 8), + RISCVPT_PPN32 = GENMASK(31, 10), + + RISCVPT_PPN64 = GENMASK_ULL(53, 10), + RISCVPT_PPN64_64K = GENMASK_ULL(53, 14), + RISCVPT_PBMT = GENMASK_ULL(62, 61), + RISCVPT_N = BIT_ULL(63), + + /* Svnapot encodings for ppn[0] */ + RISCVPT_PPN64_64K_SZ = BIT(13), +}; + +#ifdef PT_RISCV_32BIT +#define RISCVPT_PPN RISCVPT_PPN32 +#define pt_riscv pt_riscv_32 +#else +#define RISCVPT_PPN RISCVPT_PPN64 +#define pt_riscv pt_riscv_64 +#endif + +#define common_to_riscvpt(common_ptr) \ + container_of_const(common_ptr, struct pt_riscv, common) +#define to_riscvpt(pts) common_to_riscvpt((pts)->range->common) + +static inline pt_oaddr_t riscvpt_table_pa(const struct pt_state *pts) +{ + return oalog2_mul(FIELD_GET(RISCVPT_PPN, pts->entry), PT_GRANULE_LG2SZ); +} +#define pt_table_pa riscvpt_table_pa + +static inline pt_oaddr_t riscvpt_entry_oa(const struct pt_state *pts) +{ + if (pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K) && + pts->entry & RISCVPT_N) { + PT_WARN_ON(pts->level != 0); + return oalog2_mul(FIELD_GET(RISCVPT_PPN64_64K, pts->entry), + ilog2(SZ_64K)); + } + return oalog2_mul(FIELD_GET(RISCVPT_PPN, pts->entry), PT_GRANULE_LG2SZ); +} +#define pt_entry_oa riscvpt_entry_oa + +static inline bool riscvpt_can_have_leaf(const struct pt_state *pts) +{ + return true; +} +#define pt_can_have_leaf riscvpt_can_have_leaf + +/* Body in pt_fmt_defaults.h */ +static inline unsigned int pt_table_item_lg2sz(const struct pt_state *pts); + +static inline unsigned int +riscvpt_entry_num_contig_lg2(const struct pt_state *pts) +{ + if (PT_SUPPORTED_FEATURE(PT_FEAT_RISCV_SVNAPOT_64K) && + pts->entry & RISCVPT_N) { + PT_WARN_ON(!pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K)); + PT_WARN_ON(pts->level); + return ilog2(16); + } + return ilog2(1); +} +#define pt_entry_num_contig_lg2 riscvpt_entry_num_contig_lg2 + +static inline unsigned int riscvpt_num_items_lg2(const struct pt_state *pts) +{ + return PT_TABLEMEM_LG2SZ - ilog2(sizeof(u64)); +} +#define pt_num_items_lg2 riscvpt_num_items_lg2 + +static inline unsigned short +riscvpt_contig_count_lg2(const struct pt_state *pts) +{ + if (pts->level == 0 && pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K)) + return ilog2(16); + return ilog2(1); +} +#define pt_contig_count_lg2 riscvpt_contig_count_lg2 + +static inline enum pt_entry_type riscvpt_load_entry_raw(struct pt_state *pts) +{ + const pt_riscv_entry_t *tablep = pt_cur_table(pts, pt_riscv_entry_t); + pt_riscv_entry_t entry; + + pts->entry = entry = READ_ONCE(tablep[pts->index]); + if (!(entry & RISCVPT_V)) + return PT_ENTRY_EMPTY; + if (pts->level == 0 || + ((entry & (RISCVPT_X | RISCVPT_W | RISCVPT_R)) != 0)) + return PT_ENTRY_OA; + return PT_ENTRY_TABLE; +} +#define pt_load_entry_raw riscvpt_load_entry_raw + +static inline void +riscvpt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa, + unsigned int oasz_lg2, + const struct pt_write_attrs *attrs) +{ + pt_riscv_entry_t *tablep = pt_cur_table(pts, pt_riscv_entry_t); + pt_riscv_entry_t entry; + + if (!pt_check_install_leaf_args(pts, oa, oasz_lg2)) + return; + + entry = RISCVPT_V | + FIELD_PREP(RISCVPT_PPN, log2_div(oa, PT_GRANULE_LG2SZ)) | + attrs->descriptor_bits; + + if (pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K) && pts->level == 0 && + oasz_lg2 != PT_GRANULE_LG2SZ) { + u64 *end; + + entry |= RISCVPT_N | RISCVPT_PPN64_64K_SZ; + tablep += pts->index; + end = tablep + log2_div(SZ_64K, PT_GRANULE_LG2SZ); + for (; tablep != end; tablep++) + WRITE_ONCE(*tablep, entry); + } else { + /* FIXME does riscv need this to be cmpxchg? */ + WRITE_ONCE(tablep[pts->index], entry); + } + pts->entry = entry; +} +#define pt_install_leaf_entry riscvpt_install_leaf_entry + +static inline bool riscvpt_install_table(struct pt_state *pts, + pt_oaddr_t table_pa, + const struct pt_write_attrs *attrs) +{ + pt_riscv_entry_t entry; + + entry = RISCVPT_V | + FIELD_PREP(RISCVPT_PPN, log2_div(table_pa, PT_GRANULE_LG2SZ)); + return pt_table_install64(pts, entry); +} +#define pt_install_table riscvpt_install_table + +static inline void riscvpt_attr_from_entry(const struct pt_state *pts, + struct pt_write_attrs *attrs) +{ + attrs->descriptor_bits = + pts->entry & (RISCVPT_R | RISCVPT_W | RISCVPT_X | RISCVPT_U | + RISCVPT_G | RISCVPT_A | RISCVPT_D); +} +#define pt_attr_from_entry riscvpt_attr_from_entry + +/* --- iommu */ +#include +#include + +#define pt_iommu_table pt_iommu_riscv_64 + +/* The common struct is in the per-format common struct */ +static inline struct pt_common *common_from_iommu(struct pt_iommu *iommu_table) +{ + return &container_of(iommu_table, struct pt_iommu_table, iommu) + ->riscv_64pt.common; +} + +static inline struct pt_iommu *iommu_from_common(struct pt_common *common) +{ + return &container_of(common, struct pt_iommu_table, riscv_64pt.common) + ->iommu; +} + +static inline int riscvpt_iommu_set_prot(struct pt_common *common, + struct pt_write_attrs *attrs, + unsigned int iommu_prot) +{ + u64 pte; + + pte = RISCVPT_A | RISCVPT_U; + if (iommu_prot & IOMMU_WRITE) + pte |= RISCVPT_W | RISCVPT_R | RISCVPT_D; + if (iommu_prot & IOMMU_READ) + pte |= RISCVPT_R; + if (!(iommu_prot & IOMMU_NOEXEC)) + pte |= RISCVPT_X; + + /* Caller must specify a supported combination of flags */ + if (unlikely((pte & (RISCVPT_X | RISCVPT_W | RISCVPT_R)) == 0)) + return -EOPNOTSUPP; + + attrs->descriptor_bits = pte; + return 0; +} +#define pt_iommu_set_prot riscvpt_iommu_set_prot + +static inline int +riscvpt_iommu_fmt_init(struct pt_iommu_riscv_64 *iommu_table, + const struct pt_iommu_riscv_64_cfg *cfg) +{ + struct pt_riscv *table = &iommu_table->riscv_64pt; + + switch (cfg->common.hw_max_vasz_lg2) { + case 39: + pt_top_set_level(&table->common, 2); + break; + case 48: + pt_top_set_level(&table->common, 3); + break; + case 57: + pt_top_set_level(&table->common, 4); + break; + default: + return -EINVAL; + } + table->common.max_oasz_lg2 = + min(PT_MAX_OUTPUT_ADDRESS_LG2, cfg->common.hw_max_oasz_lg2); + return 0; +} +#define pt_iommu_fmt_init riscvpt_iommu_fmt_init + +static inline void +riscvpt_iommu_fmt_hw_info(struct pt_iommu_riscv_64 *table, + const struct pt_range *top_range, + struct pt_iommu_riscv_64_hw_info *info) +{ + phys_addr_t top_phys = virt_to_phys(top_range->top_table); + + info->ppn = oalog2_div(top_phys, PT_GRANULE_LG2SZ); + PT_WARN_ON(top_phys & ~PT_TOP_PHYS_MASK); + + /* + * See Table 3. Encodings of iosatp.MODE field" for DC.tx.SXL = 0: + * 8 = Sv39 = top level 2 + * 9 = Sv38 = top level 3 + * 10 = Sv57 = top level 4 + */ + info->fsc_iosatp_mode = top_range->top_level + 6; +} +#define pt_iommu_fmt_hw_info riscvpt_iommu_fmt_hw_info + +#if defined(GENERIC_PT_KUNIT) +static const struct pt_iommu_riscv_64_cfg riscv_64_kunit_fmt_cfgs[] = { + [0] = { .common.features = BIT(PT_FEAT_RISCV_SVNAPOT_64K), + .common.hw_max_oasz_lg2 = 56, + .common.hw_max_vasz_lg2 = 39 }, + [1] = { .common.features = 0, + .common.hw_max_oasz_lg2 = 56, + .common.hw_max_vasz_lg2 = 48 }, + [2] = { .common.features = BIT(PT_FEAT_RISCV_SVNAPOT_64K), + .common.hw_max_oasz_lg2 = 56, + .common.hw_max_vasz_lg2 = 57 }, +}; +#define kunit_fmt_cfgs riscv_64_kunit_fmt_cfgs +enum { + KUNIT_FMT_FEATURES = BIT(PT_FEAT_RISCV_SVNAPOT_64K), +}; +#endif + +#endif diff --git a/include/linux/generic_pt/common.h b/include/linux/generic_pt/common.h index 6a9a1acb5aad..fc5d0b5edadc 100644 --- a/include/linux/generic_pt/common.h +++ b/include/linux/generic_pt/common.h @@ -175,6 +175,22 @@ enum { PT_FEAT_VTDSS_FORCE_WRITEABLE, }; +struct pt_riscv_32 { + struct pt_common common; +}; + +struct pt_riscv_64 { + struct pt_common common; +}; + +enum { + /* + * Support the 64k contiguous page size following the Svnapot extension. + */ + PT_FEAT_RISCV_SVNAPOT_64K = PT_FEAT_FMT_START, + +}; + struct pt_x86_64 { struct pt_common common; }; diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h index 9eefbb74efd0..49d9addb98c5 100644 --- a/include/linux/generic_pt/iommu.h +++ b/include/linux/generic_pt/iommu.h @@ -275,6 +275,17 @@ struct pt_iommu_vtdss_hw_info { IOMMU_FORMAT(vtdss, vtdss_pt); +struct pt_iommu_riscv_64_cfg { + struct pt_iommu_cfg common; +}; + +struct pt_iommu_riscv_64_hw_info { + u64 ppn; + u8 fsc_iosatp_mode; +}; + +IOMMU_FORMAT(riscv_64, riscv_64pt); + struct pt_iommu_x86_64_cfg { struct pt_iommu_cfg common; /* 4 is a 57 bit 5 level table */ From e93e4a6363b8f812cb89f2b2d97cbaf017a3d7f8 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Feb 2026 11:25:37 -0400 Subject: [PATCH 04/52] iommu/riscv: Disable SADE In terms of the iommu subystem the SADE/GADE feature "3.4. IOMMU updating of PTE accessed (A) and dirty (D) updates" is called dirty tracking. There is no reason to enable HW support for this, and the HW cost associated with it, unless dirty tracking is actually enabled through iommufd. It should be a dynamic feature linked to user request. Further, without implementing the read dirty ops the whole thing is pointless. Do not set DC.tc.SADE just because the HW has support for dirty tracking. Tested-by: Vincent Chen Acked-by: Paul Walmsley # arch/riscv Reviewed-by: Tomasz Jeznach Tested-by: Tomasz Jeznach Signed-off-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index fa2ebfd2f912..0b2903372470 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -810,7 +810,6 @@ struct riscv_iommu_domain { struct list_head bonds; spinlock_t lock; /* protect bonds list updates. */ int pscid; - bool amo_enabled; int numa_node; unsigned int pgd_mode; unsigned long *pgd_root; @@ -1201,8 +1200,6 @@ static int riscv_iommu_map_pages(struct iommu_domain *iommu_domain, if (!(prot & IOMMU_WRITE)) pte_prot = _PAGE_BASE | _PAGE_READ; - else if (domain->amo_enabled) - pte_prot = _PAGE_BASE | _PAGE_READ | _PAGE_WRITE; else pte_prot = _PAGE_BASE | _PAGE_READ | _PAGE_WRITE | _PAGE_DIRTY; @@ -1387,7 +1384,6 @@ static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev) INIT_LIST_HEAD_RCU(&domain->bonds); spin_lock_init(&domain->lock); domain->numa_node = dev_to_node(iommu->dev); - domain->amo_enabled = !!(iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD); domain->pgd_mode = pgd_mode; domain->pgd_root = iommu_alloc_pages_node_sz(domain->numa_node, GFP_KERNEL_ACCOUNT, SZ_4K); @@ -1512,8 +1508,6 @@ static struct iommu_device *riscv_iommu_probe_device(struct device *dev) * the device directory. Do not mark the context valid yet. */ tc = 0; - if (iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD) - tc |= RISCV_IOMMU_DC_TC_SADE; for (i = 0; i < fwspec->num_ids; i++) { dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]); if (!dc) { From e5ef32191a87da48a0f6cab2ca5f7d8b4a0fa054 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Feb 2026 11:25:38 -0400 Subject: [PATCH 05/52] iommu/riscv: Use the generic iommu page table This is a fairly straightforward conversion of the RISC-V iommu driver to use the generic iommu page table code. Invalidation stays as it is now with the driver pretending to implement simple range based invalidation even though the HW is more like ARM SMMUv3 than AMD where the HW implements a single-PTE based invalidation. Future work to extend the generic invalidate mechanism to support more ARM-like semantics would benefit this driver as well. Delete the existing page table code. Tested-by: Vincent Chen Acked-by: Paul Walmsley # arch/riscv Reviewed-by: Tomasz Jeznach Tested-by: Tomasz Jeznach Signed-off-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/Kconfig | 3 + drivers/iommu/riscv/iommu.c | 287 +++++------------------------------- 2 files changed, 39 insertions(+), 251 deletions(-) diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig index c071816f59a6..a329ec634cf1 100644 --- a/drivers/iommu/riscv/Kconfig +++ b/drivers/iommu/riscv/Kconfig @@ -6,6 +6,9 @@ config RISCV_IOMMU depends on RISCV && 64BIT default y select IOMMU_API + select GENERIC_PT + select IOMMU_PT + select IOMMU_PT_RISCV64 help Support for implementations of the RISC-V IOMMU architecture that complements the RISC-V MMU capabilities, providing similar address diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index 0b2903372470..6ceca9bed917 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "../iommu-pages.h" #include "iommu-bits.h" @@ -806,14 +807,15 @@ static int riscv_iommu_iodir_set_mode(struct riscv_iommu_device *iommu, /* This struct contains protection domain specific IOMMU driver data. */ struct riscv_iommu_domain { - struct iommu_domain domain; + union { + struct iommu_domain domain; + struct pt_iommu_riscv_64 riscvpt; + }; struct list_head bonds; spinlock_t lock; /* protect bonds list updates. */ int pscid; - int numa_node; - unsigned int pgd_mode; - unsigned long *pgd_root; }; +PT_IOMMU_CHECK_DOMAIN(struct riscv_iommu_domain, riscvpt.iommu, domain); #define iommu_domain_to_riscv(iommu_domain) \ container_of(iommu_domain, struct riscv_iommu_domain, domain) @@ -1076,156 +1078,9 @@ static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain, { struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain); - riscv_iommu_iotlb_inval(domain, gather->start, gather->end); -} - -#define PT_SHIFT (PAGE_SHIFT - ilog2(sizeof(pte_t))) - -#define _io_pte_present(pte) ((pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) -#define _io_pte_leaf(pte) ((pte) & _PAGE_LEAF) -#define _io_pte_none(pte) ((pte) == 0) -#define _io_pte_entry(pn, prot) ((_PAGE_PFN_MASK & ((pn) << _PAGE_PFN_SHIFT)) | (prot)) - -static void riscv_iommu_pte_free(struct riscv_iommu_domain *domain, - unsigned long pte, - struct iommu_pages_list *freelist) -{ - unsigned long *ptr; - int i; - - if (!_io_pte_present(pte) || _io_pte_leaf(pte)) - return; - - ptr = (unsigned long *)pfn_to_virt(__page_val_to_pfn(pte)); - - /* Recursively free all sub page table pages */ - for (i = 0; i < PTRS_PER_PTE; i++) { - pte = READ_ONCE(ptr[i]); - if (!_io_pte_none(pte) && cmpxchg_relaxed(ptr + i, pte, 0) == pte) - riscv_iommu_pte_free(domain, pte, freelist); - } - - if (freelist) - iommu_pages_list_add(freelist, ptr); - else - iommu_free_pages(ptr); -} - -static unsigned long *riscv_iommu_pte_alloc(struct riscv_iommu_domain *domain, - unsigned long iova, size_t pgsize, - gfp_t gfp) -{ - unsigned long *ptr = domain->pgd_root; - unsigned long pte, old; - int level = domain->pgd_mode - RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39 + 2; - void *addr; - - do { - const int shift = PAGE_SHIFT + PT_SHIFT * level; - - ptr += ((iova >> shift) & (PTRS_PER_PTE - 1)); - /* - * Note: returned entry might be a non-leaf if there was - * existing mapping with smaller granularity. Up to the caller - * to replace and invalidate. - */ - if (((size_t)1 << shift) == pgsize) - return ptr; -pte_retry: - pte = READ_ONCE(*ptr); - /* - * This is very likely incorrect as we should not be adding - * new mapping with smaller granularity on top - * of existing 2M/1G mapping. Fail. - */ - if (_io_pte_present(pte) && _io_pte_leaf(pte)) - return NULL; - /* - * Non-leaf entry is missing, allocate and try to add to the - * page table. This might race with other mappings, retry. - */ - if (_io_pte_none(pte)) { - addr = iommu_alloc_pages_node_sz(domain->numa_node, gfp, - SZ_4K); - if (!addr) - return NULL; - old = pte; - pte = _io_pte_entry(virt_to_pfn(addr), _PAGE_TABLE); - if (cmpxchg_relaxed(ptr, old, pte) != old) { - iommu_free_pages(addr); - goto pte_retry; - } - } - ptr = (unsigned long *)pfn_to_virt(__page_val_to_pfn(pte)); - } while (level-- > 0); - - return NULL; -} - -static unsigned long *riscv_iommu_pte_fetch(struct riscv_iommu_domain *domain, - unsigned long iova, size_t *pte_pgsize) -{ - unsigned long *ptr = domain->pgd_root; - unsigned long pte; - int level = domain->pgd_mode - RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39 + 2; - - do { - const int shift = PAGE_SHIFT + PT_SHIFT * level; - - ptr += ((iova >> shift) & (PTRS_PER_PTE - 1)); - pte = READ_ONCE(*ptr); - if (_io_pte_present(pte) && _io_pte_leaf(pte)) { - *pte_pgsize = (size_t)1 << shift; - return ptr; - } - if (_io_pte_none(pte)) - return NULL; - ptr = (unsigned long *)pfn_to_virt(__page_val_to_pfn(pte)); - } while (level-- > 0); - - return NULL; -} - -static int riscv_iommu_map_pages(struct iommu_domain *iommu_domain, - unsigned long iova, phys_addr_t phys, - size_t pgsize, size_t pgcount, int prot, - gfp_t gfp, size_t *mapped) -{ - struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain); - size_t size = 0; - unsigned long *ptr; - unsigned long pte, old, pte_prot; - int rc = 0; - struct iommu_pages_list freelist = IOMMU_PAGES_LIST_INIT(freelist); - - if (!(prot & IOMMU_WRITE)) - pte_prot = _PAGE_BASE | _PAGE_READ; - else - pte_prot = _PAGE_BASE | _PAGE_READ | _PAGE_WRITE | _PAGE_DIRTY; - - while (pgcount) { - ptr = riscv_iommu_pte_alloc(domain, iova, pgsize, gfp); - if (!ptr) { - rc = -ENOMEM; - break; - } - - old = READ_ONCE(*ptr); - pte = _io_pte_entry(phys_to_pfn(phys), pte_prot); - if (cmpxchg_relaxed(ptr, old, pte) != old) - continue; - - riscv_iommu_pte_free(domain, old, &freelist); - - size += pgsize; - iova += pgsize; - phys += pgsize; - --pgcount; - } - - *mapped = size; - - if (!iommu_pages_list_empty(&freelist)) { + if (iommu_pages_list_empty(&gather->freelist)) { + riscv_iommu_iotlb_inval(domain, gather->start, gather->end); + } else { /* * In 1.0 spec version, the smallest scope we can use to * invalidate all levels of page table (i.e. leaf and non-leaf) @@ -1234,71 +1089,20 @@ static int riscv_iommu_map_pages(struct iommu_domain *iommu_domain, * capability.NL (non-leaf) IOTINVAL command. */ riscv_iommu_iotlb_inval(domain, 0, ULONG_MAX); - iommu_put_pages_list(&freelist); + iommu_put_pages_list(&gather->freelist); } - - return rc; -} - -static size_t riscv_iommu_unmap_pages(struct iommu_domain *iommu_domain, - unsigned long iova, size_t pgsize, - size_t pgcount, - struct iommu_iotlb_gather *gather) -{ - struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain); - size_t size = pgcount << __ffs(pgsize); - unsigned long *ptr, old; - size_t unmapped = 0; - size_t pte_size; - - while (unmapped < size) { - ptr = riscv_iommu_pte_fetch(domain, iova, &pte_size); - if (!ptr) - return unmapped; - - /* partial unmap is not allowed, fail. */ - if (iova & (pte_size - 1)) - return unmapped; - - old = READ_ONCE(*ptr); - if (cmpxchg_relaxed(ptr, old, 0) != old) - continue; - - iommu_iotlb_gather_add_page(&domain->domain, gather, iova, - pte_size); - - iova += pte_size; - unmapped += pte_size; - } - - return unmapped; -} - -static phys_addr_t riscv_iommu_iova_to_phys(struct iommu_domain *iommu_domain, - dma_addr_t iova) -{ - struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain); - size_t pte_size; - unsigned long *ptr; - - ptr = riscv_iommu_pte_fetch(domain, iova, &pte_size); - if (!ptr) - return 0; - - return pfn_to_phys(__page_val_to_pfn(*ptr)) | (iova & (pte_size - 1)); } static void riscv_iommu_free_paging_domain(struct iommu_domain *iommu_domain) { struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain); - const unsigned long pfn = virt_to_pfn(domain->pgd_root); WARN_ON(!list_empty(&domain->bonds)); if ((int)domain->pscid > 0) ida_free(&riscv_iommu_pscids, domain->pscid); - riscv_iommu_pte_free(domain, _io_pte_entry(pfn, _PAGE_TABLE), NULL); + pt_iommu_deinit(&domain->riscvpt.iommu); kfree(domain); } @@ -1324,13 +1128,16 @@ static int riscv_iommu_attach_paging_domain(struct iommu_domain *iommu_domain, struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain); struct riscv_iommu_device *iommu = dev_to_iommu(dev); struct riscv_iommu_info *info = dev_iommu_priv_get(dev); + struct pt_iommu_riscv_64_hw_info pt_info; u64 fsc, ta; - if (!riscv_iommu_pt_supported(iommu, domain->pgd_mode)) + pt_iommu_riscv_64_hw_info(&domain->riscvpt, &pt_info); + + if (!riscv_iommu_pt_supported(iommu, pt_info.fsc_iosatp_mode)) return -ENODEV; - fsc = FIELD_PREP(RISCV_IOMMU_PC_FSC_MODE, domain->pgd_mode) | - FIELD_PREP(RISCV_IOMMU_PC_FSC_PPN, virt_to_pfn(domain->pgd_root)); + fsc = FIELD_PREP(RISCV_IOMMU_PC_FSC_MODE, pt_info.fsc_iosatp_mode) | + FIELD_PREP(RISCV_IOMMU_PC_FSC_PPN, pt_info.ppn); ta = FIELD_PREP(RISCV_IOMMU_PC_TA_PSCID, domain->pscid) | RISCV_IOMMU_PC_TA_V; @@ -1345,37 +1152,32 @@ static int riscv_iommu_attach_paging_domain(struct iommu_domain *iommu_domain, } static const struct iommu_domain_ops riscv_iommu_paging_domain_ops = { + IOMMU_PT_DOMAIN_OPS(riscv_64), .attach_dev = riscv_iommu_attach_paging_domain, .free = riscv_iommu_free_paging_domain, - .map_pages = riscv_iommu_map_pages, - .unmap_pages = riscv_iommu_unmap_pages, - .iova_to_phys = riscv_iommu_iova_to_phys, .iotlb_sync = riscv_iommu_iotlb_sync, .flush_iotlb_all = riscv_iommu_iotlb_flush_all, }; static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev) { + struct pt_iommu_riscv_64_cfg cfg = {}; struct riscv_iommu_domain *domain; struct riscv_iommu_device *iommu; - unsigned int pgd_mode; - dma_addr_t va_mask; - int va_bits; + int ret; iommu = dev_to_iommu(dev); if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV57) { - pgd_mode = RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57; - va_bits = 57; + cfg.common.hw_max_vasz_lg2 = 57; } else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV48) { - pgd_mode = RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48; - va_bits = 48; + cfg.common.hw_max_vasz_lg2 = 48; } else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV39) { - pgd_mode = RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39; - va_bits = 39; + cfg.common.hw_max_vasz_lg2 = 39; } else { dev_err(dev, "cannot find supported page table mode\n"); return ERR_PTR(-ENODEV); } + cfg.common.hw_max_oasz_lg2 = 56; domain = kzalloc_obj(*domain); if (!domain) @@ -1383,42 +1185,23 @@ static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev) INIT_LIST_HEAD_RCU(&domain->bonds); spin_lock_init(&domain->lock); - domain->numa_node = dev_to_node(iommu->dev); - domain->pgd_mode = pgd_mode; - domain->pgd_root = iommu_alloc_pages_node_sz(domain->numa_node, - GFP_KERNEL_ACCOUNT, SZ_4K); - if (!domain->pgd_root) { - kfree(domain); - return ERR_PTR(-ENOMEM); - } + cfg.common.features = BIT(PT_FEAT_SIGN_EXTEND) | + BIT(PT_FEAT_FLUSH_RANGE); + domain->riscvpt.iommu.nid = dev_to_node(iommu->dev); + domain->domain.ops = &riscv_iommu_paging_domain_ops; domain->pscid = ida_alloc_range(&riscv_iommu_pscids, 1, RISCV_IOMMU_MAX_PSCID, GFP_KERNEL); if (domain->pscid < 0) { - iommu_free_pages(domain->pgd_root); - kfree(domain); + riscv_iommu_free_paging_domain(&domain->domain); return ERR_PTR(-ENOMEM); } - /* - * Note: RISC-V Privilege spec mandates that virtual addresses - * need to be sign-extended, so if (VA_BITS - 1) is set, all - * bits >= VA_BITS need to also be set or else we'll get a - * page fault. However the code that creates the mappings - * above us (e.g. iommu_dma_alloc_iova()) won't do that for us - * for now, so we'll end up with invalid virtual addresses - * to map. As a workaround until we get this sorted out - * limit the available virtual addresses to VA_BITS - 1. - */ - va_mask = DMA_BIT_MASK(va_bits - 1); - - domain->domain.geometry.aperture_start = 0; - domain->domain.geometry.aperture_end = va_mask; - domain->domain.geometry.force_aperture = true; - domain->domain.pgsize_bitmap = va_mask & (SZ_4K | SZ_2M | SZ_1G | SZ_512G); - - domain->domain.ops = &riscv_iommu_paging_domain_ops; - + ret = pt_iommu_riscv_64_init(&domain->riscvpt, &cfg, GFP_KERNEL); + if (ret) { + riscv_iommu_free_paging_domain(&domain->domain); + return ERR_PTR(ret); + } return &domain->domain; } @@ -1674,3 +1457,5 @@ int riscv_iommu_init(struct riscv_iommu_device *iommu) riscv_iommu_queue_disable(&iommu->cmdq); return rc; } + +MODULE_IMPORT_NS("GENERIC_PT_IOMMU"); From 69541898b71a8cb8a06706c67a2f756623598aa0 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Feb 2026 11:25:39 -0400 Subject: [PATCH 06/52] iommu/riscv: Enable SVNAPOT support for contiguous ptes This turns on a 64k page size. The "RISC-V IOMMU Architecture Specification" states: 6.4 IOMMU capabilities [..] IOMMU implementations must support the Svnapot standard extension for NAPOT Translation Contiguity. So just switch it on unconditionally. Cc: Xu Lu Tested-by: Vincent Chen Acked-by: Paul Walmsley # arch/riscv Reviewed-by: Tomasz Jeznach Tested-by: Tomasz Jeznach Signed-off-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index 6ceca9bed917..5016475587b8 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -1185,8 +1185,13 @@ static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev) INIT_LIST_HEAD_RCU(&domain->bonds); spin_lock_init(&domain->lock); + /* + * 6.4 IOMMU capabilities [..] IOMMU implementations must support the + * Svnapot standard extension for NAPOT Translation Contiguity. + */ cfg.common.features = BIT(PT_FEAT_SIGN_EXTEND) | - BIT(PT_FEAT_FLUSH_RANGE); + BIT(PT_FEAT_FLUSH_RANGE) | + BIT(PT_FEAT_RISCV_SVNAPOT_64K); domain->riscvpt.iommu.nid = dev_to_node(iommu->dev); domain->domain.ops = &riscv_iommu_paging_domain_ops; From c70d20b25ca30d68b377b9363a2adca6eb2538e3 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Feb 2026 11:25:40 -0400 Subject: [PATCH 07/52] iommu/riscv: Add missing GENERIC_MSI_IRQ The commit below added MSI related calls to the driver that depends on GENERIC_MSI_IRQ. It is possible to build RISC-V without this selected. This is also necessary to make the driver COMPILE_TEST. Fixes: d5f88acdd6ff ("iommu/riscv: Add support for platform msi") Tested-by: Vincent Chen Tested-by: Tomasz Jeznach Signed-off-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig index a329ec634cf1..849e72321d8a 100644 --- a/drivers/iommu/riscv/Kconfig +++ b/drivers/iommu/riscv/Kconfig @@ -4,6 +4,7 @@ config RISCV_IOMMU bool "RISC-V IOMMU Support" depends on RISCV && 64BIT + depends on GENERIC_MSI_IRQ default y select IOMMU_API select GENERIC_PT From 7cd0c655f02f08a5de851059ac8360e5d10fae62 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Feb 2026 11:25:41 -0400 Subject: [PATCH 08/52] iommu/riscv: Allow RISC_VIOMMU to COMPILE_TEST This driver used to use a lot of page table constants from the architecture code which prevented COMPILE_TEST on other architectures. Now that iommupt provides all of the constants internally there are only two small bumps preventing COMPILE_TEST. - Use the generic functions for the riscv specific phys_to_pfn() and pfn_to_phys() - Use CONFIG_MMIOWB to block off the mmiowb() barrier - Require 64 bit because of writeq use failing compilation on 32 bit Tested-by: Vincent Chen Acked-by: Paul Walmsley # arch/riscv Reviewed-by: Tomasz Jeznach Tested-by: Tomasz Jeznach Signed-off-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/Kconfig | 4 ++-- drivers/iommu/riscv/iommu-bits.h | 4 +++- drivers/iommu/riscv/iommu.c | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig index 849e72321d8a..b86e5ab94183 100644 --- a/drivers/iommu/riscv/Kconfig +++ b/drivers/iommu/riscv/Kconfig @@ -3,9 +3,9 @@ config RISCV_IOMMU bool "RISC-V IOMMU Support" - depends on RISCV && 64BIT + default RISCV depends on GENERIC_MSI_IRQ - default y + depends on (RISCV || COMPILE_TEST) && 64BIT select IOMMU_API select GENERIC_PT select IOMMU_PT diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h index 98daf0e1a306..29a0040b1c32 100644 --- a/drivers/iommu/riscv/iommu-bits.h +++ b/drivers/iommu/riscv/iommu-bits.h @@ -17,6 +17,7 @@ #include #include #include +#include /* * Chapter 5: Memory Mapped register interface @@ -718,7 +719,8 @@ static inline void riscv_iommu_cmd_inval_vma(struct riscv_iommu_command *cmd) static inline void riscv_iommu_cmd_inval_set_addr(struct riscv_iommu_command *cmd, u64 addr) { - cmd->dword1 = FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_ADDR, phys_to_pfn(addr)); + cmd->dword1 = + FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_ADDR, PHYS_PFN(addr)); cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_AV; } diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index 5016475587b8..c7d0342aa747 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -160,7 +160,7 @@ static int riscv_iommu_queue_alloc(struct riscv_iommu_device *iommu, if (FIELD_GET(RISCV_IOMMU_PPN_FIELD, qb)) { const size_t queue_size = entry_size << (logsz + 1); - queue->phys = pfn_to_phys(FIELD_GET(RISCV_IOMMU_PPN_FIELD, qb)); + queue->phys = PFN_PHYS(FIELD_GET(RISCV_IOMMU_PPN_FIELD, qb)); queue->base = devm_ioremap(iommu->dev, queue->phys, queue_size); } else { do { @@ -436,7 +436,9 @@ static unsigned int riscv_iommu_queue_send(struct riscv_iommu_queue *queue, * 6. Make sure the doorbell write to the device has finished before updating * the shadow tail index in normal memory. 'fence o, w' */ +#ifdef CONFIG_MMIOWB mmiowb(); +#endif atomic_inc(&queue->tail); /* 7. Complete submission and restore local interrupts */ From 199036ae01321651fe0e4488f9e19a28af4c5f1d Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Feb 2026 11:27:02 -0400 Subject: [PATCH 09/52] iommupt: Optimize the gather processing for DMA-FQ mode In PT_FEAT_FLUSH_RANGE mode the gather was accumulated but never flushed and then the accumulated range was discarded by the dma-iommu code in DMA-FQ mode. This is basically optimal. However for PT_FEAT_FLUSH_RANGE_NO_GAPS the page table would push flushes that are redundant with the flush all generated by the DMA-FQ mode. Disable all range accumulation in the gather, and iommu_pt triggered flushing when in iommu_iotlb_gather_queued() indicates it is in DMA-FQ mode. Reported-by: Robin Murphy Closes: https://lore.kernel.org/r/794b6121-b66b-4819-b291-9761ed21cd83@arm.com Signed-off-by: Jason Gunthorpe Reviewed-by: Lu Baolu Reviewed-by: Samiullah Khawaja Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/iommu_pt.h | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index 3e33fe64feab..9c08bb594e41 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -51,16 +51,27 @@ static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather, iommu_pages_stop_incoherent_list(free_list, iommu_table->iommu_device); - if (pt_feature(common, PT_FEAT_FLUSH_RANGE_NO_GAPS) && - iommu_iotlb_gather_is_disjoint(iotlb_gather, iova, len)) { - iommu_iotlb_sync(&iommu_table->domain, iotlb_gather); - /* - * Note that the sync frees the gather's free list, so we must - * not have any pages on that list that are covered by iova/len - */ + /* + * If running in DMA-FQ mode then the unmap will be followed by an IOTLB + * flush all so we need to optimize by never flushing the IOTLB here. + * + * For NO_GAPS the user gets to pick if flushing all or doing micro + * flushes is better for their work load by choosing DMA vs DMA-FQ + * operation. Drivers should also see shadow_on_flush. + */ + if (!iommu_iotlb_gather_queued(iotlb_gather)) { + if (pt_feature(common, PT_FEAT_FLUSH_RANGE_NO_GAPS) && + iommu_iotlb_gather_is_disjoint(iotlb_gather, iova, len)) { + iommu_iotlb_sync(&iommu_table->domain, iotlb_gather); + /* + * Note that the sync frees the gather's free list, so + * we must not have any pages on that list that are + * covered by iova/len + */ + } + iommu_iotlb_gather_add_range(iotlb_gather, iova, len); } - iommu_iotlb_gather_add_range(iotlb_gather, iova, len); iommu_pages_list_splice(free_list, &iotlb_gather->freelist); } From 1e0c8d6b695217cb0b16f13e31c7f08b453097b4 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Tue, 10 Feb 2026 22:58:39 -0500 Subject: [PATCH 10/52] iommu/amd: Add NUMA node affinity for IOMMU log buffers Currently, PPR Log and GA logs for AMD IOMMU are allocated using iommu_alloc_pages_sz(), which does not account for NUMA affinity. This can lead to remote memory access latencies if the memory is allocated on a different node than the IOMMU hardware. Switch to iommu_alloc_pages_node_sz() to ensure that these data structures are allocated on the same NUMA node as the IOMMU device. If the node information is unavailable, it defaults to NUMA_NO_NODE. Signed-off-by: Li RongQing Reviewed-by: Vasant Hegde Reviewed-by: Ankit Soni Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index f3fd7f39efb4..56ad020df494 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -848,10 +848,11 @@ static void __init free_command_buffer(struct amd_iommu *iommu) void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp, size_t size) { + int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; void *buf; size = PAGE_ALIGN(size); - buf = iommu_alloc_pages_sz(gfp, size); + buf = iommu_alloc_pages_node_sz(nid, gfp, size); if (!buf) return NULL; if (check_feature(FEATURE_SNP) && @@ -954,14 +955,16 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu) static int iommu_init_ga_log(struct amd_iommu *iommu) { + int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; + if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) return 0; - iommu->ga_log = iommu_alloc_pages_sz(GFP_KERNEL, GA_LOG_SIZE); + iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE); if (!iommu->ga_log) goto err_out; - iommu->ga_log_tail = iommu_alloc_pages_sz(GFP_KERNEL, 8); + iommu->ga_log_tail = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, 8); if (!iommu->ga_log_tail) goto err_out; From fa8fb60d36375ca3166a60589a624f0d0bc9ddb5 Mon Sep 17 00:00:00 2001 From: lynn Date: Sat, 14 Feb 2026 08:09:19 +0000 Subject: [PATCH 11/52] iommu/iova: Add NULL check in iova_magazine_free() When iova_domain_init_rcaches() fails to allocate an iova_magazine during the initialization of per-cpu rcaches, it jumps to out_err and calls free_iova_rcaches() for cleanup. In free_iova_rcaches(), the code iterates through all possible CPUs to free both cpu_rcache->loaded and cpu_rcache->prev. However, if the original allocation failed mid-way through the CPU loop, the pointers for the remaining CPUs remain NULL. Since kmem_cache_free() does not explicitly handle NULL pointers like kfree() does, passing these NULL pointers leads to a kernel paging request fault. Add a NULL check in iova_magazine_free() to safely handle partially initialized rcaches in error paths. Signed-off-by: lynn Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index f9cd18316d16..021daf6528de 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -611,7 +611,8 @@ static struct iova_magazine *iova_magazine_alloc(gfp_t flags) static void iova_magazine_free(struct iova_magazine *mag) { - kmem_cache_free(iova_magazine_cache, mag); + if (mag) + kmem_cache_free(iova_magazine_cache, mag); } static void From b2e5684558edf3e9bbe18d0e0043854994eab1be Mon Sep 17 00:00:00 2001 From: Fangyu Yu Date: Fri, 27 Feb 2026 19:26:40 +0800 Subject: [PATCH 12/52] iommu/riscv: Stop polling when CQCSR reports an error The cmdq wait loop busy-polls the consumer index until it advances or the software timeout expires. If the IOMMU has already signaled a command queue failure in CQCSR, continuing to poll for progress is pointless. Make riscv_iommu_queue_wait() also terminate the poll when any of these CQCSR error bits are observed. This helps the caller return earlier in failure cases and avoids spinning until the full timeout interval when the hardware has already reported an error. On single-core systems in particular, the current busy-wait can delay servicing the command-timeout interrupt until the software timeout expires (90s by default). Fixes: 856c0cfe5c5f ("iommu/riscv: Command and fault queue support") Signed-off-by: Fangyu Yu Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index aadfbc181138..6ac7e3edef8a 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -368,6 +368,8 @@ static int riscv_iommu_queue_wait(struct riscv_iommu_queue *queue, unsigned int timeout_us) { unsigned int cons = atomic_read(&queue->head); + unsigned int flags = RISCV_IOMMU_CQCSR_CQMF | RISCV_IOMMU_CQCSR_CMD_TO | + RISCV_IOMMU_CQCSR_CMD_ILL; /* Already processed by the consumer */ if ((int)(cons - index) > 0) @@ -375,6 +377,7 @@ static int riscv_iommu_queue_wait(struct riscv_iommu_queue *queue, /* Monitor consumer index */ return readx_poll_timeout(riscv_iommu_queue_cons, queue, cons, + (riscv_iommu_readl(queue->iommu, queue->qcr) & flags) || (int)(cons - index) > 0, 0, timeout_us); } From 99fb8afa16add85ed016baee9735231bca0c32b4 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Feb 2026 15:30:10 -0400 Subject: [PATCH 13/52] iommupt: Directly call iommupt's unmap_range() The common algorithm in iommupt does not require the iommu_pgsize() calculations, it can directly unmap any arbitrary range. Add a new function pointer to directly call an iommupt unmap_range op and make __iommu_unmap() call it directly. Gives about a 5% gain on single page unmappings. The function pointer is run through pt_iommu_ops instead of iommu_domain_ops to discourage using it outside iommupt. All drivers with their own page tables should continue to use the simplified map/unmap_pages() style interfaces. Reviewed-by: Samiullah Khawaja Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe Reviewed-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/iommu_pt.h | 29 ++++------------------ drivers/iommu/iommu.c | 27 ++++++++++++++++----- include/linux/generic_pt/iommu.h | 37 ++++++++++++++++++++++++----- include/linux/iommu.h | 1 + 4 files changed, 57 insertions(+), 37 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index 9c08bb594e41..a627c26fa62d 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -1031,34 +1031,12 @@ static __maybe_unused int __unmap_range(struct pt_range *range, void *arg, return ret; } -/** - * unmap_pages() - Make a range of IOVA empty/not present - * @domain: Domain to manipulate - * @iova: IO virtual address to start - * @pgsize: Length of each page - * @pgcount: Length of the range in pgsize units starting from @iova - * @iotlb_gather: Gather struct that must be flushed on return - * - * unmap_pages() will remove a translation created by map_pages(). It cannot - * subdivide a mapping created by map_pages(), so it should be called with IOVA - * ranges that match those passed to map_pages(). The IOVA range can aggregate - * contiguous map_pages() calls so long as no individual range is split. - * - * Context: The caller must hold a write range lock that includes - * the whole range. - * - * Returns: Number of bytes of VA unmapped. iova + res will be the point - * unmapping stopped. - */ -size_t DOMAIN_NS(unmap_pages)(struct iommu_domain *domain, unsigned long iova, - size_t pgsize, size_t pgcount, +static size_t NS(unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova, + dma_addr_t len, struct iommu_iotlb_gather *iotlb_gather) { - struct pt_iommu *iommu_table = - container_of(domain, struct pt_iommu, domain); struct pt_unmap_args unmap = { .free_list = IOMMU_PAGES_LIST_INIT( unmap.free_list) }; - pt_vaddr_t len = pgsize * pgcount; struct pt_range range; int ret; @@ -1073,7 +1051,6 @@ size_t DOMAIN_NS(unmap_pages)(struct iommu_domain *domain, unsigned long iova, return unmap.unmapped; } -EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(unmap_pages), "GENERIC_PT_IOMMU"); static void NS(get_info)(struct pt_iommu *iommu_table, struct pt_iommu_info *info) @@ -1121,6 +1098,7 @@ static void NS(deinit)(struct pt_iommu *iommu_table) } static const struct pt_iommu_ops NS(ops) = { + .unmap_range = NS(unmap_range), #if IS_ENABLED(CONFIG_IOMMUFD_DRIVER) && defined(pt_entry_is_write_dirty) && \ IS_ENABLED(CONFIG_IOMMUFD_TEST) && defined(pt_entry_make_write_dirty) .set_dirty = NS(set_dirty), @@ -1183,6 +1161,7 @@ static int pt_iommu_init_domain(struct pt_iommu *iommu_table, domain->type = __IOMMU_DOMAIN_PAGING; domain->pgsize_bitmap = info.pgsize_bitmap; + domain->is_iommupt = true; if (pt_feature(common, PT_FEAT_DYNAMIC_TOP)) range = _pt_top_range(common, diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 35db51780954..f68269707101 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "dma-iommu.h" #include "iommu-priv.h" @@ -2666,13 +2667,12 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, } EXPORT_SYMBOL_GPL(iommu_map); -static size_t __iommu_unmap(struct iommu_domain *domain, - unsigned long iova, size_t size, - struct iommu_iotlb_gather *iotlb_gather) +static size_t +__iommu_unmap_domain_pgtbl(struct iommu_domain *domain, unsigned long iova, + size_t size, struct iommu_iotlb_gather *iotlb_gather) { const struct iommu_domain_ops *ops = domain->ops; size_t unmapped_page, unmapped = 0; - unsigned long orig_iova = iova; unsigned int min_pagesz; if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) @@ -2718,8 +2718,23 @@ static size_t __iommu_unmap(struct iommu_domain *domain, unmapped += unmapped_page; } - trace_unmap(orig_iova, size, unmapped); - iommu_debug_unmap_end(domain, orig_iova, size, unmapped); + return unmapped; +} + +static size_t __iommu_unmap(struct iommu_domain *domain, unsigned long iova, + size_t size, + struct iommu_iotlb_gather *iotlb_gather) +{ + struct pt_iommu *pt = iommupt_from_domain(domain); + size_t unmapped; + + if (pt) + unmapped = pt->ops->unmap_range(pt, iova, size, iotlb_gather); + else + unmapped = __iommu_unmap_domain_pgtbl(domain, iova, size, + iotlb_gather); + trace_unmap(iova, size, unmapped); + iommu_debug_unmap_end(domain, iova, size, unmapped); return unmapped; } diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h index 49d9addb98c5..0da971134a37 100644 --- a/include/linux/generic_pt/iommu.h +++ b/include/linux/generic_pt/iommu.h @@ -66,6 +66,13 @@ struct pt_iommu { struct device *iommu_device; }; +static inline struct pt_iommu *iommupt_from_domain(struct iommu_domain *domain) +{ + if (!IS_ENABLED(CONFIG_IOMMU_PT) || !domain->is_iommupt) + return NULL; + return container_of(domain, struct pt_iommu, domain); +} + /** * struct pt_iommu_info - Details about the IOMMU page table * @@ -80,6 +87,29 @@ struct pt_iommu_info { }; struct pt_iommu_ops { + /** + * @unmap_range: Make a range of IOVA empty/not present + * @iommu_table: Table to manipulate + * @iova: IO virtual address to start + * @len: Length of the range starting from @iova + * @iotlb_gather: Gather struct that must be flushed on return + * + * unmap_range() will remove a translation created by map_range(). It + * cannot subdivide a mapping created by map_range(), so it should be + * called with IOVA ranges that match those passed to map_pages. The + * IOVA range can aggregate contiguous map_range() calls so long as no + * individual range is split. + * + * Context: The caller must hold a write range lock that includes + * the whole range. + * + * Returns: Number of bytes of VA unmapped. iova + res will be the + * point unmapping stopped. + */ + size_t (*unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova, + dma_addr_t len, + struct iommu_iotlb_gather *iotlb_gather); + /** * @set_dirty: Make the iova write dirty * @iommu_table: Table to manipulate @@ -198,10 +228,6 @@ struct pt_iommu_cfg { unsigned long iova, phys_addr_t paddr, \ size_t pgsize, size_t pgcount, \ int prot, gfp_t gfp, size_t *mapped); \ - size_t pt_iommu_##fmt##_unmap_pages( \ - struct iommu_domain *domain, unsigned long iova, \ - size_t pgsize, size_t pgcount, \ - struct iommu_iotlb_gather *iotlb_gather); \ int pt_iommu_##fmt##_read_and_clear_dirty( \ struct iommu_domain *domain, unsigned long iova, size_t size, \ unsigned long flags, struct iommu_dirty_bitmap *dirty); \ @@ -223,8 +249,7 @@ struct pt_iommu_cfg { */ #define IOMMU_PT_DOMAIN_OPS(fmt) \ .iova_to_phys = &pt_iommu_##fmt##_iova_to_phys, \ - .map_pages = &pt_iommu_##fmt##_map_pages, \ - .unmap_pages = &pt_iommu_##fmt##_unmap_pages + .map_pages = &pt_iommu_##fmt##_map_pages #define IOMMU_PT_DIRTY_OPS(fmt) \ .read_and_clear_dirty = &pt_iommu_##fmt##_read_and_clear_dirty diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 54b8b48c762e..7ca648c01336 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -223,6 +223,7 @@ enum iommu_domain_cookie_type { struct iommu_domain { unsigned type; enum iommu_domain_cookie_type cookie_type; + bool is_iommupt; const struct iommu_domain_ops *ops; const struct iommu_dirty_ops *dirty_ops; const struct iommu_ops *owner; /* Whose domain_alloc we came from */ From d6c65b0fd6218bd21ed0be7a8d3218e8f6dc91de Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Feb 2026 15:30:11 -0400 Subject: [PATCH 14/52] iommupt: Avoid rewalking during map Currently the core code provides a simplified interface to drivers where it fragments a requested multi-page map into single page size steps after doing all the calculations to figure out what page size is appropriate. Each step rewalks the page tables from the start. Since iommupt has a single implementation of the mapping algorithm it can internally compute each step as it goes while retaining its current position in the walk. Add a new function pt_pgsz_count() which computes the same page size fragement of a large mapping operations. Compute the next fragment when all the leaf entries of the current fragement have been written, then continue walking from the current point. The function pointer is run through pt_iommu_ops instead of iommu_domain_ops to discourage using it outside iommupt. All drivers with their own page tables should continue to use the simplified map_pages() style interfaces. Reviewed-by: Samiullah Khawaja Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe Reviewed-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/iommu_pt.h | 133 ++++++++++++-------- drivers/iommu/generic_pt/kunit_generic_pt.h | 12 ++ drivers/iommu/generic_pt/pt_iter.h | 22 ++++ drivers/iommu/iommu.c | 39 ++++-- include/linux/generic_pt/iommu.h | 34 ++++- 5 files changed, 175 insertions(+), 65 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index a627c26fa62d..17b72dbd7d51 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -477,6 +477,7 @@ struct pt_iommu_map_args { pt_oaddr_t oa; unsigned int leaf_pgsize_lg2; unsigned int leaf_level; + pt_vaddr_t num_leaves; }; /* @@ -529,11 +530,15 @@ static int clear_contig(const struct pt_state *start_pts, static int __map_range_leaf(struct pt_range *range, void *arg, unsigned int level, struct pt_table_p *table) { + struct pt_iommu *iommu_table = iommu_from_common(range->common); struct pt_state pts = pt_init(range, level, table); struct pt_iommu_map_args *map = arg; unsigned int leaf_pgsize_lg2 = map->leaf_pgsize_lg2; unsigned int start_index; pt_oaddr_t oa = map->oa; + unsigned int num_leaves; + unsigned int orig_end; + pt_vaddr_t last_va; unsigned int step; bool need_contig; int ret = 0; @@ -547,6 +552,15 @@ static int __map_range_leaf(struct pt_range *range, void *arg, _pt_iter_first(&pts); start_index = pts.index; + orig_end = pts.end_index; + if (pts.index + map->num_leaves < pts.end_index) { + /* Need to stop in the middle of the table to change sizes */ + pts.end_index = pts.index + map->num_leaves; + num_leaves = 0; + } else { + num_leaves = map->num_leaves - (pts.end_index - pts.index); + } + do { pts.type = pt_load_entry_raw(&pts); if (pts.type != PT_ENTRY_EMPTY || need_contig) { @@ -572,7 +586,40 @@ static int __map_range_leaf(struct pt_range *range, void *arg, flush_writes_range(&pts, start_index, pts.index); map->oa = oa; - return ret; + map->num_leaves = num_leaves; + if (ret || num_leaves) + return ret; + + /* range->va is not valid if we reached the end of the table */ + pts.index -= step; + pt_index_to_va(&pts); + pts.index += step; + last_va = range->va + log2_to_int(leaf_pgsize_lg2); + + if (last_va - 1 == range->last_va) { + PT_WARN_ON(pts.index != orig_end); + return 0; + } + + /* + * Reached a point where the page size changed, compute the new + * parameters. + */ + map->leaf_pgsize_lg2 = pt_compute_best_pgsize( + iommu_table->domain.pgsize_bitmap, last_va, range->last_va, oa); + map->leaf_level = + pt_pgsz_lg2_to_level(range->common, map->leaf_pgsize_lg2); + map->num_leaves = pt_pgsz_count(iommu_table->domain.pgsize_bitmap, + last_va, range->last_va, oa, + map->leaf_pgsize_lg2); + + /* Didn't finish this table level, caller will repeat it */ + if (pts.index != orig_end) { + if (pts.index != start_index) + pt_index_to_va(&pts); + return -EAGAIN; + } + return 0; } static int __map_range(struct pt_range *range, void *arg, unsigned int level, @@ -595,14 +642,9 @@ static int __map_range(struct pt_range *range, void *arg, unsigned int level, if (pts.type != PT_ENTRY_EMPTY) return -EADDRINUSE; ret = pt_iommu_new_table(&pts, &map->attrs); - if (ret) { - /* - * Racing with another thread installing a table - */ - if (ret == -EAGAIN) - continue; + /* EAGAIN on a race will loop again */ + if (ret) return ret; - } } else { pts.table_lower = pt_table_ptr(&pts); /* @@ -626,10 +668,12 @@ static int __map_range(struct pt_range *range, void *arg, unsigned int level, * The already present table can possibly be shared with another * concurrent map. */ - if (map->leaf_level == level - 1) - ret = pt_descend(&pts, arg, __map_range_leaf); - else - ret = pt_descend(&pts, arg, __map_range); + do { + if (map->leaf_level == level - 1) + ret = pt_descend(&pts, arg, __map_range_leaf); + else + ret = pt_descend(&pts, arg, __map_range); + } while (ret == -EAGAIN); if (ret) return ret; @@ -637,6 +681,14 @@ static int __map_range(struct pt_range *range, void *arg, unsigned int level, pt_index_to_va(&pts); if (pts.index >= pts.end_index) break; + + /* + * This level is currently running __map_range_leaf() which is + * not correct if the target level has been updated to this + * level. Have the caller invoke __map_range_leaf. + */ + if (map->leaf_level == level) + return -EAGAIN; } while (true); return 0; } @@ -808,12 +860,13 @@ static int check_map_range(struct pt_iommu *iommu_table, struct pt_range *range, static int do_map(struct pt_range *range, struct pt_common *common, bool single_page, struct pt_iommu_map_args *map) { + int ret; + /* * The __map_single_page() fast path does not support DMA_INCOHERENT * flushing to keep its .text small. */ if (single_page && !pt_feature(common, PT_FEAT_DMA_INCOHERENT)) { - int ret; ret = pt_walk_range(range, __map_single_page, map); if (ret != -EAGAIN) @@ -821,50 +874,25 @@ static int do_map(struct pt_range *range, struct pt_common *common, /* EAGAIN falls through to the full path */ } - if (map->leaf_level == range->top_level) - return pt_walk_range(range, __map_range_leaf, map); - return pt_walk_range(range, __map_range, map); + do { + if (map->leaf_level == range->top_level) + ret = pt_walk_range(range, __map_range_leaf, map); + else + ret = pt_walk_range(range, __map_range, map); + } while (ret == -EAGAIN); + return ret; } -/** - * map_pages() - Install translation for an IOVA range - * @domain: Domain to manipulate - * @iova: IO virtual address to start - * @paddr: Physical/Output address to start - * @pgsize: Length of each page - * @pgcount: Length of the range in pgsize units starting from @iova - * @prot: A bitmap of IOMMU_READ/WRITE/CACHE/NOEXEC/MMIO - * @gfp: GFP flags for any memory allocations - * @mapped: Total bytes successfully mapped - * - * The range starting at IOVA will have paddr installed into it. The caller - * must specify a valid pgsize and pgcount to segment the range into compatible - * blocks. - * - * On error the caller will probably want to invoke unmap on the range from iova - * up to the amount indicated by @mapped to return the table back to an - * unchanged state. - * - * Context: The caller must hold a write range lock that includes the whole - * range. - * - * Returns: -ERRNO on failure, 0 on success. The number of bytes of VA that were - * mapped are added to @mapped, @mapped is not zerod first. - */ -int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, size_t pgsize, size_t pgcount, - int prot, gfp_t gfp, size_t *mapped) +static int NS(map_range)(struct pt_iommu *iommu_table, dma_addr_t iova, + phys_addr_t paddr, dma_addr_t len, unsigned int prot, + gfp_t gfp, size_t *mapped) { - struct pt_iommu *iommu_table = - container_of(domain, struct pt_iommu, domain); pt_vaddr_t pgsize_bitmap = iommu_table->domain.pgsize_bitmap; struct pt_common *common = common_from_iommu(iommu_table); struct iommu_iotlb_gather iotlb_gather; - pt_vaddr_t len = pgsize * pgcount; struct pt_iommu_map_args map = { .iotlb_gather = &iotlb_gather, .oa = paddr, - .leaf_pgsize_lg2 = vaffs(pgsize), }; bool single_page = false; struct pt_range range; @@ -892,13 +920,13 @@ int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova, return ret; /* Calculate target page size and level for the leaves */ - if (pt_has_system_page_size(common) && pgsize == PAGE_SIZE && - pgcount == 1) { + if (pt_has_system_page_size(common) && len == PAGE_SIZE) { PT_WARN_ON(!(pgsize_bitmap & PAGE_SIZE)); if (log2_mod(iova | paddr, PAGE_SHIFT)) return -ENXIO; map.leaf_pgsize_lg2 = PAGE_SHIFT; map.leaf_level = 0; + map.num_leaves = 1; single_page = true; } else { map.leaf_pgsize_lg2 = pt_compute_best_pgsize( @@ -907,6 +935,9 @@ int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova, return -ENXIO; map.leaf_level = pt_pgsz_lg2_to_level(common, map.leaf_pgsize_lg2); + map.num_leaves = pt_pgsz_count(pgsize_bitmap, range.va, + range.last_va, paddr, + map.leaf_pgsize_lg2); } ret = check_map_range(iommu_table, &range, &map); @@ -929,7 +960,6 @@ int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova, *mapped += map.oa - paddr; return ret; } -EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(map_pages), "GENERIC_PT_IOMMU"); struct pt_unmap_args { struct iommu_pages_list free_list; @@ -1098,6 +1128,7 @@ static void NS(deinit)(struct pt_iommu *iommu_table) } static const struct pt_iommu_ops NS(ops) = { + .map_range = NS(map_range), .unmap_range = NS(unmap_range), #if IS_ENABLED(CONFIG_IOMMUFD_DRIVER) && defined(pt_entry_is_write_dirty) && \ IS_ENABLED(CONFIG_IOMMUFD_TEST) && defined(pt_entry_make_write_dirty) diff --git a/drivers/iommu/generic_pt/kunit_generic_pt.h b/drivers/iommu/generic_pt/kunit_generic_pt.h index 68278bf15cfe..374e475f591e 100644 --- a/drivers/iommu/generic_pt/kunit_generic_pt.h +++ b/drivers/iommu/generic_pt/kunit_generic_pt.h @@ -312,6 +312,17 @@ static void test_best_pgsize(struct kunit *test) } } +static void test_pgsz_count(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, + pt_pgsz_count(SZ_4K, 0, SZ_1G - 1, 0, ilog2(SZ_4K)), + SZ_1G / SZ_4K); + KUNIT_EXPECT_EQ(test, + pt_pgsz_count(SZ_2M | SZ_4K, SZ_4K, SZ_1G - 1, SZ_4K, + ilog2(SZ_4K)), + (SZ_2M - SZ_4K) / SZ_4K); +} + /* * Check that pt_install_table() and pt_table_pa() match */ @@ -770,6 +781,7 @@ static struct kunit_case generic_pt_test_cases[] = { KUNIT_CASE_FMT(test_init), KUNIT_CASE_FMT(test_bitops), KUNIT_CASE_FMT(test_best_pgsize), + KUNIT_CASE_FMT(test_pgsz_count), KUNIT_CASE_FMT(test_table_ptr), KUNIT_CASE_FMT(test_max_va), KUNIT_CASE_FMT(test_table_radix), diff --git a/drivers/iommu/generic_pt/pt_iter.h b/drivers/iommu/generic_pt/pt_iter.h index c0d8617cce29..3e45dbde6b83 100644 --- a/drivers/iommu/generic_pt/pt_iter.h +++ b/drivers/iommu/generic_pt/pt_iter.h @@ -569,6 +569,28 @@ static inline unsigned int pt_compute_best_pgsize(pt_vaddr_t pgsz_bitmap, return pgsz_lg2; } +/* + * Return the number of pgsize_lg2 leaf entries that can be mapped for + * va to oa. This accounts for any requirement to reduce or increase the page + * size across the VA range. + */ +static inline pt_vaddr_t pt_pgsz_count(pt_vaddr_t pgsz_bitmap, pt_vaddr_t va, + pt_vaddr_t last_va, pt_oaddr_t oa, + unsigned int pgsize_lg2) +{ + pt_vaddr_t len = last_va - va + 1; + pt_vaddr_t next_pgsizes = log2_set_mod(pgsz_bitmap, 0, pgsize_lg2 + 1); + + if (next_pgsizes) { + unsigned int next_pgsize_lg2 = vaffs(next_pgsizes); + + if (log2_mod(va ^ oa, next_pgsize_lg2) == 0) + len = min(len, log2_set_mod_max(va, next_pgsize_lg2) - + va + 1); + } + return log2_div(len, pgsize_lg2); +} + #define _PT_MAKE_CALL_LEVEL(fn) \ static __always_inline int fn(struct pt_range *range, void *arg, \ unsigned int level, \ diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index f68269707101..33cee64686e3 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2569,14 +2569,14 @@ static size_t iommu_pgsize(struct iommu_domain *domain, unsigned long iova, return pgsize; } -int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +static int __iommu_map_domain_pgtbl(struct iommu_domain *domain, + unsigned long iova, phys_addr_t paddr, + size_t size, int prot, gfp_t gfp) { const struct iommu_domain_ops *ops = domain->ops; unsigned long orig_iova = iova; unsigned int min_pagesz; size_t orig_size = size; - phys_addr_t orig_paddr = paddr; int ret = 0; might_sleep_if(gfpflags_allow_blocking(gfp)); @@ -2633,12 +2633,9 @@ int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, /* unroll mapping in case something went wrong */ if (ret) { iommu_unmap(domain, orig_iova, orig_size - size); - } else { - trace_map(orig_iova, orig_paddr, orig_size); - iommu_debug_map(domain, orig_paddr, orig_size); + return ret; } - - return ret; + return 0; } int iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) @@ -2650,6 +2647,32 @@ int iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) return ops->iotlb_sync_map(domain, iova, size); } +int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +{ + struct pt_iommu *pt = iommupt_from_domain(domain); + int ret; + + if (pt) { + size_t mapped = 0; + + ret = pt->ops->map_range(pt, iova, paddr, size, prot, gfp, + &mapped); + if (ret) { + iommu_unmap(domain, iova, mapped); + return ret; + } + return 0; + } + ret = __iommu_map_domain_pgtbl(domain, iova, paddr, size, prot, gfp); + if (!ret) + return ret; + + trace_map(iova, paddr, size); + iommu_debug_map(domain, paddr, size); + return 0; +} + int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h index 0da971134a37..dd0edd02a48a 100644 --- a/include/linux/generic_pt/iommu.h +++ b/include/linux/generic_pt/iommu.h @@ -87,6 +87,33 @@ struct pt_iommu_info { }; struct pt_iommu_ops { + /** + * @map_range: Install translation for an IOVA range + * @iommu_table: Table to manipulate + * @iova: IO virtual address to start + * @paddr: Physical/Output address to start + * @len: Length of the range starting from @iova + * @prot: A bitmap of IOMMU_READ/WRITE/CACHE/NOEXEC/MMIO + * @gfp: GFP flags for any memory allocations + * + * The range starting at IOVA will have paddr installed into it. The + * rage is automatically segmented into optimally sized table entries, + * and can have any valid alignment. + * + * On error the caller will probably want to invoke unmap on the range + * from iova up to the amount indicated by @mapped to return the table + * back to an unchanged state. + * + * Context: The caller must hold a write range lock that includes + * the whole range. + * + * Returns: -ERRNO on failure, 0 on success. The number of bytes of VA + * that were mapped are added to @mapped, @mapped is not zerod first. + */ + int (*map_range)(struct pt_iommu *iommu_table, dma_addr_t iova, + phys_addr_t paddr, dma_addr_t len, unsigned int prot, + gfp_t gfp, size_t *mapped); + /** * @unmap_range: Make a range of IOVA empty/not present * @iommu_table: Table to manipulate @@ -224,10 +251,6 @@ struct pt_iommu_cfg { #define IOMMU_PROTOTYPES(fmt) \ phys_addr_t pt_iommu_##fmt##_iova_to_phys(struct iommu_domain *domain, \ dma_addr_t iova); \ - int pt_iommu_##fmt##_map_pages(struct iommu_domain *domain, \ - unsigned long iova, phys_addr_t paddr, \ - size_t pgsize, size_t pgcount, \ - int prot, gfp_t gfp, size_t *mapped); \ int pt_iommu_##fmt##_read_and_clear_dirty( \ struct iommu_domain *domain, unsigned long iova, size_t size, \ unsigned long flags, struct iommu_dirty_bitmap *dirty); \ @@ -248,8 +271,7 @@ struct pt_iommu_cfg { * iommu_pt */ #define IOMMU_PT_DOMAIN_OPS(fmt) \ - .iova_to_phys = &pt_iommu_##fmt##_iova_to_phys, \ - .map_pages = &pt_iommu_##fmt##_map_pages + .iova_to_phys = &pt_iommu_##fmt##_iova_to_phys #define IOMMU_PT_DIRTY_OPS(fmt) \ .read_and_clear_dirty = &pt_iommu_##fmt##_read_and_clear_dirty From a82efb8747d1b8a7c0a377dc79c2aac204eae788 Mon Sep 17 00:00:00 2001 From: Shameer Kolothum Date: Tue, 17 Mar 2026 11:16:02 +0000 Subject: [PATCH 15/52] iommu: Add device ATS supported capability PCIe ATS may be disabled by platform firmware, root complex limitations, or kernel policy even when a device advertises the ATS capability in its PCI configuration space. Add a new IOMMU_CAP_PCI_ATS_SUPPORTED capability to allow IOMMU drivers to report the effective ATS decision for a device. When this capability is true for a device, ATS may be enabled for that device, but it does not imply that ATS is currently enabled. A subsequent patch will extend iommufd to expose the effective ATS status to userspace. Suggested-by: Jason Gunthorpe Reviewed-by: Jason Gunthorpe Signed-off-by: Shameer Kolothum Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 6 ++++++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 +++ drivers/iommu/intel/iommu.c | 2 ++ include/linux/iommu.h | 2 ++ 4 files changed, 13 insertions(+) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 81c4d7733872..f1814fee5182 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2985,6 +2985,12 @@ static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) return amd_iommu_hd_support(iommu); } + case IOMMU_CAP_PCI_ATS_SUPPORTED: { + struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); + + return amd_iommu_iotlb_sup && + (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP); + } default: break; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4d00d796f078..dec5cac98f7c 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -107,6 +107,7 @@ static const char * const event_class_str[] = { }; static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master); +static bool arm_smmu_ats_supported(struct arm_smmu_master *master); static void parse_driver_options(struct arm_smmu_device *smmu) { @@ -2494,6 +2495,8 @@ static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) return true; case IOMMU_CAP_DIRTY_TRACKING: return arm_smmu_dbm_capable(master->smmu); + case IOMMU_CAP_PCI_ATS_SUPPORTED: + return arm_smmu_ats_supported(master); default: return false; } diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index ef7613b177b9..5dca8e525c73 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3220,6 +3220,8 @@ static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap) return ecap_sc_support(info->iommu->ecap); case IOMMU_CAP_DIRTY_TRACKING: return ssads_supported(info->iommu); + case IOMMU_CAP_PCI_ATS_SUPPORTED: + return info->ats_supported; default: return false; } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 7ca648c01336..a904821ed169 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -272,6 +272,8 @@ enum iommu_cap { */ IOMMU_CAP_DEFERRED_FLUSH, IOMMU_CAP_DIRTY_TRACKING, /* IOMMU supports dirty tracking */ + /* ATS is supported and may be enabled for this device */ + IOMMU_CAP_PCI_ATS_SUPPORTED, }; /* These are the possible reserved region types */ From a11661a58c06f7fdfef03a368ef20d05a4ea4ed0 Mon Sep 17 00:00:00 2001 From: Shameer Kolothum Date: Tue, 17 Mar 2026 11:16:03 +0000 Subject: [PATCH 16/52] iommufd: Report ATS not supported status via IOMMU_GET_HW_INFO If the IOMMU driver reports that ATS is not supported for a device, set the IOMMU_HW_CAP_PCI_ATS_NOT_SUPPORTED flag in the returned hardware capabilities. This uses a negative flag for UAPI compatibility. Existing userspace assumes ATS is supported if no flag is present. This also ensures that new userspace works correctly on both old and new kernels, where a zero value implies ATS support. When this flag is set, ATS cannot be used for the device. When it is clear, ATS may be enabled when an appropriate HWPT is attached. Reviewed-by: Samiullah Khawaja Reviewed-by: Jason Gunthorpe Signed-off-by: Shameer Kolothum Signed-off-by: Joerg Roedel --- drivers/iommu/iommufd/device.c | 4 ++++ include/uapi/linux/iommufd.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index 344d620cdecc..92c5d5ef8d00 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -1624,6 +1624,10 @@ int iommufd_get_hw_info(struct iommufd_ucmd *ucmd) if (device_iommu_capable(idev->dev, IOMMU_CAP_DIRTY_TRACKING)) cmd->out_capabilities |= IOMMU_HW_CAP_DIRTY_TRACKING; + /* Report when ATS cannot be used for this device */ + if (!device_iommu_capable(idev->dev, IOMMU_CAP_PCI_ATS_SUPPORTED)) + cmd->out_capabilities |= IOMMU_HW_CAP_PCI_ATS_NOT_SUPPORTED; + cmd->out_max_pasid_log2 = 0; /* * Currently, all iommu drivers enable PASID in the probe_device() diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h index 1dafbc552d37..507ee9bcba01 100644 --- a/include/uapi/linux/iommufd.h +++ b/include/uapi/linux/iommufd.h @@ -695,11 +695,15 @@ enum iommu_hw_info_type { * @IOMMU_HW_CAP_PCI_PASID_PRIV: Privileged Mode Supported, user ignores it * when the struct * iommu_hw_info::out_max_pasid_log2 is zero. + * @IOMMU_HW_CAP_PCI_ATS_NOT_SUPPORTED: ATS is not supported or cannot be used + * on this device (absence implies ATS + * may be enabled) */ enum iommufd_hw_capabilities { IOMMU_HW_CAP_DIRTY_TRACKING = 1 << 0, IOMMU_HW_CAP_PCI_PASID_EXEC = 1 << 1, IOMMU_HW_CAP_PCI_PASID_PRIV = 1 << 2, + IOMMU_HW_CAP_PCI_ATS_NOT_SUPPORTED = 1 << 3, }; /** From 889cdd9e1b375e2423e5d69c2dd96722d28777b8 Mon Sep 17 00:00:00 2001 From: Abel Vesa Date: Mon, 23 Feb 2026 10:46:39 +0200 Subject: [PATCH 17/52] dt-bindings: arm-smmu: Add compatible for Eliza SoC Qualcomm Eliza SoC implements arm,mmu-500. Document its compatible. Signed-off-by: Abel Vesa Acked-by: Rob Herring (Arm) Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index cdbd23b5c08c..27d25bc98cbe 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -35,6 +35,7 @@ properties: - description: Qcom SoCs implementing "qcom,smmu-500" and "arm,mmu-500" items: - enum: + - qcom,eliza-smmu-500 - qcom,glymur-smmu-500 - qcom,kaanapali-smmu-500 - qcom,milos-smmu-500 From 6fabce53f6b9c2419012a9103e1a46d40888cefa Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 17 Mar 2026 00:59:16 -0700 Subject: [PATCH 18/52] iommu/arm-smmu-v3: Add a missing dma_wmb() for hitless STE update When writing a new (previously invalid) valid IOPTE to a page table, then installing the page table into an STE hitlesslessly (e.g. in S2TTB field), there is a window before an STE invalidation, where the page-table may be accessed by SMMU but the new IOPTE is still siting in the CPU cache. This could occur when we allocate an iommu_domain and immediately install it hitlessly, while there would be no dma_wmb() for the page table memory prior to the earliest point of HW reading the STE. Fix it by adding a dma_wmb() prior to updating the STE. Fixes: 56e1a4cc2588 ("iommu/arm-smmu-v3: Add unit tests for arm_smmu_write_entry") Cc: stable@vger.kernel.org Reported-by: Will Deacon Closes: https://lore.kernel.org/linux-iommu/aXdlnLLFUBwjT0V5@willie-the-truck/ Suggested-by: Jason Gunthorpe Signed-off-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4d00d796f078..606abe051e68 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1236,6 +1236,13 @@ void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry, __le64 unused_update[NUM_ENTRY_QWORDS]; u8 used_qword_diff; + /* + * Many of the entry structures have pointers to other structures that + * need to have their updates be visible before any writes of the entry + * happen. + */ + dma_wmb(); + used_qword_diff = arm_smmu_entry_qword_diff(writer, entry, target, unused_update); if (hweight8(used_qword_diff) == 1) { From 9b056856880a0a3de04e7b09521fe1f5df94e311 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 17 Mar 2026 00:59:17 -0700 Subject: [PATCH 19/52] iommu/arm-smmu-v3: Explicitly set smmu_domain->stage for SVA Both the ARM_SMMU_DOMAIN_S1 case and the SVA case use ASID, requiring ASID based invalidation commands to flush the TLB. Define an ARM_SMMU_DOMAIN_SVA to make the SVA case clear to share the same path with the ARM_SMMU_DOMAIN_S1 case, which will be a part of the routine to build a new per-domain invalidation array. There is no function change. Suggested-by: Jason Gunthorpe Acked-by: Balbir Singh Reviewed-by: Jason Gunthorpe Reviewed-by: Pranjal Shrivastava Signed-off-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 1 + drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 +++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 + 3 files changed, 5 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 59a480974d80..6097f1f540d8 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -346,6 +346,7 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, * ARM_SMMU_FEAT_RANGE_INV is present */ smmu_domain->domain.pgsize_bitmap = PAGE_SIZE; + smmu_domain->stage = ARM_SMMU_DOMAIN_SVA; smmu_domain->smmu = smmu; ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 606abe051e68..117979c96b76 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3132,6 +3132,9 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev, arm_smmu_install_ste_for_dev(master, &target); arm_smmu_clear_cd(master, IOMMU_NO_PASID); break; + default: + WARN_ON(true); + break; } arm_smmu_attach_commit(&state); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 3c6d65d36164..24894b163004 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -856,6 +856,7 @@ struct arm_smmu_master { enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_S1 = 0, ARM_SMMU_DOMAIN_S2, + ARM_SMMU_DOMAIN_SVA, }; struct arm_smmu_domain { From c317452f5a224b4ac97d51162395bd6bddaf478c Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 17 Mar 2026 00:59:18 -0700 Subject: [PATCH 20/52] iommu/arm-smmu-v3: Add an inline arm_smmu_domain_free() There will be a bit more things to free than smmu_domain itself. So keep a simple inline function in the header to share aross files. Suggested-by: Jason Gunthorpe Reviewed-by: Jason Gunthorpe Acked-by: Balbir Singh Reviewed-by: Pranjal Shrivastava Signed-off-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 5 +++-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 4 ++-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 6097f1f540d8..440ad8cc07de 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -197,7 +197,8 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) { - kfree(container_of(mn, struct arm_smmu_domain, mmu_notifier)); + arm_smmu_domain_free( + container_of(mn, struct arm_smmu_domain, mmu_notifier)); } static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { @@ -365,6 +366,6 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, err_asid: xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); err_free: - kfree(smmu_domain); + arm_smmu_domain_free(smmu_domain); return ERR_PTR(ret); } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 117979c96b76..375f2143efc1 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2559,7 +2559,7 @@ static void arm_smmu_domain_free_paging(struct iommu_domain *domain) ida_free(&smmu->vmid_map, cfg->vmid); } - kfree(smmu_domain); + arm_smmu_domain_free(smmu_domain); } static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu, @@ -3427,7 +3427,7 @@ arm_smmu_domain_alloc_paging_flags(struct device *dev, u32 flags, return &smmu_domain->domain; err_free: - kfree(smmu_domain); + arm_smmu_domain_free(smmu_domain); return ERR_PTR(ret); } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 24894b163004..cfbedb76c8ba 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -956,6 +956,11 @@ extern struct mutex arm_smmu_asid_lock; struct arm_smmu_domain *arm_smmu_domain_alloc(void); +static inline void arm_smmu_domain_free(struct arm_smmu_domain *smmu_domain) +{ + kfree(smmu_domain); +} + void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid); struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master, u32 ssid); From 15a2a5645ad79df78965a7c49bdd4b6a63b2033a Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Tue, 17 Mar 2026 00:59:19 -0700 Subject: [PATCH 21/52] iommu/arm-smmu-v3: Introduce a per-domain arm_smmu_invs array Create a new data structure to hold an array of invalidations that need to be performed for the domain based on what masters are attached, to replace the single smmu pointer and linked list of masters in the current design. Each array entry holds one of the invalidation actions - S1_ASID, S2_VMID, ATS or their variant with information to feed invalidation commands to HW. It is structured so that multiple SMMUs can participate in the same array, removing one key limitation of the current system. To maximize performance, a sorted array is used as the data structure. It allows grouping SYNCs together to parallelize invalidations. For instance, it will group all the ATS entries after the ASID/VMID entry, so they will all be pushed to the PCI devices in parallel with one SYNC. To minimize the locking cost on the invalidation fast path (reader of the invalidation array), the array is managed with RCU. Provide a set of APIs to add/delete entries to/from an array, which cover cannot-fail attach cases, e.g. attaching to arm_smmu_blocked_domain. Also add kunit coverage for those APIs. Signed-off-by: Jason Gunthorpe Reviewed-by: Jason Gunthorpe Reviewed-by: Pranjal Shrivastava Co-developed-by: Nicolin Chen Signed-off-by: Nicolin Chen Signed-off-by: Will Deacon --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 135 +++++++++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 270 ++++++++++++++++++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 97 +++++++ 3 files changed, 502 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index 69c9ef441fc1..add671363c82 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -637,6 +637,140 @@ static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test) NUM_EXPECTED_SYNCS(2)); } +static void arm_smmu_v3_invs_test_verify(struct kunit *test, + struct arm_smmu_invs *invs, + int num_invs, const int num_trashes, + const int *ids, const int *users, + const int *ssids) +{ + KUNIT_EXPECT_EQ(test, invs->num_invs, num_invs); + KUNIT_EXPECT_EQ(test, invs->num_trashes, num_trashes); + while (num_invs--) { + KUNIT_EXPECT_EQ(test, invs->inv[num_invs].id, ids[num_invs]); + KUNIT_EXPECT_EQ(test, READ_ONCE(invs->inv[num_invs].users), + users[num_invs]); + KUNIT_EXPECT_EQ(test, invs->inv[num_invs].ssid, ssids[num_invs]); + } +} + +static struct arm_smmu_invs invs1 = { + .num_invs = 3, + .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, + { .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, }, + { .type = INV_TYPE_ATS, .id = 3, }, }, +}; + +static struct arm_smmu_invs invs2 = { + .num_invs = 3, + .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ + { .type = INV_TYPE_ATS, .id = 4, }, + { .type = INV_TYPE_ATS, .id = 5, }, }, +}; + +static struct arm_smmu_invs invs3 = { + .num_invs = 3, + .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ + { .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */ + { .type = INV_TYPE_ATS, .id = 6, }, }, +}; + +static struct arm_smmu_invs invs4 = { + .num_invs = 3, + .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 1 }, + { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, + { .type = INV_TYPE_ATS, .id = 12, .ssid = 1 }, }, +}; + +static struct arm_smmu_invs invs5 = { + .num_invs = 3, + .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 2 }, + { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */ + { .type = INV_TYPE_ATS, .id = 12, .ssid = 2 }, }, +}; + +static void arm_smmu_v3_invs_test(struct kunit *test) +{ + const int results1[3][3] = { { 1, 1, 3, }, { 1, 1, 1, }, { 0, 0, 0, } }; + const int results2[3][5] = { { 1, 1, 3, 4, 5, }, { 2, 1, 1, 1, 1, }, { 0, 0, 0, 0, 0, } }; + const int results3[3][3] = { { 1, 1, 3, }, { 1, 1, 1, }, { 0, 0, 0, } }; + const int results4[3][5] = { { 1, 1, 3, 5, 6, }, { 2, 1, 1, 1, 1, }, { 0, 0, 0, 0, 0, } }; + const int results5[3][5] = { { 1, 1, 3, 5, 6, }, { 1, 0, 0, 1, 1, }, { 0, 0, 0, 0, 0, } }; + const int results6[3][3] = { { 1, 5, 6, }, { 1, 1, 1, }, { 0, 0, 0, } }; + const int results7[3][3] = { { 10, 10, 12, }, { 1, 1, 1, }, { 1, 3, 1, } }; + const int results8[3][5] = { { 10, 10, 10, 12, 12, }, { 1, 1, 2, 1, 1, }, { 1, 2, 3, 1, 2, } }; + const int results9[3][4] = { { 10, 10, 10, 12, }, { 1, 0, 1, 1, }, { 1, 2, 3, 1, } }; + const int results10[3][3] = { { 10, 10, 12, }, { 1, 1, 1, }, { 1, 3, 1, } }; + struct arm_smmu_invs *test_a, *test_b; + + /* New array */ + test_a = arm_smmu_invs_alloc(0); + KUNIT_EXPECT_EQ(test, test_a->num_invs, 0); + + /* Test1: merge invs1 (new array) */ + test_b = arm_smmu_invs_merge(test_a, &invs1); + kfree(test_a); + arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results1[0]), 0, + results1[0], results1[1], results1[2]); + + /* Test2: merge invs2 (new array) */ + test_a = arm_smmu_invs_merge(test_b, &invs2); + kfree(test_b); + arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results2[0]), 0, + results2[0], results2[1], results2[2]); + + /* Test3: unref invs2 (same array) */ + arm_smmu_invs_unref(test_a, &invs2); + arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results3[0]), 0, + results3[0], results3[1], results3[2]); + + /* Test4: merge invs3 (new array) */ + test_b = arm_smmu_invs_merge(test_a, &invs3); + kfree(test_a); + arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results4[0]), 0, + results4[0], results4[1], results4[2]); + + /* Test5: unref invs1 (same array) */ + arm_smmu_invs_unref(test_b, &invs1); + arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results5[0]), 2, + results5[0], results5[1], results5[2]); + + /* Test6: purge test_b (new array) */ + test_a = arm_smmu_invs_purge(test_b); + kfree(test_b); + arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results6[0]), 0, + results6[0], results6[1], results6[2]); + + /* Test7: unref invs3 (same array) */ + arm_smmu_invs_unref(test_a, &invs3); + KUNIT_EXPECT_EQ(test, test_a->num_invs, 0); + KUNIT_EXPECT_EQ(test, test_a->num_trashes, 0); + + /* Test8: merge invs4 (new array) */ + test_b = arm_smmu_invs_merge(test_a, &invs4); + kfree(test_a); + arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results7[0]), 0, + results7[0], results7[1], results7[2]); + + /* Test9: merge invs5 (new array) */ + test_a = arm_smmu_invs_merge(test_b, &invs5); + kfree(test_b); + arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results8[0]), 0, + results8[0], results8[1], results8[2]); + + /* Test10: unref invs5 (same array) */ + arm_smmu_invs_unref(test_a, &invs5); + arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results9[0]), 1, + results9[0], results9[1], results9[2]); + + /* Test11: purge test_a (new array) */ + test_b = arm_smmu_invs_purge(test_a); + kfree(test_a); + arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results10[0]), 0, + results10[0], results10[1], results10[2]); + + kfree(test_b); +} + static struct kunit_case arm_smmu_v3_test_cases[] = { KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_abort), KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_bypass), @@ -662,6 +796,7 @@ static struct kunit_case arm_smmu_v3_test_cases[] = { KUNIT_CASE(arm_smmu_v3_write_ste_test_nested_s1bypass_to_s1dssbypass), KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear), KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_release), + KUNIT_CASE(arm_smmu_v3_invs_test), {}, }; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 375f2143efc1..10050b1a6d94 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -1026,6 +1027,267 @@ static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused */ } +/* Invalidation array manipulation functions */ +static inline struct arm_smmu_inv * +arm_smmu_invs_iter_next(struct arm_smmu_invs *invs, size_t next, size_t *idx) +{ + while (true) { + if (next >= invs->num_invs) { + *idx = next; + return NULL; + } + if (!READ_ONCE(invs->inv[next].users)) { + next++; + continue; + } + *idx = next; + return &invs->inv[next]; + } +} + +/** + * arm_smmu_invs_for_each_entry - Iterate over all non-trash entries in invs + * @invs: the base invalidation array + * @idx: a stack variable of 'size_t', to store the array index + * @cur: a stack variable of 'struct arm_smmu_inv *' + */ +#define arm_smmu_invs_for_each_entry(invs, idx, cur) \ + for (cur = arm_smmu_invs_iter_next(invs, 0, &(idx)); cur; \ + cur = arm_smmu_invs_iter_next(invs, idx + 1, &(idx))) + +static int arm_smmu_inv_cmp(const struct arm_smmu_inv *inv_l, + const struct arm_smmu_inv *inv_r) +{ + if (inv_l->smmu != inv_r->smmu) + return cmp_int((uintptr_t)inv_l->smmu, (uintptr_t)inv_r->smmu); + if (inv_l->type != inv_r->type) + return cmp_int(inv_l->type, inv_r->type); + if (inv_l->id != inv_r->id) + return cmp_int(inv_l->id, inv_r->id); + if (arm_smmu_inv_is_ats(inv_l)) + return cmp_int(inv_l->ssid, inv_r->ssid); + return 0; +} + +static inline int arm_smmu_invs_iter_next_cmp(struct arm_smmu_invs *invs_l, + size_t next_l, size_t *idx_l, + struct arm_smmu_invs *invs_r, + size_t next_r, size_t *idx_r) +{ + struct arm_smmu_inv *cur_l = + arm_smmu_invs_iter_next(invs_l, next_l, idx_l); + + /* + * We have to update the idx_r manually, because the invs_r cannot call + * arm_smmu_invs_iter_next() as the invs_r never sets any users counter. + */ + *idx_r = next_r; + + /* + * Compare of two sorted arrays items. If one side is past the end of + * the array, return the other side to let it run out the iteration. + * + * If the left entry is empty, return 1 to pick the right entry. + * If the right entry is empty, return -1 to pick the left entry. + */ + if (!cur_l) + return 1; + if (next_r >= invs_r->num_invs) + return -1; + return arm_smmu_inv_cmp(cur_l, &invs_r->inv[next_r]); +} + +/** + * arm_smmu_invs_for_each_cmp - Iterate over two sorted arrays computing for + * arm_smmu_invs_merge() or arm_smmu_invs_unref() + * @invs_l: the base invalidation array + * @idx_l: a stack variable of 'size_t', to store the base array index + * @invs_r: the build_invs array as to_merge or to_unref + * @idx_r: a stack variable of 'size_t', to store the build_invs index + * @cmp: a stack variable of 'int', to store return value (-1, 0, or 1) + */ +#define arm_smmu_invs_for_each_cmp(invs_l, idx_l, invs_r, idx_r, cmp) \ + for (idx_l = idx_r = 0, \ + cmp = arm_smmu_invs_iter_next_cmp(invs_l, 0, &(idx_l), \ + invs_r, 0, &(idx_r)); \ + idx_l < invs_l->num_invs || idx_r < invs_r->num_invs; \ + cmp = arm_smmu_invs_iter_next_cmp( \ + invs_l, idx_l + (cmp <= 0 ? 1 : 0), &(idx_l), \ + invs_r, idx_r + (cmp >= 0 ? 1 : 0), &(idx_r))) + +/** + * arm_smmu_invs_merge() - Merge @to_merge into @invs and generate a new array + * @invs: the base invalidation array + * @to_merge: an array of invalidations to merge + * + * Return: a newly allocated array on success, or ERR_PTR + * + * This function must be locked and serialized with arm_smmu_invs_unref() and + * arm_smmu_invs_purge(), but do not lockdep on any lock for KUNIT test. + * + * Both @invs and @to_merge must be sorted, to ensure the returned array will be + * sorted as well. + * + * Caller is resposible for freeing the @invs and the returned new one. + * + * Entries marked as trash will be purged in the returned array. + */ +VISIBLE_IF_KUNIT +struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs, + struct arm_smmu_invs *to_merge) +{ + struct arm_smmu_invs *new_invs; + struct arm_smmu_inv *new; + size_t num_invs = 0; + size_t i, j; + int cmp; + + arm_smmu_invs_for_each_cmp(invs, i, to_merge, j, cmp) + num_invs++; + + new_invs = arm_smmu_invs_alloc(num_invs); + if (!new_invs) + return ERR_PTR(-ENOMEM); + + new = new_invs->inv; + arm_smmu_invs_for_each_cmp(invs, i, to_merge, j, cmp) { + if (cmp < 0) { + *new = invs->inv[i]; + } else if (cmp == 0) { + *new = invs->inv[i]; + WRITE_ONCE(new->users, READ_ONCE(new->users) + 1); + } else { + *new = to_merge->inv[j]; + WRITE_ONCE(new->users, 1); + } + + /* + * Check that the new array is sorted. This also validates that + * to_merge is sorted. + */ + if (new != new_invs->inv) + WARN_ON_ONCE(arm_smmu_inv_cmp(new - 1, new) == 1); + if (arm_smmu_inv_is_ats(new)) + new_invs->has_ats = true; + new++; + } + + WARN_ON(new != new_invs->inv + new_invs->num_invs); + + return new_invs; +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_merge); + +/** + * arm_smmu_invs_unref() - Find in @invs for all entries in @to_unref, decrease + * the user counts without deletions + * @invs: the base invalidation array + * @to_unref: an array of invalidations to decrease their user counts + * + * Return: the number of trash entries in the array, for arm_smmu_invs_purge() + * + * This function will not fail. Any entry with users=0 will be marked as trash, + * and caller will be notified about the trashed entry via @to_unref by setting + * a users=0. + * + * All tailing trash entries in the array will be dropped. And the size of the + * array will be trimmed properly. All trash entries in-between will remain in + * the @invs until being completely deleted by the next arm_smmu_invs_merge() + * or an arm_smmu_invs_purge() function call. + * + * This function must be locked and serialized with arm_smmu_invs_merge() and + * arm_smmu_invs_purge(), but do not lockdep on any mutex for KUNIT test. + * + * Note that the final @invs->num_invs might not reflect the actual number of + * invalidations due to trash entries. Any reader should take the read lock to + * iterate each entry and check its users counter till the last entry. + */ +VISIBLE_IF_KUNIT +void arm_smmu_invs_unref(struct arm_smmu_invs *invs, + struct arm_smmu_invs *to_unref) +{ + unsigned long flags; + size_t num_invs = 0; + size_t i, j; + int cmp; + + arm_smmu_invs_for_each_cmp(invs, i, to_unref, j, cmp) { + if (cmp < 0) { + /* not found in to_unref, leave alone */ + num_invs = i + 1; + } else if (cmp == 0) { + int users = READ_ONCE(invs->inv[i].users) - 1; + + if (WARN_ON(users < 0)) + continue; + + /* same item */ + WRITE_ONCE(invs->inv[i].users, users); + if (users) { + WRITE_ONCE(to_unref->inv[j].users, 1); + num_invs = i + 1; + continue; + } + + /* Notify the caller about the trash entry */ + WRITE_ONCE(to_unref->inv[j].users, 0); + invs->num_trashes++; + } else { + /* item in to_unref is not in invs or already a trash */ + WARN_ON(true); + } + } + + /* Exclude any tailing trash */ + invs->num_trashes -= invs->num_invs - num_invs; + + /* The lock is required to fence concurrent ATS operations. */ + write_lock_irqsave(&invs->rwlock, flags); + WRITE_ONCE(invs->num_invs, num_invs); /* Remove tailing trash entries */ + write_unlock_irqrestore(&invs->rwlock, flags); +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_unref); + +/** + * arm_smmu_invs_purge() - Purge all the trash entries in the @invs + * @invs: the base invalidation array + * + * Return: a newly allocated array on success removing all the trash entries, or + * NULL if there is no trash entry in the array or if allocation failed + * + * This function must be locked and serialized with arm_smmu_invs_merge() and + * arm_smmu_invs_unref(), but do not lockdep on any lock for KUNIT test. + * + * Caller is resposible for freeing the @invs and the returned new one. + */ +VISIBLE_IF_KUNIT +struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs) +{ + struct arm_smmu_invs *new_invs; + struct arm_smmu_inv *inv; + size_t i, num_invs = 0; + + if (WARN_ON(invs->num_invs < invs->num_trashes)) + return NULL; + if (!invs->num_invs || !invs->num_trashes) + return NULL; + + new_invs = arm_smmu_invs_alloc(invs->num_invs - invs->num_trashes); + if (!new_invs) + return NULL; + + arm_smmu_invs_for_each_entry(invs, i, inv) { + new_invs->inv[num_invs] = *inv; + if (arm_smmu_inv_is_ats(inv)) + new_invs->has_ats = true; + num_invs++; + } + + WARN_ON(num_invs != new_invs->num_invs); + return new_invs; +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_purge); + /* Context descriptor manipulation functions */ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) { @@ -2529,13 +2791,21 @@ static bool arm_smmu_enforce_cache_coherency(struct iommu_domain *domain) struct arm_smmu_domain *arm_smmu_domain_alloc(void) { struct arm_smmu_domain *smmu_domain; + struct arm_smmu_invs *new_invs; smmu_domain = kzalloc_obj(*smmu_domain); if (!smmu_domain) return ERR_PTR(-ENOMEM); + new_invs = arm_smmu_invs_alloc(0); + if (!new_invs) { + kfree(smmu_domain); + return ERR_PTR(-ENOMEM); + } + INIT_LIST_HEAD(&smmu_domain->devices); spin_lock_init(&smmu_domain->devices_lock); + rcu_assign_pointer(smmu_domain->invs, new_invs); return smmu_domain; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index cfbedb76c8ba..ed8820f12ba3 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -648,6 +648,93 @@ struct arm_smmu_cmdq_batch { int num; }; +/* + * The order here also determines the sequence in which commands are sent to the + * command queue. E.g. TLBI must be done before ATC_INV. + */ +enum arm_smmu_inv_type { + INV_TYPE_S1_ASID, + INV_TYPE_S2_VMID, + INV_TYPE_S2_VMID_S1_CLEAR, + INV_TYPE_ATS, + INV_TYPE_ATS_FULL, +}; + +struct arm_smmu_inv { + struct arm_smmu_device *smmu; + u8 type; + u8 size_opcode; + u8 nsize_opcode; + u32 id; /* ASID or VMID or SID */ + union { + size_t pgsize; /* ARM_SMMU_FEAT_RANGE_INV */ + u32 ssid; /* INV_TYPE_ATS */ + }; + + int users; /* users=0 to mark as a trash to be purged */ +}; + +static inline bool arm_smmu_inv_is_ats(const struct arm_smmu_inv *inv) +{ + return inv->type == INV_TYPE_ATS || inv->type == INV_TYPE_ATS_FULL; +} + +/** + * struct arm_smmu_invs - Per-domain invalidation array + * @max_invs: maximum capacity of the flexible array + * @num_invs: number of invalidations in the flexible array. May be smaller than + * @max_invs after a tailing trash entry is excluded, but must not be + * greater than @max_invs + * @num_trashes: number of trash entries in the array for arm_smmu_invs_purge(). + * Must not be greater than @num_invs + * @rwlock: optional rwlock to fench ATS operations + * @has_ats: flag if the array contains an INV_TYPE_ATS or INV_TYPE_ATS_FULL + * @rcu: rcu head for kfree_rcu() + * @inv: flexible invalidation array + * + * The arm_smmu_invs is an RCU data structure. During a ->attach_dev callback, + * arm_smmu_invs_merge(), arm_smmu_invs_unref() and arm_smmu_invs_purge() will + * be used to allocate a new copy of an old array for addition and deletion in + * the old domain's and new domain's invs arrays. + * + * The arm_smmu_invs_unref() mutates a given array, by internally reducing the + * users counts of some given entries. This exists to support a no-fail routine + * like attaching to an IOMMU_DOMAIN_BLOCKED. And it could pair with a followup + * arm_smmu_invs_purge() call to generate a new clean array. + * + * Concurrent invalidation thread will push every invalidation described in the + * array into the command queue for each invalidation event. It is designed like + * this to optimize the invalidation fast path by avoiding locks. + * + * A domain can be shared across SMMU instances. When an instance gets removed, + * it would delete all the entries that belong to that SMMU instance. Then, a + * synchronize_rcu() would have to be called to sync the array, to prevent any + * concurrent invalidation thread accessing the old array from issuing commands + * to the command queue of a removed SMMU instance. + */ +struct arm_smmu_invs { + size_t max_invs; + size_t num_invs; + size_t num_trashes; + rwlock_t rwlock; + bool has_ats; + struct rcu_head rcu; + struct arm_smmu_inv inv[] __counted_by(max_invs); +}; + +static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs) +{ + struct arm_smmu_invs *new_invs; + + new_invs = kzalloc(struct_size(new_invs, inv, num_invs), GFP_KERNEL); + if (!new_invs) + return NULL; + new_invs->max_invs = num_invs; + new_invs->num_invs = num_invs; + rwlock_init(&new_invs->rwlock); + return new_invs; +} + struct arm_smmu_evtq { struct arm_smmu_queue q; struct iopf_queue *iopf; @@ -873,6 +960,8 @@ struct arm_smmu_domain { struct iommu_domain domain; + struct arm_smmu_invs __rcu *invs; + /* List of struct arm_smmu_master_domain */ struct list_head devices; spinlock_t devices_lock; @@ -925,6 +1014,12 @@ void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, struct arm_smmu_master *master, struct mm_struct *mm, u16 asid); + +struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs, + struct arm_smmu_invs *to_merge); +void arm_smmu_invs_unref(struct arm_smmu_invs *invs, + struct arm_smmu_invs *to_unref); +struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs); #endif struct arm_smmu_master_domain { @@ -958,6 +1053,8 @@ struct arm_smmu_domain *arm_smmu_domain_alloc(void); static inline void arm_smmu_domain_free(struct arm_smmu_domain *smmu_domain) { + /* No concurrency with invalidation is possible at this point */ + kfree(rcu_dereference_protected(smmu_domain->invs, true)); kfree(smmu_domain); } From e3a56b37bf7546ecde4332d70a5bd092b9fe061b Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 17 Mar 2026 00:59:20 -0700 Subject: [PATCH 22/52] iommu/arm-smmu-v3: Pre-allocate a per-master invalidation array When a master is attached from an old domain to a new domain, it needs to build an invalidation array to delete and add the array entries from/onto the invalidation arrays of those two domains, passed via the to_merge and to_unref arguments into arm_smmu_invs_merge/unref() respectively. Since the master->num_streams might differ across masters, a memory would have to be allocated when building an to_merge/to_unref array which might fail with -ENOMEM. On the other hand, an attachment to arm_smmu_blocked_domain must not fail so it's the best to avoid any memory allocation in that path. Pre-allocate a fixed size invalidation array for every master. This array will be used as a scratch to fill dynamically when building a to_merge or to_unref invs array. Sort fwspec->ids in an ascending order to fit to the arm_smmu_invs_merge() function. Co-developed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe Reviewed-by: Jason Gunthorpe Reviewed-by: Pranjal Shrivastava Signed-off-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 41 +++++++++++++++++++-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 8 ++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 10050b1a6d94..8e651cf6b86e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3789,26 +3789,57 @@ static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid) return 0; } +static int arm_smmu_stream_id_cmp(const void *_l, const void *_r) +{ + const typeof_member(struct arm_smmu_stream, id) *l = _l; + const typeof_member(struct arm_smmu_stream, id) *r = _r; + + return cmp_int(*l, *r); +} + static int arm_smmu_insert_master(struct arm_smmu_device *smmu, struct arm_smmu_master *master) { int i; int ret = 0; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev); + bool ats_supported = dev_is_pci(master->dev) && + pci_ats_supported(to_pci_dev(master->dev)); master->streams = kzalloc_objs(*master->streams, fwspec->num_ids); if (!master->streams) return -ENOMEM; master->num_streams = fwspec->num_ids; + if (!ats_supported) { + /* Base case has 1 ASID entry or maximum 2 VMID entries */ + master->build_invs = arm_smmu_invs_alloc(2); + } else { + /* ATS case adds num_ids of entries, on top of the base case */ + master->build_invs = arm_smmu_invs_alloc(2 + fwspec->num_ids); + } + if (!master->build_invs) { + kfree(master->streams); + return -ENOMEM; + } + + for (i = 0; i < fwspec->num_ids; i++) { + struct arm_smmu_stream *new_stream = &master->streams[i]; + + new_stream->id = fwspec->ids[i]; + new_stream->master = master; + } + + /* Put the ids into order for sorted to_merge/to_unref arrays */ + sort_nonatomic(master->streams, master->num_streams, + sizeof(master->streams[0]), arm_smmu_stream_id_cmp, + NULL); + mutex_lock(&smmu->streams_mutex); for (i = 0; i < fwspec->num_ids; i++) { struct arm_smmu_stream *new_stream = &master->streams[i]; struct rb_node *existing; - u32 sid = fwspec->ids[i]; - - new_stream->id = sid; - new_stream->master = master; + u32 sid = new_stream->id; ret = arm_smmu_init_sid_strtab(smmu, sid); if (ret) @@ -3838,6 +3869,7 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu, for (i--; i >= 0; i--) rb_erase(&master->streams[i].node, &smmu->streams); kfree(master->streams); + kfree(master->build_invs); } mutex_unlock(&smmu->streams_mutex); @@ -3859,6 +3891,7 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master) mutex_unlock(&smmu->streams_mutex); kfree(master->streams); + kfree(master->build_invs); } static struct iommu_device *arm_smmu_probe_device(struct device *dev) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index ed8820f12ba3..5e0e5055af1e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -928,6 +928,14 @@ struct arm_smmu_master { struct arm_smmu_device *smmu; struct device *dev; struct arm_smmu_stream *streams; + /* + * Scratch memory for a to_merge or to_unref array to build a per-domain + * invalidation array. It'll be pre-allocated with enough enries for all + * possible build scenarios. It can be used by only one caller at a time + * until the arm_smmu_invs_merge/unref() finishes. Must be locked by the + * iommu_group mutex. + */ + struct arm_smmu_invs *build_invs; struct arm_smmu_vmaster *vmaster; /* use smmu->streams_mutex */ /* Locked by the iommu core using the group mutex */ struct arm_smmu_ctx_desc_cfg cd_table; From b77429757e4501e00f62cd4328bcfe6a9dbbf65e Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 17 Mar 2026 00:59:21 -0700 Subject: [PATCH 23/52] iommu/arm-smmu-v3: Populate smmu_domain->invs when attaching masters Update the invs array with the invalidations required by each domain type during attachment operations. Only an SVA domain or a paging domain will have an invs array: a. SVA domain will add an INV_TYPE_S1_ASID per SMMU and an INV_TYPE_ATS per SID b. Non-nesting-parent paging domain with no ATS-enabled master will add a single INV_TYPE_S1_ASID or INV_TYPE_S2_VMID per SMMU c. Non-nesting-parent paging domain with ATS-enabled master(s) will do (b) and add an INV_TYPE_ATS per SID d. Nesting-parent paging domain will add an INV_TYPE_S2_VMID followed by an INV_TYPE_S2_VMID_S1_CLEAR per vSMMU. For an ATS-enabled master, it will add an INV_TYPE_ATS_FULL per SID Note that case #d prepares for a future implementation of VMID allocation which requires a followup series for S2 domain sharing. So when a nesting parent domain is attached through a vSMMU instance using a nested domain. VMID will be allocated per vSMMU instance v.s. currectly per S2 domain. The per-domain invalidation is not needed until the domain is attached to a master (when it starts to possibly use TLB). This will make it possible to attach the domain to multiple SMMUs and avoid unnecessary invalidation overhead during teardown if no STEs/CDs refer to the domain. It also means that when the last device is detached, the old domain must flush its ASID or VMID, since any new iommu_unmap() call would not trigger invalidations given an empty domain->invs array. Introduce some arm_smmu_invs helper functions for building scratch arrays, preparing and installing old/new domain's invalidation arrays. Co-developed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe Reviewed-by: Jason Gunthorpe Signed-off-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 262 +++++++++++++++++++- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 17 ++ 2 files changed, 278 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 8e651cf6b86e..1d0f96813864 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3147,6 +3147,121 @@ static void arm_smmu_disable_iopf(struct arm_smmu_master *master, iopf_queue_remove_device(master->smmu->evtq.iopf, master->dev); } +static struct arm_smmu_inv * +arm_smmu_master_build_inv(struct arm_smmu_master *master, + enum arm_smmu_inv_type type, u32 id, ioasid_t ssid, + size_t pgsize) +{ + struct arm_smmu_invs *build_invs = master->build_invs; + struct arm_smmu_inv *cur, inv = { + .smmu = master->smmu, + .type = type, + .id = id, + .pgsize = pgsize, + }; + + if (WARN_ON(build_invs->num_invs >= build_invs->max_invs)) + return NULL; + cur = &build_invs->inv[build_invs->num_invs]; + build_invs->num_invs++; + + *cur = inv; + switch (type) { + case INV_TYPE_S1_ASID: + /* + * For S1 page tables the driver always uses VMID=0, and the + * invalidation logic for this type will set it as well. + */ + if (master->smmu->features & ARM_SMMU_FEAT_E2H) { + cur->size_opcode = CMDQ_OP_TLBI_EL2_VA; + cur->nsize_opcode = CMDQ_OP_TLBI_EL2_ASID; + } else { + cur->size_opcode = CMDQ_OP_TLBI_NH_VA; + cur->nsize_opcode = CMDQ_OP_TLBI_NH_ASID; + } + break; + case INV_TYPE_S2_VMID: + cur->size_opcode = CMDQ_OP_TLBI_S2_IPA; + cur->nsize_opcode = CMDQ_OP_TLBI_S12_VMALL; + break; + case INV_TYPE_S2_VMID_S1_CLEAR: + cur->size_opcode = cur->nsize_opcode = CMDQ_OP_TLBI_NH_ALL; + break; + case INV_TYPE_ATS: + case INV_TYPE_ATS_FULL: + cur->size_opcode = cur->nsize_opcode = CMDQ_OP_ATC_INV; + cur->ssid = ssid; + break; + } + + return cur; +} + +/* + * Use the preallocated scratch array at master->build_invs, to build a to_merge + * or to_unref array, to pass into a following arm_smmu_invs_merge/unref() call. + * + * Do not free the returned invs array. It is reused, and will be overwritten by + * the next arm_smmu_master_build_invs() call. + */ +static struct arm_smmu_invs * +arm_smmu_master_build_invs(struct arm_smmu_master *master, bool ats_enabled, + ioasid_t ssid, struct arm_smmu_domain *smmu_domain) +{ + const bool nesting = smmu_domain->nest_parent; + size_t pgsize = 0, i; + + iommu_group_mutex_assert(master->dev); + + master->build_invs->num_invs = 0; + + /* Range-based invalidation requires the leaf pgsize for calculation */ + if (master->smmu->features & ARM_SMMU_FEAT_RANGE_INV) + pgsize = __ffs(smmu_domain->domain.pgsize_bitmap); + + switch (smmu_domain->stage) { + case ARM_SMMU_DOMAIN_SVA: + case ARM_SMMU_DOMAIN_S1: + if (!arm_smmu_master_build_inv(master, INV_TYPE_S1_ASID, + smmu_domain->cd.asid, + IOMMU_NO_PASID, pgsize)) + return NULL; + break; + case ARM_SMMU_DOMAIN_S2: + if (!arm_smmu_master_build_inv(master, INV_TYPE_S2_VMID, + smmu_domain->s2_cfg.vmid, + IOMMU_NO_PASID, pgsize)) + return NULL; + break; + default: + WARN_ON(true); + return NULL; + } + + /* All the nested S1 ASIDs have to be flushed when S2 parent changes */ + if (nesting) { + if (!arm_smmu_master_build_inv( + master, INV_TYPE_S2_VMID_S1_CLEAR, + smmu_domain->s2_cfg.vmid, IOMMU_NO_PASID, 0)) + return NULL; + } + + for (i = 0; ats_enabled && i < master->num_streams; i++) { + /* + * If an S2 used as a nesting parent is changed we have no + * option but to completely flush the ATC. + */ + if (!arm_smmu_master_build_inv( + master, nesting ? INV_TYPE_ATS_FULL : INV_TYPE_ATS, + master->streams[i].id, ssid, 0)) + return NULL; + } + + /* Note this build_invs must have been sorted */ + + return master->build_invs; +} + static void arm_smmu_remove_master_domain(struct arm_smmu_master *master, struct iommu_domain *domain, ioasid_t ssid) @@ -3176,6 +3291,135 @@ static void arm_smmu_remove_master_domain(struct arm_smmu_master *master, kfree(master_domain); } +/* + * During attachment, the updates of the two domain->invs arrays are sequenced: + * 1. new domain updates its invs array, merging master->build_invs + * 2. new domain starts to include the master during its invalidation + * 3. master updates its STE switching from the old domain to the new domain + * 4. old domain still includes the master during its invalidation + * 5. old domain updates its invs array, unreferencing master->build_invs + * + * For 1 and 5, prepare the two updated arrays in advance, handling any changes + * that can possibly failure. So the actual update of either 1 or 5 won't fail. + * arm_smmu_asid_lock ensures that the old invs in the domains are intact while + * we are sequencing to update them. + */ +static int arm_smmu_attach_prepare_invs(struct arm_smmu_attach_state *state, + struct iommu_domain *new_domain) +{ + struct arm_smmu_domain *old_smmu_domain = + to_smmu_domain_devices(state->old_domain); + struct arm_smmu_domain *new_smmu_domain = + to_smmu_domain_devices(new_domain); + struct arm_smmu_master *master = state->master; + ioasid_t ssid = state->ssid; + + /* + * At this point a NULL domain indicates the domain doesn't use the + * IOTLB, see to_smmu_domain_devices(). + */ + if (new_smmu_domain) { + struct arm_smmu_inv_state *invst = &state->new_domain_invst; + struct arm_smmu_invs *build_invs; + + invst->invs_ptr = &new_smmu_domain->invs; + invst->old_invs = rcu_dereference_protected( + new_smmu_domain->invs, + lockdep_is_held(&arm_smmu_asid_lock)); + build_invs = arm_smmu_master_build_invs( + master, state->ats_enabled, ssid, new_smmu_domain); + if (!build_invs) + return -EINVAL; + + invst->new_invs = + arm_smmu_invs_merge(invst->old_invs, build_invs); + if (IS_ERR(invst->new_invs)) + return PTR_ERR(invst->new_invs); + } + + if (old_smmu_domain) { + struct arm_smmu_inv_state *invst = &state->old_domain_invst; + + invst->invs_ptr = &old_smmu_domain->invs; + /* A re-attach case might have a different ats_enabled state */ + if (new_smmu_domain == old_smmu_domain) + invst->old_invs = state->new_domain_invst.new_invs; + else + invst->old_invs = rcu_dereference_protected( + old_smmu_domain->invs, + lockdep_is_held(&arm_smmu_asid_lock)); + /* For old_smmu_domain, new_invs points to master->build_invs */ + invst->new_invs = arm_smmu_master_build_invs( + master, master->ats_enabled, ssid, old_smmu_domain); + } + + return 0; +} + +/* Must be installed before arm_smmu_install_ste_for_dev() */ +static void +arm_smmu_install_new_domain_invs(struct arm_smmu_attach_state *state) +{ + struct arm_smmu_inv_state *invst = &state->new_domain_invst; + + if (!invst->invs_ptr) + return; + + rcu_assign_pointer(*invst->invs_ptr, invst->new_invs); + kfree_rcu(invst->old_invs, rcu); +} + +static void arm_smmu_inv_flush_iotlb_tag(struct arm_smmu_inv *inv) +{ + struct arm_smmu_cmdq_ent cmd = {}; + + switch (inv->type) { + case INV_TYPE_S1_ASID: + cmd.tlbi.asid = inv->id; + break; + case INV_TYPE_S2_VMID: + /* S2_VMID using nsize_opcode covers S2_VMID_S1_CLEAR */ + cmd.tlbi.vmid = inv->id; + break; + default: + return; + } + + cmd.opcode = inv->nsize_opcode; + arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, &cmd); +} + +/* Should be installed after arm_smmu_install_ste_for_dev() */ +static void +arm_smmu_install_old_domain_invs(struct arm_smmu_attach_state *state) +{ + struct arm_smmu_inv_state *invst = &state->old_domain_invst; + struct arm_smmu_invs *old_invs = invst->old_invs; + struct arm_smmu_invs *new_invs; + + lockdep_assert_held(&arm_smmu_asid_lock); + + if (!invst->invs_ptr) + return; + + arm_smmu_invs_unref(old_invs, invst->new_invs); + /* + * When an IOTLB tag (the first entry in invs->new_invs) is no longer used, + * it means the ASID or VMID will no longer be invalidated by map/unmap and + * must be cleaned right now. The rule is that any ASID/VMID not in an invs + * array must be left cleared in the IOTLB. + */ + if (!READ_ONCE(invst->new_invs->inv[0].users)) + arm_smmu_inv_flush_iotlb_tag(&invst->new_invs->inv[0]); + + new_invs = arm_smmu_invs_purge(old_invs); + if (!new_invs) + return; + + rcu_assign_pointer(*invst->invs_ptr, new_invs); + kfree_rcu(old_invs, rcu); +} + /* * Start the sequence to attach a domain to a master. The sequence contains three * steps: @@ -3233,12 +3477,16 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state, arm_smmu_ats_supported(master); } + ret = arm_smmu_attach_prepare_invs(state, new_domain); + if (ret) + return ret; + if (smmu_domain) { if (new_domain->type == IOMMU_DOMAIN_NESTED) { ret = arm_smmu_attach_prepare_vmaster( state, to_smmu_nested_domain(new_domain)); if (ret) - return ret; + goto err_unprepare_invs; } master_domain = kzalloc_obj(*master_domain); @@ -3286,6 +3534,8 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state, atomic_inc(&smmu_domain->nr_ats_masters); list_add(&master_domain->devices_elm, &smmu_domain->devices); spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); + + arm_smmu_install_new_domain_invs(state); } if (!state->ats_enabled && master->ats_enabled) { @@ -3305,6 +3555,8 @@ int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state, kfree(master_domain); err_free_vmaster: kfree(state->vmaster); +err_unprepare_invs: + kfree(state->new_domain_invst.new_invs); return ret; } @@ -3336,6 +3588,7 @@ void arm_smmu_attach_commit(struct arm_smmu_attach_state *state) } arm_smmu_remove_master_domain(master, state->old_domain, state->ssid); + arm_smmu_install_old_domain_invs(state); master->ats_enabled = state->ats_enabled; } @@ -3518,12 +3771,19 @@ static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain, { struct arm_smmu_domain *smmu_domain = to_smmu_domain(old_domain); struct arm_smmu_master *master = dev_iommu_priv_get(dev); + struct arm_smmu_attach_state state = { + .master = master, + .old_domain = old_domain, + .ssid = pasid, + }; mutex_lock(&arm_smmu_asid_lock); + arm_smmu_attach_prepare_invs(&state, NULL); arm_smmu_clear_cd(master, pasid); if (master->ats_enabled) arm_smmu_atc_inv_master(master, pasid); arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid); + arm_smmu_install_old_domain_invs(&state); mutex_unlock(&arm_smmu_asid_lock); /* diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 5e0e5055af1e..83d7e4952dff 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -1102,6 +1102,21 @@ static inline bool arm_smmu_master_canwbs(struct arm_smmu_master *master) IOMMU_FWSPEC_PCI_RC_CANWBS; } +/** + * struct arm_smmu_inv_state - Per-domain invalidation array state + * @invs_ptr: points to the domain->invs (unwinding nesting/etc.) or is NULL if + * no change should be made + * @old_invs: the original invs array + * @new_invs: for new domain, this is the new invs array to update domain->invs; + * for old domain, this is the master->build_invs to pass in as the + * to_unref argument to an arm_smmu_invs_unref() call + */ +struct arm_smmu_inv_state { + struct arm_smmu_invs __rcu **invs_ptr; + struct arm_smmu_invs *old_invs; + struct arm_smmu_invs *new_invs; +}; + struct arm_smmu_attach_state { /* Inputs */ struct iommu_domain *old_domain; @@ -1111,6 +1126,8 @@ struct arm_smmu_attach_state { ioasid_t ssid; /* Resulting state */ struct arm_smmu_vmaster *vmaster; + struct arm_smmu_inv_state old_domain_invst; + struct arm_smmu_inv_state new_domain_invst; bool ats_enabled; }; From 587bb3e56a2c37bbd58efff24e56fe7dae472199 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 17 Mar 2026 00:59:22 -0700 Subject: [PATCH 24/52] iommu/arm-smmu-v3: Add arm_smmu_invs based arm_smmu_domain_inv_range() Each smmu_domain now has an arm_smmu_invs that specifies the invalidation steps to perform after any change the IOPTEs. This includes supports for basic ASID/VMID, the special case for nesting, and ATC invalidations. Introduce a new arm_smmu_domain_inv helper iterating smmu_domain->invs to convert the invalidation array to commands. Any invalidation request with no size specified means an entire flush over a range based one. Take advantage of the sorted array to compatible batch operations together to the same SMMU. For instance, ATC invaliations for multiple SIDs can be pushed as a batch. ATC invalidations must be completed before the driver disables ATS. Or the device is permitted to ignore any racing invalidation that would cause an SMMU timeout. The sequencing is done with a rwlock where holding the write side of the rwlock means that there are no outstanding ATC invalidations. If ATS is not used the rwlock is ignored, similar to the existing code. Co-developed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe Reviewed-by: Jason Gunthorpe Signed-off-by: Nicolin Chen Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 225 ++++++++++++++++++-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 9 + 2 files changed, 221 insertions(+), 13 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 1d0f96813864..19e47c614ba7 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2595,23 +2595,19 @@ static void arm_smmu_tlb_inv_context(void *cookie) arm_smmu_atc_inv_domain(smmu_domain, 0, 0); } -static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd, - unsigned long iova, size_t size, - size_t granule, - struct arm_smmu_domain *smmu_domain) +static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, + struct arm_smmu_cmdq_batch *cmds, + struct arm_smmu_cmdq_ent *cmd, + unsigned long iova, size_t size, + size_t granule, size_t pgsize) { - struct arm_smmu_device *smmu = smmu_domain->smmu; - unsigned long end = iova + size, num_pages = 0, tg = 0; + unsigned long end = iova + size, num_pages = 0, tg = pgsize; size_t inv_range = granule; - struct arm_smmu_cmdq_batch cmds; if (!size) return; if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { - /* Get the leaf page size */ - tg = __ffs(smmu_domain->domain.pgsize_bitmap); - num_pages = size >> tg; /* Convert page size of 12,14,16 (log2) to 1,2,3 */ @@ -2631,8 +2627,6 @@ static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd, num_pages++; } - arm_smmu_cmdq_batch_init(smmu, &cmds, cmd); - while (iova < end) { if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { /* @@ -2660,9 +2654,26 @@ static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd, } cmd->tlbi.addr = iova; - arm_smmu_cmdq_batch_add(smmu, &cmds, cmd); + arm_smmu_cmdq_batch_add(smmu, cmds, cmd); iova += inv_range; } +} + +static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd, + unsigned long iova, size_t size, + size_t granule, + struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_cmdq_batch cmds; + size_t pgsize; + + /* Get the leaf page size */ + pgsize = __ffs(smmu_domain->domain.pgsize_bitmap); + + arm_smmu_cmdq_batch_init(smmu, &cmds, cmd); + arm_smmu_cmdq_batch_add_range(smmu, &cmds, cmd, iova, size, granule, + pgsize); arm_smmu_cmdq_batch_submit(smmu, &cmds); } @@ -2718,6 +2729,194 @@ void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain); } +static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size, + size_t granule) +{ + size_t max_tlbi_ops; + + /* 0 size means invalidate all */ + if (!size || size == SIZE_MAX) + return true; + + if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) + return false; + + /* + * Borrowed from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, + * this is used as a threshold to replace "size_opcode" commands with a + * single "nsize_opcode" command, when SMMU doesn't implement the range + * invalidation feature, where there can be too many per-granule TLBIs, + * resulting in a soft lockup. + */ + max_tlbi_ops = 1 << (ilog2(granule) - 3); + return size >= max_tlbi_ops * granule; +} + +/* Used by non INV_TYPE_ATS* invalidations */ +static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv, + struct arm_smmu_cmdq_batch *cmds, + struct arm_smmu_cmdq_ent *cmd, + unsigned long iova, size_t size, + unsigned int granule) +{ + if (arm_smmu_inv_size_too_big(inv->smmu, size, granule)) { + cmd->opcode = inv->nsize_opcode; + arm_smmu_cmdq_batch_add(inv->smmu, cmds, cmd); + return; + } + + cmd->opcode = inv->size_opcode; + arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, iova, size, granule, + inv->pgsize); +} + +static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, + struct arm_smmu_inv *next) +{ + /* Changing smmu means changing command queue */ + if (cur->smmu != next->smmu) + return true; + /* The batch for S2 TLBI must be done before nested S1 ASIDs */ + if (cur->type != INV_TYPE_S2_VMID_S1_CLEAR && + next->type == INV_TYPE_S2_VMID_S1_CLEAR) + return true; + /* ATS must be after a sync of the S1/S2 invalidations */ + if (!arm_smmu_inv_is_ats(cur) && arm_smmu_inv_is_ats(next)) + return true; + return false; +} + +static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, + unsigned long iova, size_t size, + unsigned int granule, bool leaf) +{ + struct arm_smmu_cmdq_batch cmds = {}; + struct arm_smmu_inv *cur; + struct arm_smmu_inv *end; + + cur = invs->inv; + end = cur + READ_ONCE(invs->num_invs); + /* Skip any leading entry marked as a trash */ + for (; cur != end; cur++) + if (READ_ONCE(cur->users)) + break; + while (cur != end) { + struct arm_smmu_device *smmu = cur->smmu; + struct arm_smmu_cmdq_ent cmd = { + /* + * Pick size_opcode to run arm_smmu_get_cmdq(). This can + * be changed to nsize_opcode, which would result in the + * same CMDQ pointer. + */ + .opcode = cur->size_opcode, + }; + struct arm_smmu_inv *next; + + if (!cmds.num) + arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd); + + switch (cur->type) { + case INV_TYPE_S1_ASID: + cmd.tlbi.asid = cur->id; + cmd.tlbi.leaf = leaf; + arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, iova, size, + granule); + break; + case INV_TYPE_S2_VMID: + cmd.tlbi.vmid = cur->id; + cmd.tlbi.leaf = leaf; + arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, iova, size, + granule); + break; + case INV_TYPE_S2_VMID_S1_CLEAR: + /* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */ + if (arm_smmu_inv_size_too_big(cur->smmu, size, granule)) + continue; + cmd.tlbi.vmid = cur->id; + arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); + break; + case INV_TYPE_ATS: + arm_smmu_atc_inv_to_cmd(cur->ssid, iova, size, &cmd); + cmd.atc.sid = cur->id; + arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); + break; + case INV_TYPE_ATS_FULL: + arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd); + cmd.atc.sid = cur->id; + arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); + break; + default: + WARN_ON_ONCE(1); + continue; + } + + /* Skip any trash entry in-between */ + for (next = cur + 1; next != end; next++) + if (READ_ONCE(next->users)) + break; + + if (cmds.num && + (next == end || arm_smmu_invs_end_batch(cur, next))) { + arm_smmu_cmdq_batch_submit(smmu, &cmds); + cmds.num = 0; + } + cur = next; + } +} + +void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, + unsigned long iova, size_t size, + unsigned int granule, bool leaf) +{ + struct arm_smmu_invs *invs; + + /* + * An invalidation request must follow some IOPTE change and then load + * an invalidation array. In the meantime, a domain attachment mutates + * the array and then stores an STE/CD asking SMMU HW to acquire those + * changed IOPTEs. + * + * When running alone, a domain attachment relies on the dma_wmb() in + * arm_smmu_write_entry() used by arm_smmu_install_ste_for_dev(). + * + * But in a race, these two can be interdependent, making it a special + * case requiring an additional smp_mb() for the write->read ordering. + * Pairing with the dma_wmb() in arm_smmu_install_ste_for_dev(), this + * makes sure that IOPTE update prior to this point is visable to SMMU + * hardware before we load the updated invalidation array. + * + * [CPU0] | [CPU1] + * change IOPTE on new domain: | + * arm_smmu_domain_inv_range() { | arm_smmu_install_new_domain_invs() + * smp_mb(); // ensures IOPTE | arm_smmu_install_ste_for_dev { + * // seen by SMMU | dma_wmb(); // ensures invs update + * // load the updated invs | // before updating STE + * invs = rcu_dereference(); | STE = TTB0; + * ... | ... + * } | } + */ + smp_mb(); + + rcu_read_lock(); + invs = rcu_dereference(smmu_domain->invs); + + /* + * Avoid locking unless ATS is being used. No ATC invalidation can be + * going on after a domain is detached. + */ + if (invs->has_ats) { + unsigned long flags; + + read_lock_irqsave(&invs->rwlock, flags); + __arm_smmu_domain_inv_range(invs, iova, size, granule, leaf); + read_unlock_irqrestore(&invs->rwlock, flags); + } else { + __arm_smmu_domain_inv_range(invs, iova, size, granule, leaf); + } + + rcu_read_unlock(); +} + static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather, unsigned long iova, size_t granule, void *cookie) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 83d7e4952dff..534e9a5ddca3 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -1087,6 +1087,15 @@ void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, unsigned long iova, size_t size); +void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, + unsigned long iova, size_t size, + unsigned int granule, bool leaf); + +static inline void arm_smmu_domain_inv(struct arm_smmu_domain *smmu_domain) +{ + arm_smmu_domain_inv_range(smmu_domain, 0, 0, 0, false); +} + void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, struct arm_smmu_cmdq *cmdq); int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, From 4202fddd01c74fedc301ca2058623e28b8211dc1 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Tue, 17 Mar 2026 00:59:23 -0700 Subject: [PATCH 25/52] iommu/arm-smmu-v3: Perform per-domain invalidations using arm_smmu_invs Replace the old invalidation functions with arm_smmu_domain_inv_range() in all the existing invalidation routines. And deprecate the old functions. The new arm_smmu_domain_inv_range() handles the CMDQ_MAX_TLBI_OPS as well, so drop it in the SVA function. Since arm_smmu_cmdq_batch_add_range() has only one caller now, and it must be given a valid size, add a WARN_ON_ONCE to catch any missed case. Also update the comments in arm_smmu_tlb_inv_context() to clarify things with the new invalidation functions. Reviewed-by: Jason Gunthorpe Signed-off-by: Nicolin Chen Signed-off-by: Will Deacon --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 29 +-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 183 ++---------------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 7 - 3 files changed, 24 insertions(+), 195 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 440ad8cc07de..f1f8e01a7e91 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -122,15 +122,6 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, } EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd); -/* - * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this - * is used as a threshold to replace per-page TLBI commands to issue in the - * command queue with an address-space TLBI command, when SMMU w/o a range - * invalidation feature handles too many per-page TLBI commands, which will - * otherwise result in a soft lockup. - */ -#define CMDQ_MAX_TLBI_OPS (1 << (PAGE_SHIFT - 3)) - static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, struct mm_struct *mm, unsigned long start, @@ -146,21 +137,8 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, * range. So do a simple translation here by calculating size correctly. */ size = end - start; - if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) { - if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE) - size = 0; - } else { - if (size == ULONG_MAX) - size = 0; - } - if (!size) - arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); - else - arm_smmu_tlb_inv_range_asid(start, size, smmu_domain->cd.asid, - PAGE_SIZE, false, smmu_domain); - - arm_smmu_atc_inv_domain(smmu_domain, start, size); + arm_smmu_domain_inv_range(smmu_domain, start, size, PAGE_SIZE, false); } static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) @@ -191,8 +169,7 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) } spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); - arm_smmu_atc_inv_domain(smmu_domain, 0, 0); + arm_smmu_domain_inv(smmu_domain); } static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) @@ -302,7 +279,7 @@ static void arm_smmu_sva_domain_free(struct iommu_domain *domain) /* * Ensure the ASID is empty in the iommu cache before allowing reuse. */ - arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); + arm_smmu_domain_inv(smmu_domain); /* * Notice that the arm_smmu_mm_arch_invalidate_secondary_tlbs op can diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 19e47c614ba7..01030ffd2fe2 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1289,16 +1289,6 @@ struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs) EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_purge); /* Context descriptor manipulation functions */ -void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) -{ - struct arm_smmu_cmdq_ent cmd = { - .opcode = smmu->features & ARM_SMMU_FEAT_E2H ? - CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID, - .tlbi.asid = asid, - }; - - arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); -} /* * Based on the value of ent report which bits of the STE the HW will access. It @@ -2509,90 +2499,27 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master, return arm_smmu_cmdq_batch_submit(master->smmu, &cmds); } -int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, - unsigned long iova, size_t size) -{ - struct arm_smmu_master_domain *master_domain; - int i; - unsigned long flags; - struct arm_smmu_cmdq_ent cmd = { - .opcode = CMDQ_OP_ATC_INV, - }; - struct arm_smmu_cmdq_batch cmds; - - if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS)) - return 0; - - /* - * Ensure that we've completed prior invalidation of the main TLBs - * before we read 'nr_ats_masters' in case of a concurrent call to - * arm_smmu_enable_ats(): - * - * // unmap() // arm_smmu_enable_ats() - * TLBI+SYNC atomic_inc(&nr_ats_masters); - * smp_mb(); [...] - * atomic_read(&nr_ats_masters); pci_enable_ats() // writel() - * - * Ensures that we always see the incremented 'nr_ats_masters' count if - * ATS was enabled at the PCI device before completion of the TLBI. - */ - smp_mb(); - if (!atomic_read(&smmu_domain->nr_ats_masters)) - return 0; - - arm_smmu_cmdq_batch_init(smmu_domain->smmu, &cmds, &cmd); - - spin_lock_irqsave(&smmu_domain->devices_lock, flags); - list_for_each_entry(master_domain, &smmu_domain->devices, - devices_elm) { - struct arm_smmu_master *master = master_domain->master; - - if (!master->ats_enabled) - continue; - - if (master_domain->nested_ats_flush) { - /* - * If a S2 used as a nesting parent is changed we have - * no option but to completely flush the ATC. - */ - arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd); - } else { - arm_smmu_atc_inv_to_cmd(master_domain->ssid, iova, size, - &cmd); - } - - for (i = 0; i < master->num_streams; i++) { - cmd.atc.sid = master->streams[i].id; - arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd); - } - } - spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); - - return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds); -} - /* IO_PGTABLE API */ static void arm_smmu_tlb_inv_context(void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct arm_smmu_cmdq_ent cmd; /* - * NOTE: when io-pgtable is in non-strict mode, we may get here with - * PTEs previously cleared by unmaps on the current CPU not yet visible - * to the SMMU. We are relying on the dma_wmb() implicit during cmd - * insertion to guarantee those are observed before the TLBI. Do be - * careful, 007. + * If the DMA API is running in non-strict mode then another CPU could + * have changed the page table and not invoked any flush op. Instead the + * other CPU will do an atomic_read() and this CPU will have done an + * atomic_write(). That handshake is enough to acquire the page table + * writes from the other CPU. + * + * All command execution has a dma_wmb() to release all the in-memory + * structures written by this CPU, that barrier must also release the + * writes acquired from all the other CPUs too. + * + * There are other barriers and atomics on this path, but the above is + * the essential mechanism for ensuring that HW sees the page table + * writes from another CPU before it executes the IOTLB invalidation. */ - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { - arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid); - } else { - cmd.opcode = CMDQ_OP_TLBI_S12_VMALL; - cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; - arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); - } - arm_smmu_atc_inv_domain(smmu_domain, 0, 0); + arm_smmu_domain_inv(smmu_domain); } static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, @@ -2604,7 +2531,7 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, unsigned long end = iova + size, num_pages = 0, tg = pgsize; size_t inv_range = granule; - if (!size) + if (WARN_ON_ONCE(!size)) return; if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { @@ -2659,76 +2586,6 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, } } -static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd, - unsigned long iova, size_t size, - size_t granule, - struct arm_smmu_domain *smmu_domain) -{ - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct arm_smmu_cmdq_batch cmds; - size_t pgsize; - - /* Get the leaf page size */ - pgsize = __ffs(smmu_domain->domain.pgsize_bitmap); - - arm_smmu_cmdq_batch_init(smmu, &cmds, cmd); - arm_smmu_cmdq_batch_add_range(smmu, &cmds, cmd, iova, size, granule, - pgsize); - arm_smmu_cmdq_batch_submit(smmu, &cmds); -} - -static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size, - size_t granule, bool leaf, - struct arm_smmu_domain *smmu_domain) -{ - struct arm_smmu_cmdq_ent cmd = { - .tlbi = { - .leaf = leaf, - }, - }; - - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { - cmd.opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ? - CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA; - cmd.tlbi.asid = smmu_domain->cd.asid; - } else { - cmd.opcode = CMDQ_OP_TLBI_S2_IPA; - cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; - } - __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain); - - if (smmu_domain->nest_parent) { - /* - * When the S2 domain changes all the nested S1 ASIDs have to be - * flushed too. - */ - cmd.opcode = CMDQ_OP_TLBI_NH_ALL; - arm_smmu_cmdq_issue_cmd_with_sync(smmu_domain->smmu, &cmd); - } - - /* - * Unfortunately, this can't be leaf-only since we may have - * zapped an entire table. - */ - arm_smmu_atc_inv_domain(smmu_domain, iova, size); -} - -void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, - size_t granule, bool leaf, - struct arm_smmu_domain *smmu_domain) -{ - struct arm_smmu_cmdq_ent cmd = { - .opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ? - CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA, - .tlbi = { - .asid = asid, - .leaf = leaf, - }, - }; - - __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain); -} - static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size, size_t granule) { @@ -2930,7 +2787,9 @@ static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather, static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size, size_t granule, void *cookie) { - arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie); + struct arm_smmu_domain *smmu_domain = cookie; + + arm_smmu_domain_inv_range(smmu_domain, iova, size, granule, false); } static const struct iommu_flush_ops arm_smmu_flush_ops = { @@ -4201,9 +4060,9 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain, if (!gather->pgsize) return; - arm_smmu_tlb_inv_range_domain(gather->start, - gather->end - gather->start + 1, - gather->pgsize, true, smmu_domain); + arm_smmu_domain_inv_range(smmu_domain, gather->start, + gather->end - gather->start + 1, + gather->pgsize, true); } static phys_addr_t diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 534e9a5ddca3..36de2b0b2ebe 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -1080,13 +1080,6 @@ int arm_smmu_set_pasid(struct arm_smmu_master *master, struct arm_smmu_domain *smmu_domain, ioasid_t pasid, struct arm_smmu_cd *cd, struct iommu_domain *old); -void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid); -void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, - size_t granule, bool leaf, - struct arm_smmu_domain *smmu_domain); -int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, - unsigned long iova, size_t size); - void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, unsigned long iova, size_t size, unsigned int granule, bool leaf); From 6a01b9f0a5ec38112db54370ce7794db2be5a5de Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Sat, 21 Mar 2026 15:50:40 -0700 Subject: [PATCH 26/52] iommu/arm-smmu-v3: Do not continue in __arm_smmu_domain_inv_range() The loop in the __arm_smmu_domain_inv_range() is a while loop, not a for loop. So, using "continue" is wrong that would fail to move the needle. Meanwhile, though the current command is skipped, the batch still has to go through arm_smmu_invs_end_batch() to be issued accordingly. Thus, use "break" to fix the issue. Fixes: 587bb3e56a2c ("iommu/arm-smmu-v3: Add arm_smmu_invs based arm_smmu_domain_inv_range()") Signed-off-by: Nicolin Chen Reviewed-by: Jason Gunthorpe Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 01030ffd2fe2..c3c6987da950 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2688,7 +2688,7 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, case INV_TYPE_S2_VMID_S1_CLEAR: /* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */ if (arm_smmu_inv_size_too_big(cur->smmu, size, granule)) - continue; + break; cmd.tlbi.vmid = cur->id; arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); break; @@ -2704,7 +2704,7 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, break; default: WARN_ON_ONCE(1); - continue; + break; } /* Skip any trash entry in-between */ From 86bf8580d5b873d165350f61441d0649e4c232f4 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Sat, 21 Mar 2026 15:50:41 -0700 Subject: [PATCH 27/52] iommu/arm-smmu-v3: Fix typos introduced by arm_smmu_invs These are introduced by separate commits, so not submitting with a "Fixes" line, since they aren't critical. Signed-off-by: Nicolin Chen Reviewed-by: Jason Gunthorpe Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 6 +++--- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index c3c6987da950..b841efbcc9e9 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1128,7 +1128,7 @@ static inline int arm_smmu_invs_iter_next_cmp(struct arm_smmu_invs *invs_l, * Both @invs and @to_merge must be sorted, to ensure the returned array will be * sorted as well. * - * Caller is resposible for freeing the @invs and the returned new one. + * Caller is responsible for freeing the @invs and the returned new one. * * Entries marked as trash will be purged in the returned array. */ @@ -1258,7 +1258,7 @@ EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_unref); * This function must be locked and serialized with arm_smmu_invs_merge() and * arm_smmu_invs_unref(), but do not lockdep on any lock for KUNIT test. * - * Caller is resposible for freeing the @invs and the returned new one. + * Caller is responsible for freeing the @invs and the returned new one. */ VISIBLE_IF_KUNIT struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs) @@ -2739,7 +2739,7 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, * But in a race, these two can be interdependent, making it a special * case requiring an additional smp_mb() for the write->read ordering. * Pairing with the dma_wmb() in arm_smmu_install_ste_for_dev(), this - * makes sure that IOPTE update prior to this point is visable to SMMU + * makes sure that IOPTE update prior to this point is visible to SMMU * hardware before we load the updated invalidation array. * * [CPU0] | [CPU1] diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 36de2b0b2ebe..ef42df4753ec 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -687,7 +687,7 @@ static inline bool arm_smmu_inv_is_ats(const struct arm_smmu_inv *inv) * greater than @max_invs * @num_trashes: number of trash entries in the array for arm_smmu_invs_purge(). * Must not be greater than @num_invs - * @rwlock: optional rwlock to fench ATS operations + * @rwlock: optional rwlock to fence ATS operations * @has_ats: flag if the array contains an INV_TYPE_ATS or INV_TYPE_ATS_FULL * @rcu: rcu head for kfree_rcu() * @inv: flexible invalidation array From 3b793983834e9484a834912548f4786b742abc92 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 19 Mar 2026 15:24:34 +0000 Subject: [PATCH 28/52] iommu/arm-smmu-v3: Update Arm errata MMU-700 r1p1 has subsequently fixed some of the errata for which we've been applying the workarounds unconditionally, so we can now make those conditional. However, there have also been some more new cases identified where we must rely on range invalidation commands, and thus still nominally avoid DVM being inadvertently enabled. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- Documentation/arch/arm64/silicon-errata.rst | 8 +++++++- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 4c300caad901..c81d7fc2c68c 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -207,8 +207,14 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-600 | #1076982,1209401| N/A | +----------------+-----------------+-----------------+-----------------------------+ -| ARM | MMU-700 | #2268618,2812531| N/A | +| ARM | MMU-700 | #2133013, | N/A | +| | | #2268618, | | +| | | #2812531, | | +| | | #3777127 | | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | MMU L1 | #3878312 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ +| ARM | MMU S3 | #3995052 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | GIC-700 | #2941627 | ARM64_ERRATUM_2941627 | +----------------+-----------------+-----------------+-----------------------------+ diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index b841efbcc9e9..f6901c5437ed 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -4939,6 +4939,8 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu) #define IIDR_IMPLEMENTER_ARM 0x43b #define IIDR_PRODUCTID_ARM_MMU_600 0x483 #define IIDR_PRODUCTID_ARM_MMU_700 0x487 +#define IIDR_PRODUCTID_ARM_MMU_L1 0x48a +#define IIDR_PRODUCTID_ARM_MMU_S3 0x498 static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu) { @@ -4963,11 +4965,19 @@ static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu) smmu->features &= ~ARM_SMMU_FEAT_NESTING; break; case IIDR_PRODUCTID_ARM_MMU_700: - /* Arm erratum 2812531 */ + /* Many errata... */ + smmu->features &= ~ARM_SMMU_FEAT_BTM; + if (variant < 1 || revision < 1) { + /* Arm erratum 2812531 */ + smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC; + /* Arm errata 2268618, 2812531 */ + smmu->features &= ~ARM_SMMU_FEAT_NESTING; + } + break; + case IIDR_PRODUCTID_ARM_MMU_L1: + case IIDR_PRODUCTID_ARM_MMU_S3: + /* Arm errata 3878312/3995052 */ smmu->features &= ~ARM_SMMU_FEAT_BTM; - smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC; - /* Arm errata 2268618, 2812531 */ - smmu->features &= ~ARM_SMMU_FEAT_NESTING; break; } break; From 803e41f36d227022ab9bbe780c82283fd4713b2e Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Thu, 12 Mar 2026 17:36:34 -0700 Subject: [PATCH 29/52] iommu/tegra241-cmdqv: Set supports_cmd op in tegra241_vcmdq_hw_init() vintf->hyp_own is finalized in tegra241_vintf_hw_init(). On the other hand, tegra241_vcmdq_alloc_smmu_cmdq() is called via an init_structures callback, which is earlier than tegra241_vintf_hw_init(). This results in the supports_cmd op always being set to the guest function, although this doesn't break any functionality nor have some noticeable perf impact since non-invalidation commands are not issued in the perf sensitive context. Fix this by moving supports_cmd to tegra241_vcmdq_hw_init(). After this change, - For a guest kernel, this will be a status quo - For a host kernel, non-invalidation commands will be issued to VCMDQ(s) Fixes: a9d40285bdef ("iommu/tegra241-cmdqv: Limit CMDs for VCMDQs of a guest owned VINTF") Reported-by: Eric Auger Reported-by: Shameer Kolothum Closes: https://lore.kernel.org/qemu-devel/CH3PR12MB754836BEE54E39B30C7210C0AB44A@CH3PR12MB7548.namprd12.prod.outlook.com/ Signed-off-by: Nicolin Chen Reviewed-by: Eric Auger Tested-by: Shameer Kolothum Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c index 6fe5563eaf9e..83f6e9f6c51d 100644 --- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c +++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c @@ -479,6 +479,10 @@ static int tegra241_vcmdq_hw_init(struct tegra241_vcmdq *vcmdq) /* Reset VCMDQ */ tegra241_vcmdq_hw_deinit(vcmdq); + /* vintf->hyp_own is a HW state finalized in tegra241_vintf_hw_init() */ + if (!vcmdq->vintf->hyp_own) + vcmdq->cmdq.supports_cmd = tegra241_guest_vcmdq_supports_cmd; + /* Configure and enable VCMDQ */ writeq_relaxed(vcmdq->cmdq.q.q_base, REG_VCMDQ_PAGE1(vcmdq, BASE)); @@ -639,9 +643,6 @@ static int tegra241_vcmdq_alloc_smmu_cmdq(struct tegra241_vcmdq *vcmdq) q->q_base = q->base_dma & VCMDQ_ADDR; q->q_base |= FIELD_PREP(VCMDQ_LOG2SIZE, q->llq.max_n_shift); - if (!vcmdq->vintf->hyp_own) - cmdq->supports_cmd = tegra241_guest_vcmdq_supports_cmd; - return arm_smmu_cmdq_init(smmu, cmdq); } From 9dcef98dbee35b8ae784df04c041efffdd42a69c Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Thu, 12 Mar 2026 17:36:35 -0700 Subject: [PATCH 30/52] iommu/tegra241-cmdqv: Update uAPI to clarify HYP_OWN requirement >From hardware implementation perspective, a guest tegra241-cmdqv hardware is different than the host hardware: - Host HW is backed by a VINTF (HYP_OWN=1) - Guest HW is backed by a VINTF (HYP_OWN=0) The kernel driver has an implementation requirement of the HYP_OWN bit in the VM. So, VMM must follow that to allow the same copy of Linux to work. Add this requirement to the uAPI, which is currently missing. Fixes: 4dc0d12474f9 ("iommu/tegra241-cmdqv: Add user-space use support") Signed-off-by: Nicolin Chen Reviewed-by: Eric Auger Reviewed-by: Jason Gunthorpe Signed-off-by: Will Deacon --- include/uapi/linux/iommufd.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h index 1dafbc552d37..f63edbe71d54 100644 --- a/include/uapi/linux/iommufd.h +++ b/include/uapi/linux/iommufd.h @@ -1052,6 +1052,11 @@ struct iommu_fault_alloc { enum iommu_viommu_type { IOMMU_VIOMMU_TYPE_DEFAULT = 0, IOMMU_VIOMMU_TYPE_ARM_SMMUV3 = 1, + /* + * TEGRA241_CMDQV requirements (otherwise, VCMDQs will not work) + * - Kernel will allocate a VINTF (HYP_OWN=0) to back this VIOMMU. So, + * VMM must wire the HYP_OWN bit to 0 in guest VINTF_CONFIG register + */ IOMMU_VIOMMU_TYPE_TEGRA241_CMDQV = 2, }; From 90c5def10bea574b101b7a520c015ca81742183f Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 2 Mar 2026 18:22:52 -0400 Subject: [PATCH 31/52] iommu: Do not call drivers for empty gathers An empty gather is coded with start=U64_MAX, end=0 and several drivers go on to convert that to a size with: end - start + 1 Which gives 2 for an empty gather. This then causes Weird Stuff to happen (for example an UBSAN splat in VT-d) that is hopefully harmless, but maybe not. Prevent drivers from being called right in iommu_iotlb_sync(). Auditing shows that AMD, Intel, Mediatek and RSIC-V drivers all do things on these empty gathers. Further, there are several callers that can trigger empty gathers, especially in unusual conditions. For example iommu_map_nosync() will call a 0 size unmap on some error paths. Also in VFIO, iommupt and other places. Cc: stable@vger.kernel.org Reported-by: Janusz Krzysztofik Closes: https://lore.kernel.org/r/11145826.aFP6jjVeTY@jkrzyszt-mobl2.ger.corp.intel.com Signed-off-by: Jason Gunthorpe Reviewed-by: Lu Baolu Reviewed-by: Samiullah Khawaja Reviewed-by: Robin Murphy Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 54b8b48c762e..555597b54083 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -980,7 +980,8 @@ static inline void iommu_flush_iotlb_all(struct iommu_domain *domain) static inline void iommu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *iotlb_gather) { - if (domain->ops->iotlb_sync) + if (domain->ops->iotlb_sync && + likely(iotlb_gather->start < iotlb_gather->end)) domain->ops->iotlb_sync(domain, iotlb_gather); iommu_iotlb_gather_init(iotlb_gather); From ee6e69d032550687a3422504bfca3f834c7b5061 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 2 Mar 2026 18:22:53 -0400 Subject: [PATCH 32/52] iommupt: Fix short gather if the unmap goes into a large mapping unmap has the odd behavior that it can unmap more than requested if the ending point lands within the middle of a large or contiguous IOPTE. In this case the gather should flush everything unmapped which can be larger than what was requested to be unmapped. The gather was only flushing the range requested to be unmapped, not extending to the extra range, resulting in a short invalidation if the caller hits this special condition. This was found by the new invalidation/gather test I am adding in preparation for ARMv8. Claude deduced the root cause. As far as I remember nothing relies on unmapping a large entry, so this is likely not a triggerable bug. Cc: stable@vger.kernel.org Fixes: 7c53f4238aa8 ("iommupt: Add unmap_pages op") Signed-off-by: Jason Gunthorpe Reviewed-by: Lu Baolu Reviewed-by: Samiullah Khawaja Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/iommu_pt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index 3e33fe64feab..7e7a6e7abdee 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -1057,7 +1057,7 @@ size_t DOMAIN_NS(unmap_pages)(struct iommu_domain *domain, unsigned long iova, pt_walk_range(&range, __unmap_range, &unmap); - gather_range_pages(iotlb_gather, iommu_table, iova, len, + gather_range_pages(iotlb_gather, iommu_table, iova, unmap.unmapped, &unmap.free_list); return unmap.unmapped; From e4172c5b53fba04fa48b13bc3afde809d0087a7f Mon Sep 17 00:00:00 2001 From: Guanghui Feng Date: Thu, 19 Mar 2026 15:37:53 +0800 Subject: [PATCH 33/52] iommu/amd: Fix illegal device-id access in IOMMU debugfs In the current AMD IOMMU debugFS, when multiple processes use the IOMMU debugFS process simultaneously, illegal access issues can occur in the following execution flow: 1. CPU1: Sets a valid sbdf via devid_write, then checks the sbdf's validity in execution flows such as devid_show, iommu_devtbl_show, and iommu_irqtbl_show. 2. CPU2: Sets an invalid sbdf via devid_write, at which point the sbdf value is -1. 3. CPU1: accesses the IOMMU device table, IRQ table, based on the invalid SBDF value of -1, resulting in illegal access. This is especially problematic in monitoring scripts, where multiple scripts may access debugFS simultaneously, and some scripts may unexpectedly set invalid values, which triggers illegal access in debugfs. This patch modifies the execution flow of devid_show, iommu_devtbl_show, and iommu_irqtbl_show to ensure that these processes determine the validity and access based on the same device-id, thus guaranteeing correctness and robustness. Signed-off-by: Guanghui Feng Signed-off-by: Joerg Roedel --- drivers/iommu/amd/debugfs.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c index 20b04996441d..0b03e0622f67 100644 --- a/drivers/iommu/amd/debugfs.c +++ b/drivers/iommu/amd/debugfs.c @@ -197,10 +197,11 @@ static ssize_t devid_write(struct file *filp, const char __user *ubuf, static int devid_show(struct seq_file *m, void *unused) { u16 devid; + int sbdf_shadow = sbdf; - if (sbdf >= 0) { - devid = PCI_SBDF_TO_DEVID(sbdf); - seq_printf(m, "%04x:%02x:%02x.%x\n", PCI_SBDF_TO_SEGID(sbdf), + if (sbdf_shadow >= 0) { + devid = PCI_SBDF_TO_DEVID(sbdf_shadow); + seq_printf(m, "%04x:%02x:%02x.%x\n", PCI_SBDF_TO_SEGID(sbdf_shadow), PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid)); } else seq_puts(m, "No or Invalid input provided\n"); @@ -237,13 +238,14 @@ static int iommu_devtbl_show(struct seq_file *m, void *unused) { struct amd_iommu_pci_seg *pci_seg; u16 seg, devid; + int sbdf_shadow = sbdf; - if (sbdf < 0) { + if (sbdf_shadow < 0) { seq_puts(m, "Enter a valid device ID to 'devid' file\n"); return 0; } - seg = PCI_SBDF_TO_SEGID(sbdf); - devid = PCI_SBDF_TO_DEVID(sbdf); + seg = PCI_SBDF_TO_SEGID(sbdf_shadow); + devid = PCI_SBDF_TO_DEVID(sbdf_shadow); for_each_pci_segment(pci_seg) { if (pci_seg->id != seg) @@ -336,19 +338,20 @@ static int iommu_irqtbl_show(struct seq_file *m, void *unused) { struct amd_iommu_pci_seg *pci_seg; u16 devid, seg; + int sbdf_shadow = sbdf; if (!irq_remapping_enabled) { seq_puts(m, "Interrupt remapping is disabled\n"); return 0; } - if (sbdf < 0) { + if (sbdf_shadow < 0) { seq_puts(m, "Enter a valid device ID to 'devid' file\n"); return 0; } - seg = PCI_SBDF_TO_SEGID(sbdf); - devid = PCI_SBDF_TO_DEVID(sbdf); + seg = PCI_SBDF_TO_SEGID(sbdf_shadow); + devid = PCI_SBDF_TO_DEVID(sbdf_shadow); for_each_pci_segment(pci_seg) { if (pci_seg->id != seg) From 0e59645683b7b6fa20eceb21a6f420e4f7412943 Mon Sep 17 00:00:00 2001 From: Guanghui Feng Date: Thu, 19 Mar 2026 15:37:54 +0800 Subject: [PATCH 34/52] iommu/amd: Fix illegal cap/mmio access in IOMMU debugfs In the current AMD IOMMU debugfs, when multiple processes simultaneously access the IOMMU mmio/cap registers using the IOMMU debugfs, illegal access issues can occur in the following execution flow: 1. CPU1: Sets a valid access address using iommu_mmio/capability_write, and verifies the access address's validity in iommu_mmio/capability_show 2. CPU2: Sets an invalid address using iommu_mmio/capability_write 3. CPU1: accesses the IOMMU mmio/cap registers based on the invalid address, resulting in an illegal access. This patch modifies the execution process to first verify the address's validity and then access it based on the same address, ensuring correctness and robustness. Signed-off-by: Guanghui Feng Signed-off-by: Joerg Roedel --- drivers/iommu/amd/debugfs.c | 42 +++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c index 0b03e0622f67..4e66473d7cea 100644 --- a/drivers/iommu/amd/debugfs.c +++ b/drivers/iommu/amd/debugfs.c @@ -26,22 +26,19 @@ static ssize_t iommu_mmio_write(struct file *filp, const char __user *ubuf, { struct seq_file *m = filp->private_data; struct amd_iommu *iommu = m->private; - int ret; - - iommu->dbg_mmio_offset = -1; + int ret, dbg_mmio_offset = iommu->dbg_mmio_offset = -1; if (cnt > OFS_IN_SZ) return -EINVAL; - ret = kstrtou32_from_user(ubuf, cnt, 0, &iommu->dbg_mmio_offset); + ret = kstrtou32_from_user(ubuf, cnt, 0, &dbg_mmio_offset); if (ret) return ret; - if (iommu->dbg_mmio_offset > iommu->mmio_phys_end - sizeof(u64)) { - iommu->dbg_mmio_offset = -1; - return -EINVAL; - } + if (dbg_mmio_offset > iommu->mmio_phys_end - sizeof(u64)) + return -EINVAL; + iommu->dbg_mmio_offset = dbg_mmio_offset; return cnt; } @@ -49,14 +46,16 @@ static int iommu_mmio_show(struct seq_file *m, void *unused) { struct amd_iommu *iommu = m->private; u64 value; + int dbg_mmio_offset = iommu->dbg_mmio_offset; - if (iommu->dbg_mmio_offset < 0) { + if (dbg_mmio_offset < 0 || dbg_mmio_offset > + iommu->mmio_phys_end - sizeof(u64)) { seq_puts(m, "Please provide mmio register's offset\n"); return 0; } - value = readq(iommu->mmio_base + iommu->dbg_mmio_offset); - seq_printf(m, "Offset:0x%x Value:0x%016llx\n", iommu->dbg_mmio_offset, value); + value = readq(iommu->mmio_base + dbg_mmio_offset); + seq_printf(m, "Offset:0x%x Value:0x%016llx\n", dbg_mmio_offset, value); return 0; } @@ -67,23 +66,20 @@ static ssize_t iommu_capability_write(struct file *filp, const char __user *ubuf { struct seq_file *m = filp->private_data; struct amd_iommu *iommu = m->private; - int ret; - - iommu->dbg_cap_offset = -1; + int ret, dbg_cap_offset = iommu->dbg_cap_offset = -1; if (cnt > OFS_IN_SZ) return -EINVAL; - ret = kstrtou32_from_user(ubuf, cnt, 0, &iommu->dbg_cap_offset); + ret = kstrtou32_from_user(ubuf, cnt, 0, &dbg_cap_offset); if (ret) return ret; /* Capability register at offset 0x14 is the last IOMMU capability register. */ - if (iommu->dbg_cap_offset > 0x14) { - iommu->dbg_cap_offset = -1; + if (dbg_cap_offset > 0x14) return -EINVAL; - } + iommu->dbg_cap_offset = dbg_cap_offset; return cnt; } @@ -91,21 +87,21 @@ static int iommu_capability_show(struct seq_file *m, void *unused) { struct amd_iommu *iommu = m->private; u32 value; - int err; + int err, dbg_cap_offset = iommu->dbg_cap_offset; - if (iommu->dbg_cap_offset < 0) { + if (dbg_cap_offset < 0 || dbg_cap_offset > 0x14) { seq_puts(m, "Please provide capability register's offset in the range [0x00 - 0x14]\n"); return 0; } - err = pci_read_config_dword(iommu->dev, iommu->cap_ptr + iommu->dbg_cap_offset, &value); + err = pci_read_config_dword(iommu->dev, iommu->cap_ptr + dbg_cap_offset, &value); if (err) { seq_printf(m, "Not able to read capability register at 0x%x\n", - iommu->dbg_cap_offset); + dbg_cap_offset); return 0; } - seq_printf(m, "Offset:0x%x Value:0x%08x\n", iommu->dbg_cap_offset, value); + seq_printf(m, "Offset:0x%x Value:0x%08x\n", dbg_cap_offset, value); return 0; } From 8b72aa5704c77380742346d4ac755b074b7f9eaa Mon Sep 17 00:00:00 2001 From: Sherry Yang Date: Thu, 26 Mar 2026 09:17:19 -0700 Subject: [PATCH 35/52] iommupt/amdv1: mark amdv1pt_install_leaf_entry as __always_inline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After enabling CONFIG_GCOV_KERNEL and CONFIG_GCOV_PROFILE_ALL, following build failure is observed under GCC 14.2.1: In function 'amdv1pt_install_leaf_entry', inlined from '__do_map_single_page' at drivers/iommu/generic_pt/fmt/../iommu_pt.h:650:3, inlined from '__map_single_page0' at drivers/iommu/generic_pt/fmt/../iommu_pt.h:661:1, inlined from 'pt_descend' at drivers/iommu/generic_pt/fmt/../pt_iter.h:391:9, inlined from '__do_map_single_page' at drivers/iommu/generic_pt/fmt/../iommu_pt.h:657:10, inlined from '__map_single_page1.constprop' at drivers/iommu/generic_pt/fmt/../iommu_pt.h:661:1: ././include/linux/compiler_types.h:706:45: error: call to '__compiletime_assert_71' declared with attribute error: FIELD_PREP: value too large for the field 706 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__) | ...... drivers/iommu/generic_pt/fmt/amdv1.h:220:26: note: in expansion of macro 'FIELD_PREP' 220 | FIELD_PREP(AMDV1PT_FMT_OA, | ^~~~~~~~~~ In the path '__do_map_single_page()', level 0 always invokes 'pt_install_leaf_entry(&pts, map->oa, PAGE_SHIFT, …)'. At runtime that lands in the 'if (oasz_lg2 == isz_lg2)' arm of 'amdv1pt_install_leaf_entry()'; the contiguous-only 'else' block is unreachable for 4 KiB pages. With CONFIG_GCOV_KERNEL + CONFIG_GCOV_PROFILE_ALL, the extra instrumentation changes GCC's inlining so that the "dead" 'else' branch still gets instantiated. The compiler constant-folds the contiguous OA expression, runs the 'FIELD_PREP()' compile-time check, and produces: FIELD_PREP: value too large for the field gcov-enabled builds therefore fail even though the code path never executes. Fix this by marking amdv1pt_install_leaf_entry as __always_inline. Fixes: dcd6a011a8d5 ("iommupt: Add map_pages op") Suggested-by: Jason Gunthorpe Signed-off-by: Sherry Yang Signed-off-by: Joerg Roedel --- drivers/iommu/generic_pt/fmt/amdv1.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/generic_pt/fmt/amdv1.h b/drivers/iommu/generic_pt/fmt/amdv1.h index 3b2c41d9654d..8d11b08291d7 100644 --- a/drivers/iommu/generic_pt/fmt/amdv1.h +++ b/drivers/iommu/generic_pt/fmt/amdv1.h @@ -191,7 +191,7 @@ static inline enum pt_entry_type amdv1pt_load_entry_raw(struct pt_state *pts) } #define pt_load_entry_raw amdv1pt_load_entry_raw -static inline void +static __always_inline void amdv1pt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa, unsigned int oasz_lg2, const struct pt_write_attrs *attrs) From 553a127cb66523089bc10eb54640205495f4bb5b Mon Sep 17 00:00:00 2001 From: Ethan Tidmore Date: Thu, 19 Mar 2026 13:26:44 -0500 Subject: [PATCH 36/52] iommu/riscv: Fix signedness bug The function platform_irq_count() returns negative error codes and iommu->irqs_count is an unsigned integer, so the check (iommu->irqs_count <= 0) is always impossible. Make the return value of platform_irq_count() be assigned to ret, check for error, and then assign iommu->irqs_count to ret. Detected by Smatch: drivers/iommu/riscv/iommu-platform.c:119 riscv_iommu_platform_probe() warn: 'iommu->irqs_count' unsigned <= 0 Signed-off-by: Ethan Tidmore Fixes: 5c0ebbd3c6c6 ("iommu/riscv: Add RISC-V IOMMU platform device driver") Reviewed-by: Andrew Jones Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu-platform.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/riscv/iommu-platform.c b/drivers/iommu/riscv/iommu-platform.c index 8f15b06e8499..399ba8fe1b3e 100644 --- a/drivers/iommu/riscv/iommu-platform.c +++ b/drivers/iommu/riscv/iommu-platform.c @@ -115,10 +115,13 @@ static int riscv_iommu_platform_probe(struct platform_device *pdev) fallthrough; case RISCV_IOMMU_CAPABILITIES_IGS_WSI: - iommu->irqs_count = platform_irq_count(pdev); - if (iommu->irqs_count <= 0) + ret = platform_irq_count(pdev); + if (ret <= 0) return dev_err_probe(dev, -ENODEV, "no IRQ resources provided\n"); + + iommu->irqs_count = ret; + if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT) iommu->irqs_count = RISCV_IOMMU_INTR_COUNT; From 1c18a1212c772b6a19e8583f2fca73f3a47b60fd Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 26 Mar 2026 16:30:32 -0300 Subject: [PATCH 37/52] iommu/dma: Always allow DMA-FQ when iommupt provides the iommu_domain iommupt always supports the semantics required for DMA-FQ, when drivers are converted to use it they automatically get support. Detect iommpt directly instead of using IOMMU_CAP_DEFERRED_FLUSH and remove IOMMU_CAP_DEFERRED_FLUSH from converted drivers. This will also enable DMA-FQ on RISC-V. Signed-off-by: Jason Gunthorpe Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 2 -- drivers/iommu/dma-iommu.c | 13 ++++++++++++- drivers/iommu/intel/iommu.c | 2 -- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index f1814fee5182..2e553e2051aa 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2978,8 +2978,6 @@ static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) return amdr_ivrs_remap_support; case IOMMU_CAP_ENFORCE_CACHE_COHERENCY: return true; - case IOMMU_CAP_DEFERRED_FLUSH: - return true; case IOMMU_CAP_DIRTY_TRACKING: { struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 5dac64be61bb..fbed93f8bf0a 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -648,6 +649,15 @@ static void iommu_dma_init_options(struct iommu_dma_options *options, } } +static bool iommu_domain_supports_fq(struct device *dev, + struct iommu_domain *domain) +{ + /* iommupt always supports DMA-FQ */ + if (iommupt_from_domain(domain)) + return true; + return device_iommu_capable(dev, IOMMU_CAP_DEFERRED_FLUSH); +} + /** * iommu_dma_init_domain - Initialise a DMA mapping domain * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() @@ -706,7 +716,8 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, struct device *dev /* If the FQ fails we can simply fall back to strict mode */ if (domain->type == IOMMU_DOMAIN_DMA_FQ && - (!device_iommu_capable(dev, IOMMU_CAP_DEFERRED_FLUSH) || iommu_dma_init_fq(domain))) + (!iommu_domain_supports_fq(dev, domain) || + iommu_dma_init_fq(domain))) domain->type = IOMMU_DOMAIN_DMA; return iova_reserve_iommu_regions(dev, domain); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 5dca8e525c73..80b183e207e5 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3212,8 +3212,6 @@ static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap) switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: - case IOMMU_CAP_DEFERRED_FLUSH: - return true; case IOMMU_CAP_PRE_BOOT_PROTECTION: return dmar_platform_optin(); case IOMMU_CAP_ENFORCE_CACHE_COHERENCY: From cc5bd898ff70710ffc41cd8e5c2741cb64750047 Mon Sep 17 00:00:00 2001 From: Zhenzhong Duan Date: Thu, 2 Apr 2026 14:57:24 +0800 Subject: [PATCH 38/52] iommu/vt-d: Block PASID attachment to nested domain with dirty tracking Kernel lacks dirty tracking support on nested domain attached to PASID, fails the attachment early if nesting parent domain is dirty tracking configured, otherwise dirty pages would be lost. Cc: stable@vger.kernel.org Fixes: 67f6f56b5912 ("iommu/vt-d: Add set_dev_pasid callback for nested domain") Suggested-by: Kevin Tian Signed-off-by: Zhenzhong Duan Reviewed-by: Kevin Tian Reviewed-by: Yi Liu Link: https://lore.kernel.org/r/20260330101108.12594-2-zhenzhong.duan@intel.com Signed-off-by: Lu Baolu Fixes: 67f6f56b5912 ("iommu/vt-d: Add set_dev_pasid callback for nested domain") Signed-off-by: Joerg Roedel --- drivers/iommu/intel/nested.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c index 2b979bec56ce..16c82ba47d30 100644 --- a/drivers/iommu/intel/nested.c +++ b/drivers/iommu/intel/nested.c @@ -148,6 +148,7 @@ static int intel_nested_set_dev_pasid(struct iommu_domain *domain, { struct device_domain_info *info = dev_iommu_priv_get(dev); struct dmar_domain *dmar_domain = to_dmar_domain(domain); + struct iommu_domain *s2_domain = &dmar_domain->s2_domain->domain; struct intel_iommu *iommu = info->iommu; struct dev_pasid_info *dev_pasid; int ret; @@ -155,10 +156,13 @@ static int intel_nested_set_dev_pasid(struct iommu_domain *domain, if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev)) return -EOPNOTSUPP; + if (s2_domain->dirty_ops) + return -EINVAL; + if (context_copied(iommu, info->bus, info->devfn)) return -EBUSY; - ret = paging_domain_compatible(&dmar_domain->s2_domain->domain, dev); + ret = paging_domain_compatible(s2_domain, dev); if (ret) return ret; From 922e2598a40f1851620144b3997aeefe066bd4de Mon Sep 17 00:00:00 2001 From: Zhenzhong Duan Date: Thu, 2 Apr 2026 14:57:25 +0800 Subject: [PATCH 39/52] iommu/vt-d: Rename device_set_dirty_tracking() and pass dmar_domain pointer device_set_dirty_tracking() sets dirty tracking on all devices attached to a domain, also on all PASIDs attached to same domain in subsequent patch. So rename it as domain_set_dirty_tracking() and pass dmar_domain pointer to better align to what it does. No functional changes intended. Suggested-by: Lu Baolu Signed-off-by: Zhenzhong Duan Reviewed-by: Yi Liu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20260330101108.12594-3-zhenzhong.duan@intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index ef7613b177b9..965e0330ec4b 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3684,16 +3684,15 @@ static void *intel_iommu_hw_info(struct device *dev, u32 *length, return vtd; } -/* - * Set dirty tracking for the device list of a domain. The caller must - * hold the domain->lock when calling it. - */ -static int device_set_dirty_tracking(struct list_head *devices, bool enable) +/* Set dirty tracking for the devices that the domain has been attached. */ +static int domain_set_dirty_tracking(struct dmar_domain *domain, bool enable) { struct device_domain_info *info; int ret = 0; - list_for_each_entry(info, devices, link) { + lockdep_assert_held(&domain->lock); + + list_for_each_entry(info, &domain->devices, link) { ret = intel_pasid_setup_dirty_tracking(info->iommu, info->dev, IOMMU_NO_PASID, enable); if (ret) @@ -3713,7 +3712,7 @@ static int parent_domain_set_dirty_tracking(struct dmar_domain *domain, spin_lock(&domain->s1_lock); list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) { spin_lock_irqsave(&s1_domain->lock, flags); - ret = device_set_dirty_tracking(&s1_domain->devices, enable); + ret = domain_set_dirty_tracking(s1_domain, enable); spin_unlock_irqrestore(&s1_domain->lock, flags); if (ret) goto err_unwind; @@ -3724,8 +3723,7 @@ static int parent_domain_set_dirty_tracking(struct dmar_domain *domain, err_unwind: list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) { spin_lock_irqsave(&s1_domain->lock, flags); - device_set_dirty_tracking(&s1_domain->devices, - domain->dirty_tracking); + domain_set_dirty_tracking(s1_domain, domain->dirty_tracking); spin_unlock_irqrestore(&s1_domain->lock, flags); } spin_unlock(&domain->s1_lock); @@ -3742,7 +3740,7 @@ static int intel_iommu_set_dirty_tracking(struct iommu_domain *domain, if (dmar_domain->dirty_tracking == enable) goto out_unlock; - ret = device_set_dirty_tracking(&dmar_domain->devices, enable); + ret = domain_set_dirty_tracking(dmar_domain, enable); if (ret) goto err_unwind; @@ -3759,8 +3757,7 @@ static int intel_iommu_set_dirty_tracking(struct iommu_domain *domain, return 0; err_unwind: - device_set_dirty_tracking(&dmar_domain->devices, - dmar_domain->dirty_tracking); + domain_set_dirty_tracking(dmar_domain, dmar_domain->dirty_tracking); spin_unlock(&dmar_domain->lock); return ret; } From ae2fafc19e7bfcdd00920888468546f35286e715 Mon Sep 17 00:00:00 2001 From: Zhenzhong Duan Date: Thu, 2 Apr 2026 14:57:26 +0800 Subject: [PATCH 40/52] iommu/vt-d: Support dirty tracking on PASID In order to support passthrough device with PASID capability in QEMU, e.g., DSA device, kernel needs to support attaching PASID to a domain. But attaching is not allowed if the domain is a second stage domain or nested domain with dirty tracking. The reason is kernel lacking support for dirty tracking on such domain attached to PASID. By adding dirty tracking on PASID, the check can be removed. Signed-off-by: Zhenzhong Duan Reviewed-by: Yi Liu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20260330101108.12594-4-zhenzhong.duan@intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 12 +++++++++--- drivers/iommu/intel/nested.c | 6 +----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 965e0330ec4b..26135ff3a289 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3618,9 +3618,6 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain, if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev)) return -EOPNOTSUPP; - if (domain->dirty_ops) - return -EINVAL; - if (context_copied(iommu, info->bus, info->devfn)) return -EBUSY; @@ -3688,6 +3685,7 @@ static void *intel_iommu_hw_info(struct device *dev, u32 *length, static int domain_set_dirty_tracking(struct dmar_domain *domain, bool enable) { struct device_domain_info *info; + struct dev_pasid_info *dev_pasid; int ret = 0; lockdep_assert_held(&domain->lock); @@ -3695,6 +3693,14 @@ static int domain_set_dirty_tracking(struct dmar_domain *domain, bool enable) list_for_each_entry(info, &domain->devices, link) { ret = intel_pasid_setup_dirty_tracking(info->iommu, info->dev, IOMMU_NO_PASID, enable); + if (ret) + return ret; + } + + list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) { + info = dev_iommu_priv_get(dev_pasid->dev); + ret = intel_pasid_setup_dirty_tracking(info->iommu, info->dev, + dev_pasid->pasid, enable); if (ret) break; } diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c index 16c82ba47d30..2b979bec56ce 100644 --- a/drivers/iommu/intel/nested.c +++ b/drivers/iommu/intel/nested.c @@ -148,7 +148,6 @@ static int intel_nested_set_dev_pasid(struct iommu_domain *domain, { struct device_domain_info *info = dev_iommu_priv_get(dev); struct dmar_domain *dmar_domain = to_dmar_domain(domain); - struct iommu_domain *s2_domain = &dmar_domain->s2_domain->domain; struct intel_iommu *iommu = info->iommu; struct dev_pasid_info *dev_pasid; int ret; @@ -156,13 +155,10 @@ static int intel_nested_set_dev_pasid(struct iommu_domain *domain, if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev)) return -EOPNOTSUPP; - if (s2_domain->dirty_ops) - return -EINVAL; - if (context_copied(iommu, info->bus, info->devfn)) return -EBUSY; - ret = paging_domain_compatible(s2_domain, dev); + ret = paging_domain_compatible(&dmar_domain->s2_domain->domain, dev); if (ret) return ret; From c9587216d991120c3cf546bcd708422b26334888 Mon Sep 17 00:00:00 2001 From: Zhenzhong Duan Date: Thu, 2 Apr 2026 14:57:27 +0800 Subject: [PATCH 41/52] iommufd/selftest: Test dirty tracking on PASID Add test case for dirty tracking on a domain attached to PASID, also confirm attachment to PASID fail if device doesn't support dirty tracking. Suggested-by: Lu Baolu Signed-off-by: Zhenzhong Duan Reviewed-by: Yi Liu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20260330101108.12594-5-zhenzhong.duan@intel.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- tools/testing/selftests/iommu/iommufd.c | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tools/testing/selftests/iommu/iommufd.c b/tools/testing/selftests/iommu/iommufd.c index dadad277f4eb..d1fe5dbc2813 100644 --- a/tools/testing/selftests/iommu/iommufd.c +++ b/tools/testing/selftests/iommu/iommufd.c @@ -2275,6 +2275,33 @@ TEST_F(iommufd_dirty_tracking, set_dirty_tracking) test_ioctl_destroy(hwpt_id); } +TEST_F(iommufd_dirty_tracking, pasid_set_dirty_tracking) +{ + uint32_t stddev_id, ioas_id, hwpt_id, pasid = 100; + uint32_t dev_flags = MOCK_FLAGS_DEVICE_PASID; + + /* Regular case */ + test_cmd_hwpt_alloc(self->idev_id, self->ioas_id, + IOMMU_HWPT_ALLOC_PASID | IOMMU_HWPT_ALLOC_DIRTY_TRACKING, + &hwpt_id); + test_cmd_mock_domain_flags(hwpt_id, dev_flags, &stddev_id, NULL, NULL); + ASSERT_EQ(0, _test_cmd_pasid_attach(self->fd, stddev_id, pasid, hwpt_id)); + test_cmd_set_dirty_tracking(hwpt_id, true); + test_cmd_set_dirty_tracking(hwpt_id, false); + ASSERT_EQ(0, _test_cmd_pasid_detach(self->fd, stddev_id, pasid)); + + test_ioctl_destroy(stddev_id); + + /* IOMMU device does not support dirty tracking */ + dev_flags |= MOCK_FLAGS_DEVICE_NO_DIRTY; + test_ioctl_ioas_alloc(&ioas_id); + test_cmd_mock_domain_flags(ioas_id, dev_flags, &stddev_id, NULL, NULL); + EXPECT_ERRNO(EINVAL, _test_cmd_pasid_attach(self->fd, stddev_id, pasid, hwpt_id)); + + test_ioctl_destroy(stddev_id); + test_ioctl_destroy(hwpt_id); +} + TEST_F(iommufd_dirty_tracking, device_dirty_capability) { uint32_t caps = 0; From 973009137138aa6372c2346d389601e26659645b Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Thu, 2 Apr 2026 14:57:28 +0800 Subject: [PATCH 42/52] iommu/vt-d: Remove dmar_readl() and dmar_readq() dmar_readl() and dmar_readq() do nothing other than expand to the generic readl() and readq(), and the dmar_read*() wrappers are used inconsistently. Remove the dmar_read*() wrappers and use readl() and readq() directly. Signed-off-by: Bjorn Helgaas Reviewed-by: Samiullah Khawaja Link: https://lore.kernel.org/r/20260217214438.3395039-2-bhelgaas@google.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/debugfs.c | 18 +++++++++--------- drivers/iommu/intel/dmar.c | 22 +++++++++++----------- drivers/iommu/intel/iommu.c | 10 +++++----- drivers/iommu/intel/iommu.h | 2 -- drivers/iommu/intel/irq_remapping.c | 2 +- drivers/iommu/intel/perfmon.c | 28 ++++++++++++++-------------- drivers/iommu/intel/prq.c | 12 ++++++------ 7 files changed, 46 insertions(+), 48 deletions(-) diff --git a/drivers/iommu/intel/debugfs.c b/drivers/iommu/intel/debugfs.c index 617fd81a80f0..21e4e465ca58 100644 --- a/drivers/iommu/intel/debugfs.c +++ b/drivers/iommu/intel/debugfs.c @@ -133,13 +133,13 @@ static int iommu_regset_show(struct seq_file *m, void *unused) */ raw_spin_lock_irqsave(&iommu->register_lock, flag); for (i = 0 ; i < ARRAY_SIZE(iommu_regs_32); i++) { - value = dmar_readl(iommu->reg + iommu_regs_32[i].offset); + value = readl(iommu->reg + iommu_regs_32[i].offset); seq_printf(m, "%-16s\t0x%02x\t\t0x%016llx\n", iommu_regs_32[i].regs, iommu_regs_32[i].offset, value); } for (i = 0 ; i < ARRAY_SIZE(iommu_regs_64); i++) { - value = dmar_readq(iommu->reg + iommu_regs_64[i].offset); + value = readq(iommu->reg + iommu_regs_64[i].offset); seq_printf(m, "%-16s\t0x%02x\t\t0x%016llx\n", iommu_regs_64[i].regs, iommu_regs_64[i].offset, value); @@ -247,7 +247,7 @@ static void ctx_tbl_walk(struct seq_file *m, struct intel_iommu *iommu, u16 bus) tbl_wlk.ctx_entry = context; m->private = &tbl_wlk; - if (dmar_readq(iommu->reg + DMAR_RTADDR_REG) & DMA_RTADDR_SMT) { + if (readq(iommu->reg + DMAR_RTADDR_REG) & DMA_RTADDR_SMT) { pasid_dir_ptr = context->lo & VTD_PAGE_MASK; pasid_dir_size = get_pasid_dir_size(context); pasid_dir_walk(m, pasid_dir_ptr, pasid_dir_size); @@ -285,7 +285,7 @@ static int dmar_translation_struct_show(struct seq_file *m, void *unused) rcu_read_lock(); for_each_active_iommu(iommu, drhd) { - sts = dmar_readl(iommu->reg + DMAR_GSTS_REG); + sts = readl(iommu->reg + DMAR_GSTS_REG); if (!(sts & DMA_GSTS_TES)) { seq_printf(m, "DMA Remapping is not enabled on %s\n", iommu->name); @@ -364,13 +364,13 @@ static int domain_translation_struct_show(struct seq_file *m, if (seg != iommu->segment) continue; - sts = dmar_readl(iommu->reg + DMAR_GSTS_REG); + sts = readl(iommu->reg + DMAR_GSTS_REG); if (!(sts & DMA_GSTS_TES)) { seq_printf(m, "DMA Remapping is not enabled on %s\n", iommu->name); continue; } - if (dmar_readq(iommu->reg + DMAR_RTADDR_REG) & DMA_RTADDR_SMT) + if (readq(iommu->reg + DMAR_RTADDR_REG) & DMA_RTADDR_SMT) scalable = true; else scalable = false; @@ -538,8 +538,8 @@ static int invalidation_queue_show(struct seq_file *m, void *unused) raw_spin_lock_irqsave(&qi->q_lock, flags); seq_printf(m, " Base: 0x%llx\tHead: %lld\tTail: %lld\n", (u64)virt_to_phys(qi->desc), - dmar_readq(iommu->reg + DMAR_IQH_REG) >> shift, - dmar_readq(iommu->reg + DMAR_IQT_REG) >> shift); + readq(iommu->reg + DMAR_IQH_REG) >> shift, + readq(iommu->reg + DMAR_IQT_REG) >> shift); invalidation_queue_entry_show(m, iommu); raw_spin_unlock_irqrestore(&qi->q_lock, flags); seq_putc(m, '\n'); @@ -620,7 +620,7 @@ static int ir_translation_struct_show(struct seq_file *m, void *unused) seq_printf(m, "Remapped Interrupt supported on IOMMU: %s\n", iommu->name); - sts = dmar_readl(iommu->reg + DMAR_GSTS_REG); + sts = readl(iommu->reg + DMAR_GSTS_REG); if (iommu->ir_table && (sts & DMA_GSTS_IRES)) { irta = virt_to_phys(iommu->ir_table->base); seq_printf(m, " IR table address:%llx\n", irta); diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index d68c06025cac..a616026b3648 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -899,8 +899,8 @@ dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg) return -EINVAL; } - cap = dmar_readq(addr + DMAR_CAP_REG); - ecap = dmar_readq(addr + DMAR_ECAP_REG); + cap = readq(addr + DMAR_CAP_REG); + ecap = readq(addr + DMAR_ECAP_REG); if (arg) iounmap(addr); @@ -982,8 +982,8 @@ static int map_iommu(struct intel_iommu *iommu, struct dmar_drhd_unit *drhd) goto release; } - iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG); - iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG); + iommu->cap = readq(iommu->reg + DMAR_CAP_REG); + iommu->ecap = readq(iommu->reg + DMAR_ECAP_REG); if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) { err = -EINVAL; @@ -1017,8 +1017,8 @@ static int map_iommu(struct intel_iommu *iommu, struct dmar_drhd_unit *drhd) int i; for (i = 0; i < DMA_MAX_NUM_ECMDCAP; i++) { - iommu->ecmdcap[i] = dmar_readq(iommu->reg + DMAR_ECCAP_REG + - i * DMA_ECMD_REG_STEP); + iommu->ecmdcap[i] = readq(iommu->reg + DMAR_ECCAP_REG + + i * DMA_ECMD_REG_STEP); } } @@ -1239,8 +1239,8 @@ static const char *qi_type_string(u8 type) static void qi_dump_fault(struct intel_iommu *iommu, u32 fault) { - unsigned int head = dmar_readl(iommu->reg + DMAR_IQH_REG); - u64 iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG); + unsigned int head = readl(iommu->reg + DMAR_IQH_REG); + u64 iqe_err = readq(iommu->reg + DMAR_IQER_REG); struct qi_desc *desc = iommu->qi->desc + head; if (fault & DMA_FSTS_IQE) @@ -1322,7 +1322,7 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) * SID field is valid only when the ITE field is Set in FSTS_REG * see Intel VT-d spec r4.1, section 11.4.9.9 */ - iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG); + iqe_err = readq(iommu->reg + DMAR_IQER_REG); ite_sid = DMAR_IQER_REG_ITESID(iqe_err); writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG); @@ -1981,8 +1981,8 @@ irqreturn_t dmar_fault(int irq, void *dev_id) source_id = dma_frcd_source_id(data); pasid_present = dma_frcd_pasid_present(data); - guest_addr = dmar_readq(iommu->reg + reg + - fault_index * PRIMARY_FAULT_REG_LEN); + guest_addr = readq(iommu->reg + reg + + fault_index * PRIMARY_FAULT_REG_LEN); guest_addr = dma_frcd_page_addr(guest_addr); } diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 26135ff3a289..4cb39000cd91 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -769,7 +769,7 @@ static void __iommu_flush_context(struct intel_iommu *iommu, /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG, - dmar_readq, (!(val & DMA_CCMD_ICC)), val); + readq, (!(val & DMA_CCMD_ICC)), val); raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } @@ -811,7 +811,7 @@ void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, tlb_offset + 8, - dmar_readq, (!(val & DMA_TLB_IVT)), val); + readq, (!(val & DMA_TLB_IVT)), val); raw_spin_unlock_irqrestore(&iommu->register_lock, flag); @@ -1533,7 +1533,7 @@ static int copy_translation_tables(struct intel_iommu *iommu) int bus, ret; bool new_ext, ext; - rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG); + rtaddr_reg = readq(iommu->reg + DMAR_RTADDR_REG); ext = !!(rtaddr_reg & DMA_RTADDR_SMT); new_ext = !!sm_supported(iommu); @@ -4188,7 +4188,7 @@ int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob) raw_spin_lock_irqsave(&iommu->register_lock, flags); - res = dmar_readq(iommu->reg + DMAR_ECRSP_REG); + res = readq(iommu->reg + DMAR_ECRSP_REG); if (res & DMA_ECMD_ECRSP_IP) { ret = -EBUSY; goto err; @@ -4204,7 +4204,7 @@ int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob) dmar_writeq(iommu->reg + DMAR_ECEO_REG, ob); dmar_writeq(iommu->reg + DMAR_ECMD_REG, ecmd | (oa << DMA_ECMD_OA_SHIFT)); - IOMMU_WAIT_OP(iommu, DMAR_ECRSP_REG, dmar_readq, + IOMMU_WAIT_OP(iommu, DMAR_ECRSP_REG, readq, !(res & DMA_ECMD_ECRSP_IP), res); if (res & DMA_ECMD_ECRSP_IP) { diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 599913fb65d5..dbd8d196d154 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -148,9 +148,7 @@ #define OFFSET_STRIDE (9) -#define dmar_readq(a) readq(a) #define dmar_writeq(a,v) writeq(v,a) -#define dmar_readl(a) readl(a) #define dmar_writel(a, v) writel(v, a) #define DMAR_VER_MAJOR(v) (((v) & 0xf0) >> 4) diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 1cd2101610df..21e54e40a17f 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -422,7 +422,7 @@ static int iommu_load_old_irte(struct intel_iommu *iommu) u64 irta; /* Check whether the old ir-table has the same size as ours */ - irta = dmar_readq(iommu->reg + DMAR_IRTA_REG); + irta = readq(iommu->reg + DMAR_IRTA_REG); if ((irta & INTR_REMAP_TABLE_REG_SIZE_MASK) != INTR_REMAP_TABLE_REG_SIZE) return -EINVAL; diff --git a/drivers/iommu/intel/perfmon.c b/drivers/iommu/intel/perfmon.c index fec51b6036b6..3f75f567f210 100644 --- a/drivers/iommu/intel/perfmon.c +++ b/drivers/iommu/intel/perfmon.c @@ -307,7 +307,7 @@ static void iommu_pmu_event_update(struct perf_event *event) again: prev_count = local64_read(&hwc->prev_count); - new_count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx)); + new_count = readq(iommu_event_base(iommu_pmu, hwc->idx)); if (local64_xchg(&hwc->prev_count, new_count) != prev_count) goto again; @@ -340,7 +340,7 @@ static void iommu_pmu_start(struct perf_event *event, int flags) hwc->state = 0; /* Always reprogram the period */ - count = dmar_readq(iommu_event_base(iommu_pmu, hwc->idx)); + count = readq(iommu_event_base(iommu_pmu, hwc->idx)); local64_set((&hwc->prev_count), count); /* @@ -496,7 +496,7 @@ static void iommu_pmu_counter_overflow(struct iommu_pmu *iommu_pmu) * Two counters may be overflowed very close. Always check * whether there are more to handle. */ - while ((status = dmar_readq(iommu_pmu->overflow))) { + while ((status = readq(iommu_pmu->overflow))) { for_each_set_bit(i, (unsigned long *)&status, iommu_pmu->num_cntr) { /* * Find the assigned event of the counter. @@ -518,7 +518,7 @@ static irqreturn_t iommu_pmu_irq_handler(int irq, void *dev_id) { struct intel_iommu *iommu = dev_id; - if (!dmar_readl(iommu->reg + DMAR_PERFINTRSTS_REG)) + if (!readl(iommu->reg + DMAR_PERFINTRSTS_REG)) return IRQ_NONE; iommu_pmu_counter_overflow(iommu->pmu); @@ -555,7 +555,7 @@ static int __iommu_pmu_register(struct intel_iommu *iommu) static inline void __iomem * get_perf_reg_address(struct intel_iommu *iommu, u32 offset) { - u32 off = dmar_readl(iommu->reg + offset); + u32 off = readl(iommu->reg + offset); return iommu->reg + off; } @@ -574,7 +574,7 @@ int alloc_iommu_pmu(struct intel_iommu *iommu) if (!cap_ecmds(iommu->cap)) return -ENODEV; - perfcap = dmar_readq(iommu->reg + DMAR_PERFCAP_REG); + perfcap = readq(iommu->reg + DMAR_PERFCAP_REG); /* The performance monitoring is not supported. */ if (!perfcap) return -ENODEV; @@ -617,8 +617,8 @@ int alloc_iommu_pmu(struct intel_iommu *iommu) for (i = 0; i < iommu_pmu->num_eg; i++) { u64 pcap; - pcap = dmar_readq(iommu->reg + DMAR_PERFEVNTCAP_REG + - i * IOMMU_PMU_CAP_REGS_STEP); + pcap = readq(iommu->reg + DMAR_PERFEVNTCAP_REG + + i * IOMMU_PMU_CAP_REGS_STEP); iommu_pmu->evcap[i] = pecap_es(pcap); } @@ -651,9 +651,9 @@ int alloc_iommu_pmu(struct intel_iommu *iommu) * Width. */ for (i = 0; i < iommu_pmu->num_cntr; i++) { - cap = dmar_readl(iommu_pmu->cfg_reg + - i * IOMMU_PMU_CFG_OFFSET + - IOMMU_PMU_CFG_CNTRCAP_OFFSET); + cap = readl(iommu_pmu->cfg_reg + + i * IOMMU_PMU_CFG_OFFSET + + IOMMU_PMU_CFG_CNTRCAP_OFFSET); if (!iommu_cntrcap_pcc(cap)) continue; @@ -675,9 +675,9 @@ int alloc_iommu_pmu(struct intel_iommu *iommu) /* Override with per-counter event capabilities */ for (j = 0; j < iommu_cntrcap_egcnt(cap); j++) { - cap = dmar_readl(iommu_pmu->cfg_reg + i * IOMMU_PMU_CFG_OFFSET + - IOMMU_PMU_CFG_CNTREVCAP_OFFSET + - (j * IOMMU_PMU_OFF_REGS_STEP)); + cap = readl(iommu_pmu->cfg_reg + i * IOMMU_PMU_CFG_OFFSET + + IOMMU_PMU_CFG_CNTREVCAP_OFFSET + + (j * IOMMU_PMU_OFF_REGS_STEP)); iommu_pmu->cntr_evcap[i][iommu_event_group(cap)] = iommu_event_select(cap); /* * Some events may only be supported by a specific counter. diff --git a/drivers/iommu/intel/prq.c b/drivers/iommu/intel/prq.c index ff63c228e6e1..c28fbd5c14a7 100644 --- a/drivers/iommu/intel/prq.c +++ b/drivers/iommu/intel/prq.c @@ -81,8 +81,8 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid) */ prq_retry: reinit_completion(&iommu->prq_complete); - tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; - head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; + tail = readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; + head = readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; while (head != tail) { struct page_req_dsc *req; @@ -208,8 +208,8 @@ static irqreturn_t prq_event_thread(int irq, void *d) */ writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG); - tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; - head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; + tail = readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; + head = readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; handled = (head != tail); while (head != tail) { req = &iommu->prq[head / sizeof(*req)]; @@ -268,8 +268,8 @@ static irqreturn_t prq_event_thread(int irq, void *d) if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { pr_info_ratelimited("IOMMU: %s: PRQ overflow detected\n", iommu->name); - head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; - tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; + head = readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; + tail = readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; if (head == tail) { iopf_queue_discard_partial(iommu->iopf_queue); writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG); From 51234c4e57c8cee33dfcbdd07e65ab06c86eb326 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Thu, 2 Apr 2026 14:57:29 +0800 Subject: [PATCH 43/52] iommu/vt-d: Remove dmar_writel() and dmar_writeq() dmar_writel() and dmar_writeq() do nothing other than expand to the generic writel() and writeq(), and the dmar_write*() wrappers are used inconsistently. Remove the dmar_write*() wrappers and use writel() and writeq() directly. Signed-off-by: Bjorn Helgaas Reviewed-by: Samiullah Khawaja Link: https://lore.kernel.org/r/20260217214438.3395039-3-bhelgaas@google.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 2 +- drivers/iommu/intel/iommu.c | 12 ++++++------ drivers/iommu/intel/iommu.h | 3 --- drivers/iommu/intel/irq_remapping.c | 4 ++-- drivers/iommu/intel/perfmon.c | 22 +++++++++++----------- drivers/iommu/intel/prq.c | 14 +++++++------- 6 files changed, 27 insertions(+), 30 deletions(-) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index a616026b3648..b958f2e6042b 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1662,7 +1662,7 @@ static void __dmar_enable_qi(struct intel_iommu *iommu) /* write zero to the tail reg */ writel(0, iommu->reg + DMAR_IQT_REG); - dmar_writeq(iommu->reg + DMAR_IQA_REG, val); + writeq(val, iommu->reg + DMAR_IQA_REG); iommu->gcmd |= DMA_GCMD_QIE; writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 4cb39000cd91..297415fe726d 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -697,7 +697,7 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) addr |= DMA_RTADDR_SMT; raw_spin_lock_irqsave(&iommu->register_lock, flag); - dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr); + writeq(addr, iommu->reg + DMAR_RTADDR_REG); writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG); @@ -765,7 +765,7 @@ static void __iommu_flush_context(struct intel_iommu *iommu, val |= DMA_CCMD_ICC; raw_spin_lock_irqsave(&iommu->register_lock, flag); - dmar_writeq(iommu->reg + DMAR_CCMD_REG, val); + writeq(val, iommu->reg + DMAR_CCMD_REG); /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG, @@ -806,8 +806,8 @@ void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, raw_spin_lock_irqsave(&iommu->register_lock, flag); /* Note: Only uses first TLB reg currently */ if (val_iva) - dmar_writeq(iommu->reg + tlb_offset, val_iva); - dmar_writeq(iommu->reg + tlb_offset + 8, val); + writeq(val_iva, iommu->reg + tlb_offset); + writeq(val, iommu->reg + tlb_offset + 8); /* Make sure hardware complete it */ IOMMU_WAIT_OP(iommu, tlb_offset + 8, @@ -4201,8 +4201,8 @@ int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob) * - It's not invoked in any critical path. The extra MMIO * write doesn't bring any performance concerns. */ - dmar_writeq(iommu->reg + DMAR_ECEO_REG, ob); - dmar_writeq(iommu->reg + DMAR_ECMD_REG, ecmd | (oa << DMA_ECMD_OA_SHIFT)); + writeq(ob, iommu->reg + DMAR_ECEO_REG); + writeq(ecmd | (oa << DMA_ECMD_OA_SHIFT), iommu->reg + DMAR_ECMD_REG); IOMMU_WAIT_OP(iommu, DMAR_ECRSP_REG, readq, !(res & DMA_ECMD_ECRSP_IP), res); diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index dbd8d196d154..10331364c0ef 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -148,9 +148,6 @@ #define OFFSET_STRIDE (9) -#define dmar_writeq(a,v) writeq(v,a) -#define dmar_writel(a, v) writel(v, a) - #define DMAR_VER_MAJOR(v) (((v) & 0xf0) >> 4) #define DMAR_VER_MINOR(v) ((v) & 0x0f) diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 21e54e40a17f..25c26f706984 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -465,8 +465,8 @@ static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) raw_spin_lock_irqsave(&iommu->register_lock, flags); - dmar_writeq(iommu->reg + DMAR_IRTA_REG, - (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); + writeq((addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE, + iommu->reg + DMAR_IRTA_REG); /* Set interrupt-remapping table pointer */ writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG); diff --git a/drivers/iommu/intel/perfmon.c b/drivers/iommu/intel/perfmon.c index 3f75f567f210..eb1df7a9b3c7 100644 --- a/drivers/iommu/intel/perfmon.c +++ b/drivers/iommu/intel/perfmon.c @@ -99,20 +99,20 @@ IOMMU_PMU_ATTR(filter_page_table, "config2:32-36", IOMMU_PMU_FILTER_PAGE_TABLE); #define iommu_pmu_set_filter(_name, _config, _filter, _idx, _econfig) \ { \ if ((iommu_pmu->filter & _filter) && iommu_pmu_en_##_name(_econfig)) { \ - dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET + \ - IOMMU_PMU_CFG_SIZE + \ - (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET, \ - iommu_pmu_get_##_name(_config) | IOMMU_PMU_FILTER_EN);\ + writel(iommu_pmu_get_##_name(_config) | IOMMU_PMU_FILTER_EN, \ + iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET + \ + IOMMU_PMU_CFG_SIZE + \ + (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET); \ } \ } #define iommu_pmu_clear_filter(_filter, _idx) \ { \ if (iommu_pmu->filter & _filter) { \ - dmar_writel(iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET + \ - IOMMU_PMU_CFG_SIZE + \ - (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET, \ - 0); \ + writel(0, \ + iommu_pmu->cfg_reg + _idx * IOMMU_PMU_CFG_OFFSET + \ + IOMMU_PMU_CFG_SIZE + \ + (ffs(_filter) - 1) * IOMMU_PMU_CFG_FILTERS_OFFSET); \ } \ } @@ -411,7 +411,7 @@ static int iommu_pmu_assign_event(struct iommu_pmu *iommu_pmu, hwc->idx = idx; /* config events */ - dmar_writeq(iommu_config_base(iommu_pmu, idx), hwc->config); + writeq(hwc->config, iommu_config_base(iommu_pmu, idx)); iommu_pmu_set_filter(requester_id, event->attr.config1, IOMMU_PMU_FILTER_REQUESTER_ID, idx, @@ -510,7 +510,7 @@ static void iommu_pmu_counter_overflow(struct iommu_pmu *iommu_pmu) iommu_pmu_event_update(event); } - dmar_writeq(iommu_pmu->overflow, status); + writeq(status, iommu_pmu->overflow); } } @@ -524,7 +524,7 @@ static irqreturn_t iommu_pmu_irq_handler(int irq, void *dev_id) iommu_pmu_counter_overflow(iommu->pmu); /* Clear the status bit */ - dmar_writel(iommu->reg + DMAR_PERFINTRSTS_REG, DMA_PERFINTRSTS_PIS); + writel(DMA_PERFINTRSTS_PIS, iommu->reg + DMAR_PERFINTRSTS_REG); return IRQ_HANDLED; } diff --git a/drivers/iommu/intel/prq.c b/drivers/iommu/intel/prq.c index c28fbd5c14a7..1460b57db129 100644 --- a/drivers/iommu/intel/prq.c +++ b/drivers/iommu/intel/prq.c @@ -259,7 +259,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) head = (head + sizeof(*req)) & PRQ_RING_MASK; } - dmar_writeq(iommu->reg + DMAR_PQH_REG, tail); + writeq(tail, iommu->reg + DMAR_PQH_REG); /* * Clear the page request overflow bit and wake up all threads that @@ -325,9 +325,9 @@ int intel_iommu_enable_prq(struct intel_iommu *iommu) iommu->name); goto free_iopfq; } - dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); - dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); - dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER); + writeq(0ULL, iommu->reg + DMAR_PQH_REG); + writeq(0ULL, iommu->reg + DMAR_PQT_REG); + writeq(virt_to_phys(iommu->prq) | PRQ_ORDER, iommu->reg + DMAR_PQA_REG); init_completion(&iommu->prq_complete); @@ -348,9 +348,9 @@ int intel_iommu_enable_prq(struct intel_iommu *iommu) int intel_iommu_finish_prq(struct intel_iommu *iommu) { - dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); - dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); - dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL); + writeq(0ULL, iommu->reg + DMAR_PQH_REG); + writeq(0ULL, iommu->reg + DMAR_PQT_REG); + writeq(0ULL, iommu->reg + DMAR_PQA_REG); if (iommu->pr_irq) { free_irq(iommu->pr_irq, iommu); From b6fd468a052e43fa4e3a00837fbf44a05cc1ca11 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 2 Apr 2026 14:57:30 +0800 Subject: [PATCH 44/52] iommu/vt-d: Split piotlb invalidation into range and all Currently these call chains are muddled up by using npages=-1, but only one caller has the possibility to do both options. Simplify qi_flush_piotlb() to qi_flush_piotlb_all() since all callers pass npages=-1. Split qi_batch_add_piotlb() into qi_batch_add_piotlb_all() and related helpers. Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/1-v1-f175e27af136+11647-iommupt_inv_vtd_jgg@nvidia.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/cache.c | 20 ++++++++++++------- drivers/iommu/intel/dmar.c | 19 ++++-------------- drivers/iommu/intel/iommu.h | 39 +++++++++++++++++-------------------- drivers/iommu/intel/pasid.c | 6 +++--- drivers/iommu/intel/prq.c | 2 +- 5 files changed, 39 insertions(+), 47 deletions(-) diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c index 249ab5886c73..3ae0d21ecb9f 100644 --- a/drivers/iommu/intel/cache.c +++ b/drivers/iommu/intel/cache.c @@ -330,15 +330,17 @@ static void qi_batch_add_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid qi_batch_increment_index(iommu, batch); } +static void qi_batch_add_piotlb_all(struct intel_iommu *iommu, u16 did, + u32 pasid, struct qi_batch *batch) +{ + qi_desc_piotlb_all(did, pasid, &batch->descs[batch->index]); + qi_batch_increment_index(iommu, batch); +} + static void qi_batch_add_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, unsigned long npages, bool ih, struct qi_batch *batch) { - /* - * npages == -1 means a PASID-selective invalidation, otherwise, - * a positive value for Page-selective-within-PASID invalidation. - * 0 is not a valid input. - */ if (!npages) return; @@ -378,8 +380,12 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag * u64 type = DMA_TLB_PSI_FLUSH; if (intel_domain_use_piotlb(domain)) { - qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr, - pages, ih, domain->qi_batch); + if (pages == -1) + qi_batch_add_piotlb_all(iommu, tag->domain_id, + tag->pasid, domain->qi_batch); + else + qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, + addr, pages, ih, domain->qi_batch); return; } diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index b958f2e6042b..b6015f3dc6db 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1551,23 +1551,12 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, qi_submit_sync(iommu, &desc, 1, 0); } -/* PASID-based IOTLB invalidation */ -void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, - unsigned long npages, bool ih) +/* PASID-selective IOTLB invalidation */ +void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid) { - struct qi_desc desc = {.qw2 = 0, .qw3 = 0}; + struct qi_desc desc = {}; - /* - * npages == -1 means a PASID-selective invalidation, otherwise, - * a positive value for Page-selective-within-PASID invalidation. - * 0 is not a valid input. - */ - if (WARN_ON(!npages)) { - pr_err("Invalid input npages = %ld\n", npages); - return; - } - - qi_desc_piotlb(did, pasid, addr, npages, ih, &desc); + qi_desc_piotlb_all(did, pasid, &desc); qi_submit_sync(iommu, &desc, 1, 0); } diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 10331364c0ef..9b193bbcfd58 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1077,31 +1077,29 @@ static inline void qi_desc_dev_iotlb(u16 sid, u16 pfsid, u16 qdep, u64 addr, desc->qw3 = 0; } +/* PASID-selective IOTLB invalidation */ +static inline void qi_desc_piotlb_all(u16 did, u32 pasid, struct qi_desc *desc) +{ + desc->qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) | + QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | QI_EIOTLB_TYPE; + desc->qw1 = 0; +} + +/* Page-selective-within-PASID IOTLB invalidation */ static inline void qi_desc_piotlb(u16 did, u32 pasid, u64 addr, unsigned long npages, bool ih, struct qi_desc *desc) { - if (npages == -1) { - desc->qw0 = QI_EIOTLB_PASID(pasid) | - QI_EIOTLB_DID(did) | - QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | - QI_EIOTLB_TYPE; - desc->qw1 = 0; - } else { - int mask = ilog2(__roundup_pow_of_two(npages)); - unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask)); + int mask = ilog2(__roundup_pow_of_two(npages)); + unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask)); - if (WARN_ON_ONCE(!IS_ALIGNED(addr, align))) - addr = ALIGN_DOWN(addr, align); + if (WARN_ON_ONCE(!IS_ALIGNED(addr, align))) + addr = ALIGN_DOWN(addr, align); - desc->qw0 = QI_EIOTLB_PASID(pasid) | - QI_EIOTLB_DID(did) | - QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | - QI_EIOTLB_TYPE; - desc->qw1 = QI_EIOTLB_ADDR(addr) | - QI_EIOTLB_IH(ih) | - QI_EIOTLB_AM(mask); - } + desc->qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) | + QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | QI_EIOTLB_TYPE; + desc->qw1 = QI_EIOTLB_ADDR(addr) | QI_EIOTLB_IH(ih) | + QI_EIOTLB_AM(mask); } static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid, @@ -1163,8 +1161,7 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, u16 qdep, u64 addr, unsigned mask); -void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, - unsigned long npages, bool ih); +void qi_flush_piotlb_all(struct intel_iommu *iommu, u16 did, u32 pasid); void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid, u32 pasid, u16 qdep, u64 addr, diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 9d30015b8940..89541b74ab8c 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -282,7 +282,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, pasid_cache_invalidation_with_pasid(iommu, did, pasid); if (pgtt == PASID_ENTRY_PGTT_PT || pgtt == PASID_ENTRY_PGTT_FL_ONLY) - qi_flush_piotlb(iommu, did, pasid, 0, -1, 0); + qi_flush_piotlb_all(iommu, did, pasid); else iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); @@ -308,7 +308,7 @@ static void pasid_flush_caches(struct intel_iommu *iommu, if (cap_caching_mode(iommu->cap)) { pasid_cache_invalidation_with_pasid(iommu, did, pasid); - qi_flush_piotlb(iommu, did, pasid, 0, -1, 0); + qi_flush_piotlb_all(iommu, did, pasid); } else { iommu_flush_write_buffer(iommu); } @@ -342,7 +342,7 @@ static void intel_pasid_flush_present(struct intel_iommu *iommu, * Addr[63:12]=0x7FFFFFFF_FFFFF) to affected functions */ pasid_cache_invalidation_with_pasid(iommu, did, pasid); - qi_flush_piotlb(iommu, did, pasid, 0, -1, 0); + qi_flush_piotlb_all(iommu, did, pasid); devtlb_invalidation_with_pasid(iommu, dev, pasid); } diff --git a/drivers/iommu/intel/prq.c b/drivers/iommu/intel/prq.c index 1460b57db129..586055e51bb2 100644 --- a/drivers/iommu/intel/prq.c +++ b/drivers/iommu/intel/prq.c @@ -113,7 +113,7 @@ void intel_iommu_drain_pasid_prq(struct device *dev, u32 pasid) qi_desc_dev_iotlb(sid, info->pfsid, info->ats_qdep, 0, MAX_AGAW_PFN_WIDTH, &desc[2]); } else { - qi_desc_piotlb(did, pasid, 0, -1, 0, &desc[1]); + qi_desc_piotlb_all(did, pasid, &desc[1]); qi_desc_dev_iotlb_pasid(sid, info->pfsid, pasid, info->ats_qdep, 0, MAX_AGAW_PFN_WIDTH, &desc[2]); } From e36ee89679f4869c6deffd5392967820464dcbc6 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 2 Apr 2026 14:57:31 +0800 Subject: [PATCH 45/52] iommu/vt-d: Pass size_order to qi_desc_piotlb() not npages It doesn't make sense for the caller to compute mask, throw it away and then have qi_desc_piotlb() compute it again. Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/2-v1-f175e27af136+11647-iommupt_inv_vtd_jgg@nvidia.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/cache.c | 10 ++++------ drivers/iommu/intel/iommu.h | 13 +++++-------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c index 3ae0d21ecb9f..20df2c16475b 100644 --- a/drivers/iommu/intel/cache.c +++ b/drivers/iommu/intel/cache.c @@ -338,13 +338,11 @@ static void qi_batch_add_piotlb_all(struct intel_iommu *iommu, u16 did, } static void qi_batch_add_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, - u64 addr, unsigned long npages, bool ih, + u64 addr, unsigned int size_order, bool ih, struct qi_batch *batch) { - if (!npages) - return; - - qi_desc_piotlb(did, pasid, addr, npages, ih, &batch->descs[batch->index]); + qi_desc_piotlb(did, pasid, addr, size_order, ih, + &batch->descs[batch->index]); qi_batch_increment_index(iommu, batch); } @@ -385,7 +383,7 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag * tag->pasid, domain->qi_batch); else qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, - addr, pages, ih, domain->qi_batch); + addr, mask, ih, domain->qi_batch); return; } diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 9b193bbcfd58..ef145560aa98 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1087,19 +1087,16 @@ static inline void qi_desc_piotlb_all(u16 did, u32 pasid, struct qi_desc *desc) /* Page-selective-within-PASID IOTLB invalidation */ static inline void qi_desc_piotlb(u16 did, u32 pasid, u64 addr, - unsigned long npages, bool ih, + unsigned int size_order, bool ih, struct qi_desc *desc) { - int mask = ilog2(__roundup_pow_of_two(npages)); - unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask)); - - if (WARN_ON_ONCE(!IS_ALIGNED(addr, align))) - addr = ALIGN_DOWN(addr, align); - + /* + * calculate_psi_aligned_address() must be used for addr and size_order + */ desc->qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) | QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | QI_EIOTLB_TYPE; desc->qw1 = QI_EIOTLB_ADDR(addr) | QI_EIOTLB_IH(ih) | - QI_EIOTLB_AM(mask); + QI_EIOTLB_AM(size_order); } static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid, From b334d7f7e230ff742993629a7a5181f33d9c54af Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 2 Apr 2026 14:57:32 +0800 Subject: [PATCH 46/52] iommu/vt-d: Remove the remaining pages along the invalidation path This was only being used to signal that a flush all should be used. Use mask/size_order >= 52 to signal this instead. Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/3-v1-f175e27af136+11647-iommupt_inv_vtd_jgg@nvidia.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel/cache.c | 27 +++++++++++---------------- drivers/iommu/intel/trace.h | 18 ++++++++---------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c index 20df2c16475b..be8410f0e841 100644 --- a/drivers/iommu/intel/cache.c +++ b/drivers/iommu/intel/cache.c @@ -255,7 +255,6 @@ void cache_tag_unassign_domain(struct dmar_domain *domain, static unsigned long calculate_psi_aligned_address(unsigned long start, unsigned long end, - unsigned long *_pages, unsigned long *_mask) { unsigned long pages = aligned_nrpages(start, end - start + 1); @@ -281,10 +280,8 @@ static unsigned long calculate_psi_aligned_address(unsigned long start, */ shared_bits = ~(pfn ^ end_pfn) & ~bitmask; mask = shared_bits ? __ffs(shared_bits) : MAX_AGAW_PFN_WIDTH; - aligned_pages = 1UL << mask; } - *_pages = aligned_pages; *_mask = mask; return ALIGN_DOWN(start, VTD_PAGE_SIZE << mask); @@ -371,14 +368,13 @@ static bool intel_domain_use_piotlb(struct dmar_domain *domain) } static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag *tag, - unsigned long addr, unsigned long pages, - unsigned long mask, int ih) + unsigned long addr, unsigned long mask, int ih) { struct intel_iommu *iommu = tag->iommu; u64 type = DMA_TLB_PSI_FLUSH; if (intel_domain_use_piotlb(domain)) { - if (pages == -1) + if (mask >= MAX_AGAW_PFN_WIDTH) qi_batch_add_piotlb_all(iommu, tag->domain_id, tag->pasid, domain->qi_batch); else @@ -392,7 +388,7 @@ static void cache_tag_flush_iotlb(struct dmar_domain *domain, struct cache_tag * * is too big. */ if (!cap_pgsel_inv(iommu->cap) || - mask > cap_max_amask_val(iommu->cap) || pages == -1) { + mask > cap_max_amask_val(iommu->cap)) { addr = 0; mask = 0; ih = 0; @@ -441,16 +437,15 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start, unsigned long end, int ih) { struct intel_iommu *iommu = NULL; - unsigned long pages, mask, addr; + unsigned long mask, addr; struct cache_tag *tag; unsigned long flags; if (start == 0 && end == ULONG_MAX) { addr = 0; - pages = -1; mask = MAX_AGAW_PFN_WIDTH; } else { - addr = calculate_psi_aligned_address(start, end, &pages, &mask); + addr = calculate_psi_aligned_address(start, end, &mask); } spin_lock_irqsave(&domain->cache_lock, flags); @@ -462,7 +457,7 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start, switch (tag->type) { case CACHE_TAG_IOTLB: case CACHE_TAG_NESTING_IOTLB: - cache_tag_flush_iotlb(domain, tag, addr, pages, mask, ih); + cache_tag_flush_iotlb(domain, tag, addr, mask, ih); break; case CACHE_TAG_NESTING_DEVTLB: /* @@ -480,7 +475,7 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start, break; } - trace_cache_tag_flush_range(tag, start, end, addr, pages, mask); + trace_cache_tag_flush_range(tag, start, end, addr, mask); } qi_batch_flush_descs(iommu, domain->qi_batch); spin_unlock_irqrestore(&domain->cache_lock, flags); @@ -510,11 +505,11 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start, unsigned long end) { struct intel_iommu *iommu = NULL; - unsigned long pages, mask, addr; + unsigned long mask, addr; struct cache_tag *tag; unsigned long flags; - addr = calculate_psi_aligned_address(start, end, &pages, &mask); + addr = calculate_psi_aligned_address(start, end, &mask); spin_lock_irqsave(&domain->cache_lock, flags); list_for_each_entry(tag, &domain->cache_tags, node) { @@ -530,9 +525,9 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start, if (tag->type == CACHE_TAG_IOTLB || tag->type == CACHE_TAG_NESTING_IOTLB) - cache_tag_flush_iotlb(domain, tag, addr, pages, mask, 0); + cache_tag_flush_iotlb(domain, tag, addr, mask, 0); - trace_cache_tag_flush_range_np(tag, start, end, addr, pages, mask); + trace_cache_tag_flush_range_np(tag, start, end, addr, mask); } qi_batch_flush_descs(iommu, domain->qi_batch); spin_unlock_irqrestore(&domain->cache_lock, flags); diff --git a/drivers/iommu/intel/trace.h b/drivers/iommu/intel/trace.h index 6311ba3f1691..9f0ab43539ea 100644 --- a/drivers/iommu/intel/trace.h +++ b/drivers/iommu/intel/trace.h @@ -132,8 +132,8 @@ DEFINE_EVENT(cache_tag_log, cache_tag_unassign, DECLARE_EVENT_CLASS(cache_tag_flush, TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end, - unsigned long addr, unsigned long pages, unsigned long mask), - TP_ARGS(tag, start, end, addr, pages, mask), + unsigned long addr, unsigned long mask), + TP_ARGS(tag, start, end, addr, mask), TP_STRUCT__entry( __string(iommu, tag->iommu->name) __string(dev, dev_name(tag->dev)) @@ -143,7 +143,6 @@ DECLARE_EVENT_CLASS(cache_tag_flush, __field(unsigned long, start) __field(unsigned long, end) __field(unsigned long, addr) - __field(unsigned long, pages) __field(unsigned long, mask) ), TP_fast_assign( @@ -155,10 +154,9 @@ DECLARE_EVENT_CLASS(cache_tag_flush, __entry->start = start; __entry->end = end; __entry->addr = addr; - __entry->pages = pages; __entry->mask = mask; ), - TP_printk("%s %s[%d] type %s did %d [0x%lx-0x%lx] addr 0x%lx pages 0x%lx mask 0x%lx", + TP_printk("%s %s[%d] type %s did %d [0x%lx-0x%lx] addr 0x%lx mask 0x%lx", __get_str(iommu), __get_str(dev), __entry->pasid, __print_symbolic(__entry->type, { CACHE_TAG_IOTLB, "iotlb" }, @@ -166,20 +164,20 @@ DECLARE_EVENT_CLASS(cache_tag_flush, { CACHE_TAG_NESTING_IOTLB, "nesting_iotlb" }, { CACHE_TAG_NESTING_DEVTLB, "nesting_devtlb" }), __entry->domain_id, __entry->start, __entry->end, - __entry->addr, __entry->pages, __entry->mask + __entry->addr, __entry->mask ) ); DEFINE_EVENT(cache_tag_flush, cache_tag_flush_range, TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end, - unsigned long addr, unsigned long pages, unsigned long mask), - TP_ARGS(tag, start, end, addr, pages, mask) + unsigned long addr, unsigned long mask), + TP_ARGS(tag, start, end, addr, mask) ); DEFINE_EVENT(cache_tag_flush, cache_tag_flush_range_np, TP_PROTO(struct cache_tag *tag, unsigned long start, unsigned long end, - unsigned long addr, unsigned long pages, unsigned long mask), - TP_ARGS(tag, start, end, addr, pages, mask) + unsigned long addr, unsigned long mask), + TP_ARGS(tag, start, end, addr, mask) ); #endif /* _TRACE_INTEL_IOMMU_H */ From faad224fe0f0857a04ff2eb3c90f0de57f47d0f3 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 1 Apr 2026 08:00:17 +0000 Subject: [PATCH 47/52] iommu/amd: Fix clone_alias() to use the original device's devid Currently clone_alias() assumes first argument (pdev) is always the original device pointer. This function is called by pci_for_each_dma_alias() which based on topology decides to send original or alias device details in first argument. This meant that the source devid used to look up and copy the DTE may be incorrect, leading to wrong or stale DTE entries being propagated to alias device. Fix this by passing the original pdev as the opaque data argument to both the direct clone_alias() call and pci_for_each_dma_alias(). Inside clone_alias(), retrieve the original device from data and compute devid from it. Fixes: 3332364e4ebc ("iommu/amd: Support multiple PCI DMA aliases in device table") Signed-off-by: Vasant Hegde Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 81c4d7733872..b6acb519fea5 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -403,11 +403,12 @@ struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid) return NULL; } -static int clone_alias(struct pci_dev *pdev, u16 alias, void *data) +static int clone_alias(struct pci_dev *pdev_origin, u16 alias, void *data) { struct dev_table_entry new; struct amd_iommu *iommu; struct iommu_dev_data *dev_data, *alias_data; + struct pci_dev *pdev = data; u16 devid = pci_dev_id(pdev); int ret = 0; @@ -454,9 +455,9 @@ static void clone_aliases(struct amd_iommu *iommu, struct device *dev) * part of the PCI DMA aliases if it's bus differs * from the original device. */ - clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL); + clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], pdev); - pci_for_each_dma_alias(pdev, clone_alias, NULL); + pci_for_each_dma_alias(pdev, clone_alias, pdev); } static void setup_aliases(struct amd_iommu *iommu, struct device *dev) From 40a13b49957937427bc23e78eb50679df4396a47 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Fri, 27 Mar 2026 12:22:10 -0300 Subject: [PATCH 48/52] iommu/riscv: Remove overflows on the invalidation path Since RISC-V supports a sign extended page table it should support a gather->end of ULONG_MAX, but if this happens it will infinite loop because of the overflow. Also avoid overflow computing the length by moving the +1 to the other side of the < Fixes: 488ffbf18171 ("iommu/riscv: Paging domain support") Signed-off-by: Jason Gunthorpe Signed-off-by: Joerg Roedel --- drivers/iommu/riscv/iommu.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index 6ac7e3edef8a..3ec99c979d47 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -931,8 +931,6 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, struct riscv_iommu_bond *bond; struct riscv_iommu_device *iommu, *prev; struct riscv_iommu_command cmd; - unsigned long len = end - start + 1; - unsigned long iova; /* * For each IOMMU linked with this protection domain (via bonds->dev), @@ -975,11 +973,14 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, riscv_iommu_cmd_inval_vma(&cmd); riscv_iommu_cmd_inval_set_pscid(&cmd, domain->pscid); - if (len && len < RISCV_IOMMU_IOTLB_INVAL_LIMIT) { - for (iova = start; iova < end; iova += PAGE_SIZE) { + if (end - start < RISCV_IOMMU_IOTLB_INVAL_LIMIT - 1) { + unsigned long iova = start; + + do { riscv_iommu_cmd_inval_set_addr(&cmd, iova); riscv_iommu_cmd_send(iommu, &cmd); - } + } while (!check_add_overflow(iova, PAGE_SIZE, &iova) && + iova < end); } else { riscv_iommu_cmd_send(iommu, &cmd); } From 5aac28784dca6819e96e5f93e644cdee59e50f6e Mon Sep 17 00:00:00 2001 From: Magnus Kalland Date: Thu, 2 Apr 2026 09:42:50 +0200 Subject: [PATCH 49/52] iommu/amd: Invalidate IRT cache for DMA aliases DMA aliasing causes interrupt remapping table entries (IRTEs) to be shared between multiple device IDs. See commit 3c124435e8dd ("iommu/amd: Support multiple PCI DMA aliases in IRQ Remapping") for more information on this. However, the AMD IOMMU driver currently invalidates IRTE cache entries on a per-device basis whenever an IRTE is updated, not for each alias. This approach leaves stale IRTE cache entries when an IRTE is cached under one DMA alias but later updated and invalidated through a different alias. In such cases, the original device ID is never invalidated, since it is programmed via aliasing. This incoherency bug has been observed when IRTEs are cached for one Non-Transparent Bridge (NTB) DMA alias, later updated via another. Fix this by invalidating the interrupt remapping table cache for all DMA aliases when updating an IRTE. Co-developed-by: Lars B. Kristiansen Signed-off-by: Lars B. Kristiansen Co-developed-by: Jonas Markussen Signed-off-by: Jonas Markussen Co-developed-by: Tore H. Larsen Signed-off-by: Tore H. Larsen Signed-off-by: Magnus Kalland Link: https://lore.kernel.org/linux-iommu/9204da81-f821-4034-b8ad-501e43383b56@amd.com/ Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index b6acb519fea5..340ae2150f91 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -3167,26 +3167,44 @@ const struct iommu_ops amd_iommu_ops = { static struct irq_chip amd_ir_chip; static DEFINE_SPINLOCK(iommu_table_lock); +static int iommu_flush_dev_irt(struct pci_dev *unused, u16 devid, void *data) +{ + int ret; + struct iommu_cmd cmd; + struct amd_iommu *iommu = data; + + build_inv_irt(&cmd, devid); + ret = __iommu_queue_command_sync(iommu, &cmd, true); + return ret; +} + static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) { int ret; u64 data; unsigned long flags; - struct iommu_cmd cmd, cmd2; + struct iommu_cmd cmd; + struct pci_dev *pdev = NULL; + struct iommu_dev_data *dev_data = search_dev_data(iommu, devid); if (iommu->irtcachedis_enabled) return; - build_inv_irt(&cmd, devid); + if (dev_data && dev_data->dev && dev_is_pci(dev_data->dev)) + pdev = to_pci_dev(dev_data->dev); raw_spin_lock_irqsave(&iommu->lock, flags); data = get_cmdsem_val(iommu); - build_completion_wait(&cmd2, iommu, data); + build_completion_wait(&cmd, iommu, data); - ret = __iommu_queue_command_sync(iommu, &cmd, true); + if (pdev) + ret = pci_for_each_dma_alias(pdev, iommu_flush_dev_irt, iommu); + else + ret = iommu_flush_dev_irt(NULL, devid, iommu); if (ret) goto out_err; - ret = __iommu_queue_command_sync(iommu, &cmd2, false); + + ret = __iommu_queue_command_sync(iommu, &cmd, false); if (ret) goto out_err; raw_spin_unlock_irqrestore(&iommu->lock, flags); From 5e8323c3d52838e3b7494062980dba9450636eb4 Mon Sep 17 00:00:00 2001 From: Mukesh Ojha Date: Fri, 3 Apr 2026 13:39:56 +0530 Subject: [PATCH 50/52] dt-bindings: arm-smmu: qcom: Add compatible for Hawi SoC Qualcomm Hawi SoC include apps smmu that implements arm,mmu-500, which is used to translate device-visible virtual addresses to physical addresses. Add compatible for these items. Signed-off-by: Mukesh Ojha Reviewed-by: Krzysztof Kozlowski Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 27d25bc98cbe..06fb5c8e7547 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -93,6 +93,7 @@ properties: items: - enum: - qcom,glymur-smmu-500 + - qcom,hawi-smmu-500 - qcom,kaanapali-smmu-500 - qcom,milos-smmu-500 - qcom,qcm2290-smmu-500 From ebfaf2bcc1902d293ed25f5a0580c96f73c47cbb Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Wed, 8 Apr 2026 12:44:42 -0600 Subject: [PATCH 51/52] iommu/vt-d: Restore IOMMU_CAP_CACHE_COHERENCY In removing IOMMU_CAP_DEFERRED_FLUSH, the below referenced commit was over-eager in removing the return, resulting in the test for IOMMU_CAP_CACHE_COHERENCY falling through to an irrelevant option. Restore dropped return. Fixes: 1c18a1212c77 ("iommu/dma: Always allow DMA-FQ when iommupt provides the iommu_domain") Signed-off-by: Alex Williamson Reviewed-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Signed-off-by: Will Deacon --- drivers/iommu/intel/iommu.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 80b183e207e5..6418f7cb865c 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3212,6 +3212,7 @@ static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap) switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: + return true; case IOMMU_CAP_PRE_BOOT_PROTECTION: return dmar_platform_optin(); case IOMMU_CAP_ENFORCE_CACHE_COHERENCY: From 7e0548525abd2bff9694e016b6a469ccd2d5a053 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 8 Apr 2026 15:40:57 +0100 Subject: [PATCH 52/52] iommu: Ensure .iotlb_sync is called correctly Many drivers have no reason to use the iotlb_gather mechanism, but do still depend on .iotlb_sync being called to properly complete an unmap. Since the core code is now relying on the gather to detect when there is legitimately something to sync, it should also take care of encoding a successful unmap when the driver does not touch the gather itself. Fixes: 90c5def10bea ("iommu: Do not call drivers for empty gathers") Reported-by: Jon Hunter Closes: https://lore.kernel.org/r/8800a38b-8515-4bbe-af15-0dae81274bf7@nvidia.com Signed-off-by: Robin Murphy Tested-by: Jon Hunter Reviewed-by: Jason Gunthorpe Tested-by: Russell King (Oracle) Signed-off-by: Will Deacon --- drivers/iommu/iommu.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 50718ab810a4..ee83850c7060 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2717,6 +2717,12 @@ static size_t __iommu_unmap(struct iommu_domain *domain, pr_debug("unmapped: iova 0x%lx size 0x%zx\n", iova, unmapped_page); + /* + * If the driver itself isn't using the gather, make sure + * it looks non-empty so iotlb_sync will still be called. + */ + if (iotlb_gather->start >= iotlb_gather->end) + iommu_iotlb_gather_add_range(iotlb_gather, iova, size); iova += unmapped_page; unmapped += unmapped_page;