Merge "qcom-iommu-util: Take a snapshot of the qcom-iommu-util module from msm-5.15"

drivers/iommu/Kconfig

@@ -48,6 +48,43 @@ config IOMMU_IO_PGTABLE_LPAE_SELFTEST
 
 	  If unsure, say N here.
 
+config IOMMU_IO_PGTABLE_FAST
+	bool "Fast ARMv7/v8 Long Descriptor Format"
+	depends on (ARM || ARM64) && IOMMU_DMA
+	help
+          Enable support for a subset of the ARM long descriptor pagetable
+	  format.  This allocator achieves fast performance by
+	  pre-allocating and pre-populating page table memory up front.
+	  only supports a 32 bit virtual address space.
+
+          This implementation is mainly optimized for use cases where the
+          buffers are small (<= 64K) since it only supports 4K page sizes.
+
+config IOMMU_IO_PGTABLE_FAST_SELFTEST
+	bool "Fast IO pgtable selftests"
+	depends on IOMMU_IO_PGTABLE_FAST
+	help
+	  Enable self-tests for "fast" page table allocator.
+	  This performs a series of page-table consistency checks
+	  during boot.
+
+	  If unsure, say N here.
+
+config IOMMU_IO_PGTABLE_FAST_PROVE_TLB
+	bool "Prove correctness of TLB maintenance in the Fast DMA mapper"
+	depends on IOMMU_IO_PGTABLE_FAST
+	help
+          Enables some debug features that help prove correctness of TLB
+          maintenance routines in the Fast DMA mapper.  This option will
+          slow things down considerably, so should only be used in a debug
+          configuration.  This relies on the ability to set bits in an
+          invalid page table entry, which is disallowed on some hardware
+          due to errata.  If you're running on such a platform then this
+          option can only be used with unit tests.  It will break real use
+          cases.
+
+	  If unsure, say N here.
+
 config IOMMU_IO_PGTABLE_ARMV7S
 	bool "ARMv7/v8 Short Descriptor Format"
 	select IOMMU_IO_PGTABLE
@@ -290,6 +327,14 @@ config SPAPR_TCE_IOMMU
 	  Enables bits of IOMMU API required by VFIO. The iommu_ops
 	  is not implemented as it is not necessary for VFIO.
 
+config QTI_IOMMU_SUPPORT
+	tristate "Support for QTI iommu drivers"
+	help
+	  The QTI GPU device may switch between multiple iommu domains,
+	  depending on usecase. This introduces a need to track all such
+	  domains in a non-driver specific manner.
+	  If in doubt say N.
+
 config APPLE_DART
 	tristate "Apple DART IOMMU Support"
 	depends on ARCH_APPLE || (COMPILE_TEST && !GENERIC_ATOMIC64)
@@ -318,6 +363,15 @@ config ARM_SMMU
 	  Say Y here if your SoC includes an IOMMU device implementing
 	  the ARM SMMU architecture.
 
+config ARM_SMMU_CONTEXT_FAULT_RETRY
+	bool "Context fault retry sequence"
+	depends on ARM_SMMU && ARCH_WAIPIO
+	help
+	  In some cases, issuing a tlb invalidate operation after a
+	  context fault may cause a subsequent retry of the failing
+	  address to succeed. Only applies to clients which have
+	  stall-on-fault enabled, like display.
+
 config ARM_SMMU_LEGACY_DT_BINDINGS
 	bool "Support the legacy \"mmu-masters\" devicetree bindings"
 	depends on ARM_SMMU=y && OF
@@ -355,6 +409,16 @@ config ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT
 	  'arm-smmu.disable_bypass' will continue to override this
 	  config.
 
+config ARM_SMMU_SELFTEST
+	bool "ARM SMMU self test support"
+	depends on ARM_SMMU
+	help
+	  Enables self tests for arm smmu. Tests basic hardware
+	  configurations like interrupts. Note that enabling this
+	  option can marginally increase the boot time.
+
+	  If unsure, say N here.
+
 config ARM_SMMU_QCOM
 	def_tristate y
 	depends on ARM_SMMU && ARCH_QCOM
@@ -388,6 +452,18 @@ config ARM_SMMU_V3_SVA
 	  Say Y here if your system supports SVA extensions such as PCIe PASID
 	  and PRI.
 
+config QCOM_LAZY_MAPPING
+	tristate "Reference counted iommu-mapping support"
+	depends on QCOM_DMABUF_HEAPS
+	depends on IOMMU_API
+	help
+	  DMA-BUFs may be shared between several software clients.
+	  Reference counting the mapping may simplify coordination between
+	  these clients, and decrease latency by preventing multiple
+	  map/unmaps of the same region.
+
+	  If unsure, say N here.
+
 config S390_IOMMU
 	def_bool y if S390 && PCI
 	depends on S390 && PCI
@@ -474,6 +550,16 @@ config VIRTIO_IOMMU
 
 	  Say Y here if you intend to run this kernel as a guest.
 
+config QCOM_IOMMU_UTIL
+	tristate "Support for qcom additions to the iommu framework"
+	help
+	  QCOM iommu drivers support a general set of functionality in
+	  addition to the functions provided by the iommu framework.
+	  This includes devicetree properties for configuring iommu
+	  groups and iova ranges.
+
+	  Say N here if unsure.
+
 config SPRD_IOMMU
 	tristate "Unisoc IOMMU Support"
 	depends on ARCH_SPRD || COMPILE_TEST
@@ -486,4 +572,14 @@ config SPRD_IOMMU
 
 	  Say Y here if you want to use the multimedia devices listed above.
 
+config QCOM_IOMMU_DEBUG
+	tristate "IOMMU debugging and testing"
+	depends on QCOM_IOMMU_UTIL
+	depends on DEBUG_FS
+	help
+	  This option is used to enable profiling and debugging in
+	  the IOMMU framework code. IOMMU profiling and debugging
+	  can be done through the debugfs nodes which this option
+	  makes available.
+
 endif # IOMMU_SUPPORT

drivers/iommu/Makefile

@@ -5,6 +5,7 @@ obj-$(CONFIG_IOMMU_API) += iommu-traces.o
 obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
 obj-$(CONFIG_IOMMU_DEBUGFS) += iommu-debugfs.o
 obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o
+obj-$(CONFIG_QCOM_LAZY_MAPPING) += msm_dma_iommu_mapping.o
 obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o
 obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o
 obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o
@@ -29,4 +30,11 @@ obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o
 obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o
 obj-$(CONFIG_IOMMU_SVA_LIB) += iommu-sva-lib.o io-pgfault.o
 obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
+obj-$(CONFIG_QCOM_IOMMU_UTIL) += qcom_iommu_util.o
+qcom_iommu_util-y += qcom-iommu-util.o
+qcom_iommu_util-$(CONFIG_IOMMU_IO_PGTABLE_FAST) += qcom-dma-iommu-generic.o io-pgtable-fast.o dma-mapping-fast.o
+qcom_iommu_util-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += qcom-io-pgtable-arm.o qcom-io-pgtable-alloc.o
+obj-$(CONFIG_QTI_IOMMU_SUPPORT) += iommu-logger.o
+obj-$(CONFIG_QCOM_IOMMU_DEBUG) += qcom_iommu_debug.o
+qcom_iommu_debug-y += qcom-iommu-debug.o qcom-iommu-debug-user.o
 obj-$(CONFIG_APPLE_DART) += apple-dart.o

drivers/iommu/io-pgtable-fast.c

@@ -0,0 +1,844 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#define pr_fmt(fmt)	"io-pgtable-fast: " fmt
+
+#include <linux/iommu.h>
+#include <linux/kernel.h>
+#include <linux/scatterlist.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/io-pgtable.h>
+#include <linux/io-pgtable-fast.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/dma-mapping.h>
+#include <linux/qcom-iommu-util.h>
+#include "qcom-io-pgtable.h"
+
+#define AV8L_FAST_MAX_ADDR_BITS		48
+
+/* Page table bits */
+#define AV8L_FAST_PTE_TYPE_SHIFT	0
+#define AV8L_FAST_PTE_TYPE_MASK		0x3
+
+#define AV8L_FAST_PTE_TYPE_BLOCK	1
+#define AV8L_FAST_PTE_TYPE_TABLE	3
+#define AV8L_FAST_PTE_TYPE_PAGE		3
+
+#define AV8L_FAST_PTE_NSTABLE		(((av8l_fast_iopte)1) << 63)
+#define AV8L_FAST_PTE_XN		(((av8l_fast_iopte)3) << 53)
+#define AV8L_FAST_PTE_AF		(((av8l_fast_iopte)1) << 10)
+#define AV8L_FAST_PTE_SH_NS		(((av8l_fast_iopte)0) << 8)
+#define AV8L_FAST_PTE_SH_OS		(((av8l_fast_iopte)2) << 8)
+#define AV8L_FAST_PTE_SH_IS		(((av8l_fast_iopte)3) << 8)
+#define AV8L_FAST_PTE_SH_MASK		(((av8l_fast_iopte)3) << 8)
+#define AV8L_FAST_PTE_NS		(((av8l_fast_iopte)1) << 5)
+#define AV8L_FAST_PTE_VALID		(((av8l_fast_iopte)1) << 0)
+
+#define AV8L_FAST_PTE_ATTR_LO_MASK	(((av8l_fast_iopte)0x3ff) << 2)
+/* Ignore the contiguous bit for block splitting */
+#define AV8L_FAST_PTE_ATTR_HI_MASK	(((av8l_fast_iopte)6) << 52)
+#define AV8L_FAST_PTE_ATTR_MASK		(AV8L_FAST_PTE_ATTR_LO_MASK |	\
+					 AV8L_FAST_PTE_ATTR_HI_MASK)
+#define AV8L_FAST_PTE_ADDR_MASK		((av8l_fast_iopte)0xfffffffff000)
+
+
+/* Stage-1 PTE */
+#define AV8L_FAST_PTE_AP_PRIV_RW	(((av8l_fast_iopte)0) << 6)
+#define AV8L_FAST_PTE_AP_RW		(((av8l_fast_iopte)1) << 6)
+#define AV8L_FAST_PTE_AP_PRIV_RO	(((av8l_fast_iopte)2) << 6)
+#define AV8L_FAST_PTE_AP_RO		(((av8l_fast_iopte)3) << 6)
+#define AV8L_FAST_PTE_ATTRINDX_SHIFT	2
+#define AV8L_FAST_PTE_ATTRINDX_MASK	0x7
+#define AV8L_FAST_PTE_nG		(((av8l_fast_iopte)1) << 11)
+
+/* Stage-2 PTE */
+#define AV8L_FAST_PTE_HAP_FAULT		(((av8l_fast_iopte)0) << 6)
+#define AV8L_FAST_PTE_HAP_READ		(((av8l_fast_iopte)1) << 6)
+#define AV8L_FAST_PTE_HAP_WRITE		(((av8l_fast_iopte)2) << 6)
+#define AV8L_FAST_PTE_MEMATTR_OIWB	(((av8l_fast_iopte)0xf) << 2)
+#define AV8L_FAST_PTE_MEMATTR_NC	(((av8l_fast_iopte)0x5) << 2)
+#define AV8L_FAST_PTE_MEMATTR_DEV	(((av8l_fast_iopte)0x1) << 2)
+
+/* Register bits */
+#define ARM_32_LPAE_TCR_EAE		(1 << 31)
+#define ARM_64_LPAE_S2_TCR_RES1		(1 << 31)
+
+#define AV8L_FAST_TCR_TG0_4K		(0 << 14)
+#define AV8L_FAST_TCR_TG0_64K		(1 << 14)
+#define AV8L_FAST_TCR_TG0_16K		(2 << 14)
+
+#define AV8L_FAST_TCR_SH0_SHIFT		12
+#define AV8L_FAST_TCR_SH0_MASK		0x3
+#define AV8L_FAST_TCR_SH_NS		0
+#define AV8L_FAST_TCR_SH_OS		2
+#define AV8L_FAST_TCR_SH_IS		3
+
+#define AV8L_FAST_TCR_ORGN0_SHIFT	10
+#define AV8L_FAST_TCR_IRGN0_SHIFT	8
+#define AV8L_FAST_TCR_RGN_MASK		0x3
+#define AV8L_FAST_TCR_RGN_NC		0
+#define AV8L_FAST_TCR_RGN_WBWA		1
+#define AV8L_FAST_TCR_RGN_WT		2
+#define AV8L_FAST_TCR_RGN_WB		3
+
+#define AV8L_FAST_TCR_SL0_SHIFT		6
+#define AV8L_FAST_TCR_SL0_MASK		0x3
+
+#define AV8L_FAST_TCR_T0SZ_SHIFT	0
+#define AV8L_FAST_TCR_SZ_MASK		0xf
+
+#define AV8L_FAST_TCR_PS_SHIFT		16
+#define AV8L_FAST_TCR_PS_MASK		0x7
+
+#define AV8L_FAST_TCR_IPS_SHIFT		32
+#define AV8L_FAST_TCR_IPS_MASK		0x7
+
+#define AV8L_FAST_TCR_PS_32_BIT		0x0ULL
+#define AV8L_FAST_TCR_PS_36_BIT		0x1ULL
+#define AV8L_FAST_TCR_PS_40_BIT		0x2ULL
+#define AV8L_FAST_TCR_PS_42_BIT		0x3ULL
+#define AV8L_FAST_TCR_PS_44_BIT		0x4ULL
+#define AV8L_FAST_TCR_PS_48_BIT		0x5ULL
+
+#define AV8L_FAST_TCR_EPD1_SHIFT	23
+#define AV8L_FAST_TCR_EPD1_FAULT	1
+
+#define AV8L_FAST_MAIR_ATTR_SHIFT(n)	((n) << 3)
+#define AV8L_FAST_MAIR_ATTR_MASK	0xff
+#define AV8L_FAST_MAIR_ATTR_DEVICE	0x04
+#define AV8L_FAST_MAIR_ATTR_NC		0x44
+#define AV8L_FAST_MAIR_ATTR_WBRWA	0xff
+#define AV8L_FAST_MAIR_ATTR_UPSTREAM	0xf4
+#define AV8L_FAST_MAIR_ATTR_IDX_NC	0
+#define AV8L_FAST_MAIR_ATTR_IDX_CACHE	1
+#define AV8L_FAST_MAIR_ATTR_IDX_DEV	2
+#define AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM	3
+
+#define AV8L_FAST_PAGE_SHIFT		12
+
+#define PTE_MAIR_IDX(pte)				\
+	((pte >> AV8L_FAST_PTE_ATTRINDX_SHIFT) &	\
+	 AV8L_FAST_PTE_ATTRINDX_MASK)
+
+#define PTE_SH_IDX(pte) (pte & AV8L_FAST_PTE_SH_MASK)
+
+#define iopte_pmd_offset(pmds, base, iova) (pmds + ((iova - base) >> 12))
+
+static inline dma_addr_t av8l_dma_addr(void *addr)
+{
+	if (is_vmalloc_addr(addr))
+		return page_to_phys(vmalloc_to_page(addr)) +
+			offset_in_page(addr);
+	return virt_to_phys(addr);
+}
+
+static void __av8l_clean_range(struct device *dev, void *start, void *end)
+{
+	size_t size;
+	void *region_end;
+	unsigned long page_end;
+
+	if (is_vmalloc_addr(start)) {
+		while (start < end) {
+			page_end = round_down((unsigned long)start + PAGE_SIZE,
+					      PAGE_SIZE);
+			region_end = min_t(void *, end, page_end);
+			size = region_end - start;
+			dma_sync_single_for_device(dev, av8l_dma_addr(start),
+						   size, DMA_TO_DEVICE);
+			start = region_end;
+		}
+	} else {
+		size = end - start;
+		dma_sync_single_for_device(dev, av8l_dma_addr(start), size,
+					   DMA_TO_DEVICE);
+	}
+}
+
+static void av8l_clean_range(struct io_pgtable_cfg *cfg, av8l_fast_iopte *start,
+			     av8l_fast_iopte *end)
+{
+	if (!cfg->coherent_walk)
+		__av8l_clean_range(cfg->iommu_dev, start, end);
+}
+
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB
+
+#include <linux/notifier.h>
+
+static ATOMIC_NOTIFIER_HEAD(av8l_notifier_list);
+
+void av8l_register_notify(struct notifier_block *nb)
+{
+	atomic_notifier_chain_register(&av8l_notifier_list, nb);
+}
+EXPORT_SYMBOL(av8l_register_notify);
+
+static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep)
+{
+	if (unlikely(*ptep)) {
+		atomic_notifier_call_chain(
+			&av8l_notifier_list, MAPPED_OVER_STALE_TLB,
+			(void *) ptep);
+		pr_err("Tried to map over a non-vacant pte: 0x%llx @ %p\n",
+		       *ptep, ptep);
+		pr_err("Nearby memory:\n");
+		print_hex_dump(KERN_ERR, "pgtbl: ", DUMP_PREFIX_ADDRESS,
+			       32, 8, ptep - 16, 32 * sizeof(*ptep), false);
+	}
+}
+
+void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, u64 base,
+		u64 end, bool skip_sync)
+{
+	int i;
+	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
+	struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops);
+	av8l_fast_iopte *pmdp = iopte_pmd_offset(data->pmds, data->base, base);
+
+	for (i = base >> AV8L_FAST_PAGE_SHIFT;
+			i <= (end >> AV8L_FAST_PAGE_SHIFT); ++i) {
+		if (!(*pmdp & AV8L_FAST_PTE_VALID)) {
+			*pmdp = 0;
+			if (!skip_sync)
+				av8l_clean_range(&iop->cfg, pmdp, pmdp + 1);
+		}
+		pmdp++;
+	}
+}
+#else
+static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep)
+{
+}
+#endif
+
+static av8l_fast_iopte
+av8l_fast_prot_to_pte(struct av8l_fast_io_pgtable *data, int prot)
+{
+	av8l_fast_iopte pte = AV8L_FAST_PTE_XN
+		| AV8L_FAST_PTE_TYPE_PAGE
+		| AV8L_FAST_PTE_AF
+		| AV8L_FAST_PTE_nG
+		| AV8L_FAST_PTE_SH_OS;
+
+	if (prot & IOMMU_MMIO)
+		pte |= (AV8L_FAST_MAIR_ATTR_IDX_DEV
+			<< AV8L_FAST_PTE_ATTRINDX_SHIFT);
+	else if (prot & IOMMU_CACHE)
+		pte |= (AV8L_FAST_MAIR_ATTR_IDX_CACHE
+			<< AV8L_FAST_PTE_ATTRINDX_SHIFT);
+	else if (prot & IOMMU_SYS_CACHE)
+		pte |= (AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM
+			<< AV8L_FAST_PTE_ATTRINDX_SHIFT);
+	if (!(prot & IOMMU_WRITE))
+		pte |= AV8L_FAST_PTE_AP_RO;
+	else
+		pte |= AV8L_FAST_PTE_AP_RW;
+
+	return pte;
+}
+
+static int av8l_fast_map(struct io_pgtable_ops *ops, unsigned long iova,
+			 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
+{
+	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
+	struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops);
+	av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova);
+	unsigned long i, nptes = size >> AV8L_FAST_PAGE_SHIFT;
+	av8l_fast_iopte pte;
+
+	pte = av8l_fast_prot_to_pte(data, prot);
+	paddr &= AV8L_FAST_PTE_ADDR_MASK;
+	for (i = 0; i < nptes; i++, paddr += SZ_4K) {
+		__av8l_check_for_stale_tlb(ptep + i);
+		*(ptep + i) = pte | paddr;
+	}
+	av8l_clean_range(&iop->cfg, ptep, ptep + nptes);
+
+	return 0;
+}
+
+int av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova,
+			 phys_addr_t paddr, size_t size, int prot)
+{
+	return av8l_fast_map(ops, iova, paddr, size, prot, GFP_ATOMIC);
+}
+
+static int av8l_fast_map_pages(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr,
+			       size_t pgsize, size_t pgcount, int prot, gfp_t gfp,
+			       size_t *mapped)
+{
+	int ret = av8l_fast_map(ops, iova, paddr, pgsize * pgcount, prot, gfp);
+
+	if (!ret)
+		*mapped = pgsize * pgcount;
+
+	return ret;
+}
+
+static size_t
+__av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova,
+			size_t size, bool allow_stale_tlb)
+{
+	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
+	struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops);
+	unsigned long nptes;
+	av8l_fast_iopte *ptep;
+	int val = allow_stale_tlb
+		? AV8L_FAST_PTE_UNMAPPED_NEED_TLBI
+		: 0;
+
+	ptep = iopte_pmd_offset(data->pmds, data->base, iova);
+	nptes = size >> AV8L_FAST_PAGE_SHIFT;
+
+	memset(ptep, val, sizeof(*ptep) * nptes);
+	av8l_clean_range(&iop->cfg, ptep, ptep + nptes);
+	if (!allow_stale_tlb)
+		io_pgtable_tlb_flush_all(&data->iop);
+
+	return size;
+}
+
+/* caller must take care of tlb cache maintenance */
+void av8l_fast_unmap_public(struct io_pgtable_ops *ops, unsigned long iova,
+				size_t size)
+{
+	__av8l_fast_unmap(ops, iova, size, true);
+}
+
+static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova,
+			      size_t size, struct iommu_iotlb_gather *gather)
+{
+	return __av8l_fast_unmap(ops, iova, size, false);
+}
+
+static size_t av8l_fast_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, size_t pgsize,
+				    size_t pgcount, struct iommu_iotlb_gather *gather)
+{
+	return __av8l_fast_unmap(ops, iova, pgsize * pgcount, false);
+}
+
+/* TODO: Add this back in android-mainline */
+static int __maybe_unused av8l_fast_map_sg(struct io_pgtable_ops *ops,
+			unsigned long iova, struct scatterlist *sgl,
+			unsigned int nents, int prot, gfp_t gfp, size_t *mapped)
+{
+	struct scatterlist *sg;
+	int i;
+
+	for_each_sg(sgl, sg, nents, i) {
+		av8l_fast_map(ops, iova, sg_phys(sg), sg->length, prot, gfp);
+		iova += sg->length;
+		*mapped += sg->length;
+	}
+
+	return 0;
+}
+
+int av8l_fast_map_sg_public(struct io_pgtable_ops *ops,
+			    unsigned long iova, struct scatterlist *sgl,
+			    unsigned int nents, int prot, size_t *mapped)
+{
+	return av8l_fast_map_sg(ops, iova, sgl, nents, prot, GFP_ATOMIC, mapped);
+}
+
+#if defined(CONFIG_ARM64)
+#define FAST_PGDNDX(va) (((va) & 0x7fc0000000) >> 27)
+#elif defined(CONFIG_ARM)
+#define FAST_PGDNDX(va) (((va) & 0xc0000000) >> 27)
+#endif
+
+static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops,
+					  unsigned long iova)
+{
+	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
+	av8l_fast_iopte pte, *pgdp, *pudp, *pmdp;
+	unsigned long pgd;
+	phys_addr_t phys;
+	const unsigned long pts = AV8L_FAST_PTE_TYPE_SHIFT;
+	const unsigned long ptm = AV8L_FAST_PTE_TYPE_MASK;
+	const unsigned long ptt = AV8L_FAST_PTE_TYPE_TABLE;
+	const unsigned long ptp = AV8L_FAST_PTE_TYPE_PAGE;
+	const av8l_fast_iopte am = AV8L_FAST_PTE_ADDR_MASK;
+
+	/* TODO: clean up some of these magic numbers... */
+
+	pgd = (unsigned long)data->pgd | FAST_PGDNDX(iova);
+	pgdp = (av8l_fast_iopte *)pgd;
+
+	pte = *pgdp;
+	if (((pte >> pts) & ptm) != ptt)
+		return 0;
+	pudp = phys_to_virt((pte & am) | ((iova & 0x3fe00000) >> 18));
+
+	pte = *pudp;
+	if (((pte >> pts) & ptm) != ptt)
+		return 0;
+	pmdp = phys_to_virt((pte & am) | ((iova & 0x1ff000) >> 9));
+
+	pte = *pmdp;
+	if (((pte >> pts) & ptm) != ptp)
+		return 0;
+	phys = pte & am;
+
+	return phys | (iova & 0xfff);
+}
+
+phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops,
+					  unsigned long iova)
+{
+	return av8l_fast_iova_to_phys(ops, iova);
+}
+
+static bool av8l_fast_iova_coherent(struct io_pgtable_ops *ops,
+					unsigned long iova)
+{
+	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
+	av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova);
+
+	return ((PTE_MAIR_IDX(*ptep) == AV8L_FAST_MAIR_ATTR_IDX_CACHE) &&
+		((PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_OS) ||
+		 (PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_IS)));
+}
+
+bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops,
+					unsigned long iova)
+{
+	return av8l_fast_iova_coherent(ops, iova);
+}
+
+static struct av8l_fast_io_pgtable *
+av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg)
+{
+	struct av8l_fast_io_pgtable *data;
+
+	data = kmalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return NULL;
+
+	data->iop.ops = (struct io_pgtable_ops) {
+		.map		= av8l_fast_map,
+		.map_pages	= av8l_fast_map_pages,
+		.unmap		= av8l_fast_unmap,
+		.unmap_pages	= av8l_fast_unmap_pages,
+		.iova_to_phys	= av8l_fast_iova_to_phys,
+	};
+
+	return data;
+}
+
+/*
+ * We need max 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and
+ * 2048 pages for pmds (each pud page contains 512 table entries, each
+ * pointing to a pmd).
+ */
+#define NUM_PGD_PAGES 1
+#define NUM_PUD_PAGES 4
+#define NUM_PMD_PAGES 2048
+#define NUM_PGTBL_PAGES (NUM_PGD_PAGES + NUM_PUD_PAGES + NUM_PMD_PAGES)
+
+/* undefine arch specific definitions which depends on page table format */
+#undef pud_index
+#undef pud_mask
+#undef pud_next
+#undef pmd_index
+#undef pmd_mask
+#undef pmd_next
+
+#define pud_index(addr)		(((addr) >> 30) & 0x3)
+#define pud_mask(addr)		((addr) & ~((1UL << 30) - 1))
+#define pud_next(addr, end)					\
+({	unsigned long __boundary = pud_mask(addr + (1UL << 30));\
+	(__boundary - 1 < (end) - 1) ? __boundary : (end);	\
+})
+
+#define pmd_index(addr)		(((addr) >> 21) & 0x1ff)
+#define pmd_mask(addr)		((addr) & ~((1UL << 21) - 1))
+#define pmd_next(addr, end)					\
+({	unsigned long __boundary = pmd_mask(addr + (1UL << 21));\
+	(__boundary - 1 < (end) - 1) ? __boundary : (end);	\
+})
+
+static int
+av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data,
+			       struct io_pgtable_cfg *cfg, void *cookie)
+{
+	int i, j, pg = 0;
+	struct page **pages, *page;
+	struct qcom_io_pgtable_info *pgtbl_info = to_qcom_io_pgtable_info(cfg);
+	dma_addr_t pud, pmd;
+	int pmd_pg_index;
+	dma_addr_t base = pgtbl_info->iova_base;
+	dma_addr_t end = pgtbl_info->iova_end;
+
+	pages = kmalloc(sizeof(*pages) * NUM_PGTBL_PAGES, __GFP_NOWARN |
+							__GFP_NORETRY);
+
+	if (!pages)
+		pages = vmalloc(sizeof(*pages) * NUM_PGTBL_PAGES);
+
+	if (!pages)
+		return -ENOMEM;
+
+	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	if (!page)
+		goto err_free_pages_arr;
+	pages[pg++] = page;
+	data->pgd = page_address(page);
+
+	/*
+	 * We need max 2048 entries at level 2 to map 4GB of VA space. A page
+	 * can hold 512 entries, so we need max 4 pages.
+	 */
+	for (i = pud_index(base), pud = base; pud < end;
+			++i, pud = pud_next(pud, end)) {
+		av8l_fast_iopte pte, *ptep;
+
+		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+		if (!page)
+			goto err_free_pages;
+		pages[pg++] = page;
+		data->puds[i] = page_address(page);
+		pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE;
+		ptep = ((av8l_fast_iopte *)data->pgd) + i;
+		*ptep = pte;
+	}
+	av8l_clean_range(cfg, data->pgd, data->pgd + 4);
+
+	/*
+	 * We have max 4 puds, each of which can point to 512 pmds, so we'll
+	 * have max 2048 pmds, each of which can hold 512 ptes, for a grand
+	 * total of 2048*512=1048576 PTEs.
+	 */
+	pmd_pg_index = pg;
+	for (i = pud_index(base), pud = base; pud < end;
+			++i, pud = pud_next(pud, end)) {
+		for (j = pmd_index(pud), pmd = pud; pmd < pud_next(pud, end);
+				++j, pmd = pmd_next(pmd, end)) {
+			av8l_fast_iopte pte, *pudp;
+			void *addr;
+
+			page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+			if (!page)
+				goto err_free_pages;
+			pages[pg++] = page;
+
+			addr = page_address(page);
+			av8l_clean_range(cfg, addr, addr + SZ_4K);
+
+			pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE;
+			pudp = data->puds[i] + j;
+			*pudp = pte;
+		}
+		av8l_clean_range(cfg, data->puds[i], data->puds[i] + 512);
+	}
+
+	/*
+	 * We map the pmds into a virtually contiguous space so that we
+	 * don't have to traverse the first two levels of the page tables
+	 * to find the appropriate pud.  Instead, it will be a simple
+	 * offset from the virtual base of the pmds.
+	 */
+	data->pmds = vmap(&pages[pmd_pg_index], pg - pmd_pg_index,
+			  VM_IOREMAP, PAGE_KERNEL);
+	if (!data->pmds)
+		goto err_free_pages;
+
+	data->pages = pages;
+	data->base = base;
+	data->end = end;
+	data->nr_pages = pg;
+	return 0;
+
+err_free_pages:
+	for (i = 0; i < pg; ++i)
+		__free_page(pages[i]);
+err_free_pages_arr:
+	kvfree(pages);
+	return -ENOMEM;
+}
+
+static struct io_pgtable *
+av8l_fast_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
+{
+	u64 reg;
+	struct av8l_fast_io_pgtable *data =
+		av8l_fast_alloc_pgtable_data(cfg);
+	typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr;
+
+	if (!data)
+		return NULL;
+
+	/* restrict according to the fast map requirements */
+	cfg->ias = 32;
+	cfg->pgsize_bitmap = SZ_4K;
+
+	/* TCR */
+	if (cfg->coherent_walk) {
+		tcr->sh = AV8L_FAST_TCR_SH_IS;
+		tcr->irgn = AV8L_FAST_TCR_RGN_WBWA;
+		tcr->orgn = AV8L_FAST_TCR_RGN_WBWA;
+		if (WARN_ON(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA))
+			goto out_free_data;
+	} else {
+		tcr->sh = AV8L_FAST_TCR_SH_OS;
+		tcr->irgn = AV8L_FAST_TCR_RGN_NC;
+		if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA))
+			tcr->orgn = AV8L_FAST_TCR_RGN_NC;
+		else
+			tcr->orgn = AV8L_FAST_TCR_RGN_WBWA;
+	}
+
+	tcr->tg = AV8L_FAST_TCR_TG0_4K;
+
+	switch (cfg->oas) {
+	case 32:
+		tcr->ips = AV8L_FAST_TCR_PS_32_BIT;
+		break;
+	case 36:
+		tcr->ips = AV8L_FAST_TCR_PS_36_BIT;
+		break;
+	case 40:
+		tcr->ips = AV8L_FAST_TCR_PS_40_BIT;
+		break;
+	case 42:
+		tcr->ips = AV8L_FAST_TCR_PS_42_BIT;
+		break;
+	case 44:
+		tcr->ips = AV8L_FAST_TCR_PS_44_BIT;
+		break;
+	case 48:
+		tcr->ips = AV8L_FAST_TCR_PS_48_BIT;
+		break;
+	default:
+		goto out_free_data;
+	}
+
+	tcr->tsz = 64ULL - cfg->ias;
+
+	/* MAIRs */
+	reg = (AV8L_FAST_MAIR_ATTR_NC
+	       << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_NC)) |
+	      (AV8L_FAST_MAIR_ATTR_WBRWA
+	       << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_CACHE)) |
+	      (AV8L_FAST_MAIR_ATTR_DEVICE
+	       << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_DEV)) |
+	      (AV8L_FAST_MAIR_ATTR_UPSTREAM
+	       << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM));
+
+	cfg->arm_lpae_s1_cfg.mair = reg;
+
+	/* Allocate all page table memory! */
+	if (av8l_fast_prepopulate_pgtables(data, cfg, cookie))
+		goto out_free_data;
+
+	/* TTBRs */
+	cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);
+	return &data->iop;
+
+out_free_data:
+	kfree(data);
+	return NULL;
+}
+
+static void av8l_fast_free_pgtable(struct io_pgtable *iop)
+{
+	int i;
+	struct av8l_fast_io_pgtable *data = iof_pgtable_to_data(iop);
+
+	vunmap(data->pmds);
+	for (i = 0; i < data->nr_pages; ++i)
+		__free_page(data->pages[i]);
+	kvfree(data->pages);
+	kfree(data);
+}
+
+struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns = {
+	.alloc	= av8l_fast_alloc_pgtable,
+	.free	= av8l_fast_free_pgtable,
+};
+
+
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_SELFTEST
+
+#include <linux/dma-map-ops.h>
+
+static struct io_pgtable_cfg *cfg_cookie;
+
+static void dummy_tlb_flush_all(void *cookie)
+{
+	WARN_ON(cookie != cfg_cookie);
+}
+
+static void dummy_tlb_flush(unsigned long iova, size_t size, size_t granule,
+			    void *cookie)
+{
+	WARN_ON(cookie != cfg_cookie);
+}
+
+static void dummy_tlb_add_page(struct iommu_iotlb_gather *gather,
+			       unsigned long iova, size_t granule, void *cookie)
+{
+	dummy_tlb_flush(iova, granule, granule, cookie);
+}
+
+static struct iommu_flush_ops dummy_tlb_ops __initdata = {
+	.tlb_flush_all	= dummy_tlb_flush_all,
+	.tlb_flush_walk	= dummy_tlb_flush,
+	.tlb_add_page	= dummy_tlb_add_page,
+};
+
+/*
+ * Returns true if the iova range is successfully mapped to the contiguous
+ * phys range in ops.
+ */
+static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops,
+						 const unsigned long iova_start,
+						 const phys_addr_t phys_start,
+						 const size_t size)
+{
+	u64 iova = iova_start;
+	phys_addr_t phys = phys_start;
+
+	while (iova < (iova_start + size)) {
+		/* + 42 just to make sure offsetting is working */
+		if (ops->iova_to_phys(ops, iova + 42) != (phys + 42))
+			return false;
+		iova += SZ_4K;
+		phys += SZ_4K;
+	}
+	return true;
+}
+
+static int __init av8l_fast_positive_testing(void)
+{
+	int failed = 0;
+	u64 iova;
+	struct io_pgtable_ops *ops;
+	struct qcom_io_pgtable_info pgtable_info;
+	struct av8l_fast_io_pgtable *data;
+	av8l_fast_iopte *pmds;
+	u64 max = SZ_1G * 4ULL - 1;
+	u64 base = 0;
+
+	pgtable_info.iova_base = base;
+	pgtable_info.iova_end = max;
+	pgtable_info.cfg = (struct io_pgtable_cfg) {
+		.quirks = 0,
+		.tlb = &dummy_tlb_ops,
+		.ias = 32,
+		.oas = 32,
+		.pgsize_bitmap = SZ_4K,
+		.coherent_walk = true,
+	};
+
+	cfg_cookie = &pgtable_info.pgtbl_cfg;
+	ops = alloc_io_pgtable_ops(ARM_V8L_FAST, &pgtable_info.pgtbl_cfg,
+				   &pgtable_info.pgtbl_cfg);
+
+	if (WARN_ON(!ops))
+		return 1;
+
+	data = iof_pgtable_ops_to_data(ops);
+	pmds = data->pmds;
+
+	/* map the entire 4GB VA space with 4K map calls */
+	for (iova = base; iova < max; iova += SZ_4K) {
+		if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) {
+			failed++;
+			continue;
+		}
+	}
+	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
+					base, max - base)))
+		failed++;
+
+	/* unmap it all */
+	for (iova = base; iova < max; iova += SZ_4K) {
+		if (WARN_ON(ops->unmap(ops, iova, SZ_4K, NULL) != SZ_4K))
+			failed++;
+	}
+
+	/* sweep up TLB proving PTEs */
+	av8l_fast_clear_stale_ptes(ops, base, max, false);
+
+	/* map the entire 4GB VA space with 8K map calls */
+	for (iova = base; iova < max; iova += SZ_8K) {
+		if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) {
+			failed++;
+			continue;
+		}
+	}
+
+	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
+					base, max - base)))
+		failed++;
+
+	/* unmap it all with 8K unmap calls */
+	for (iova = base; iova < max; iova += SZ_8K) {
+		if (WARN_ON(ops->unmap(ops, iova, SZ_8K, NULL) != SZ_8K))
+			failed++;
+	}
+
+	/* sweep up TLB proving PTEs */
+	av8l_fast_clear_stale_ptes(ops, base, max, false);
+
+	/* map the entire 4GB VA space with 16K map calls */
+	for (iova = base; iova < max; iova += SZ_16K) {
+		if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) {
+			failed++;
+			continue;
+		}
+	}
+
+	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
+					base, max - base)))
+		failed++;
+
+	/* unmap it all */
+	for (iova = base; iova < max; iova += SZ_16K) {
+		if (WARN_ON(ops->unmap(ops, iova, SZ_16K, NULL) != SZ_16K))
+			failed++;
+	}
+
+	/* sweep up TLB proving PTEs */
+	av8l_fast_clear_stale_ptes(ops, base, max, false);
+
+	/* map the entire 4GB VA space with 64K map calls */
+	for (iova = base; iova < max; iova += SZ_64K) {
+		if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) {
+			failed++;
+			continue;
+		}
+	}
+
+	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
+					base, max - base)))
+		failed++;
+
+	/* unmap it all at once */
+	if (WARN_ON(ops->unmap(ops, base, max - base, NULL) != (max - base)))
+		failed++;
+
+	free_io_pgtable_ops(ops);
+	return failed;
+}
+
+static int __init av8l_fast_do_selftests(void)
+{
+	int failed = 0;
+
+	failed += av8l_fast_positive_testing();
+
+	pr_err("selftest: completed with %d failures\n", failed);
+
+	return 0;
+}
+subsys_initcall(av8l_fast_do_selftests);
+#endif

drivers/iommu/iommu-logger.c

@@ -0,0 +1,172 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/module.h>
+#include <linux/iommu.h>
+#include <linux/slab.h>
+#include "qcom-io-pgtable.h"
+
+#include "iommu-logger.h"
+
+static DEFINE_MUTEX(iommu_debug_attachments_lock);
+static LIST_HEAD(iommu_debug_attachments);
+
+static unsigned int iommu_logger_pgtable_levels(struct io_pgtable *iop)
+{
+	unsigned int va_bits, pte_size, bits_per_level, pg_shift;
+	unsigned long ias = iop->cfg.ias;
+
+	switch ((u32)iop->fmt) {
+	case ARM_32_LPAE_S1:
+	case ARM_64_LPAE_S1:
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
+	case ARM_V8L_FAST:
+#endif
+	case QCOM_ARM_64_LPAE_S1:
+		pte_size = sizeof(u64);
+		break;
+	default:
+		return 0;
+	}
+
+	pg_shift = __ffs(iop->cfg.pgsize_bitmap);
+	bits_per_level = pg_shift - ilog2(pte_size);
+	va_bits = ias - pg_shift;
+	return DIV_ROUND_UP(va_bits, bits_per_level);
+}
+
+static enum iommu_logger_pgtable_fmt iommu_logger_pgtable_fmt_lut(
+							enum io_pgtable_fmt fmt)
+{
+	switch ((u32)fmt) {
+	case ARM_32_LPAE_S1:
+		return IOMMU_LOGGER_ARM_32_LPAE_S1;
+	case ARM_64_LPAE_S1:
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
+	case ARM_V8L_FAST:
+#endif
+	case QCOM_ARM_64_LPAE_S1:
+		return IOMMU_LOGGER_ARM_64_LPAE_S1;
+	default:
+		return IOMMU_LOGGER_MAX_PGTABLE_FMTS;
+	}
+}
+
+static int iommu_logger_domain_ttbrs(struct io_pgtable *iop, void **ttbr0_ptr,
+				     void **ttbr1_ptr)
+{
+	int ret;
+	u64 ttbr0;
+
+	switch ((u32)iop->fmt) {
+	case ARM_32_LPAE_S1:
+	case ARM_64_LPAE_S1:
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
+	case ARM_V8L_FAST:
+#endif
+	case QCOM_ARM_64_LPAE_S1:
+		ttbr0 = iop->cfg.arm_lpae_s1_cfg.ttbr;
+		ret = 0;
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	if (!ret) {
+		*ttbr0_ptr = phys_to_virt(ttbr0);
+		/*
+		 * FIXME - fix ttbr1 retrieval later. In this kernel version
+		 * struct io_pgtable no longer contains this information.
+		 */
+		*ttbr1_ptr = NULL;
+	}
+
+	return ret;
+}
+
+static struct iommu_debug_attachment *iommu_logger_init(
+						struct iommu_domain *domain,
+						struct device *dev,
+						struct io_pgtable *iop)
+{
+	struct iommu_debug_attachment *logger;
+	char *client_name;
+	struct iommu_group *group;
+	unsigned int levels = iommu_logger_pgtable_levels(iop);
+	enum iommu_logger_pgtable_fmt fmt = iommu_logger_pgtable_fmt_lut(
+								iop->fmt);
+	void *ttbr0, *ttbr1;
+	int ret;
+
+	if (!levels || fmt == IOMMU_LOGGER_MAX_PGTABLE_FMTS)
+		return ERR_PTR(-EINVAL);
+
+	ret = iommu_logger_domain_ttbrs(iop, &ttbr0, &ttbr1);
+	if (ret)
+		return ERR_PTR(ret);
+
+	logger = kzalloc(sizeof(*logger), GFP_KERNEL);
+	if (!logger)
+		return ERR_PTR(-ENOMEM);
+
+	client_name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj));
+	if (!client_name) {
+		kfree(logger);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	group = iommu_group_get(dev);
+	iommu_group_put(group);
+
+	INIT_LIST_HEAD(&logger->list);
+	logger->domain = domain;
+	logger->group = group;
+	logger->client_name = client_name;
+	logger->fmt = fmt;
+	logger->levels = levels;
+	logger->ttbr0 = ttbr0;
+	logger->ttbr1 = ttbr1;
+
+	return logger;
+}
+
+int iommu_logger_register(struct iommu_debug_attachment **logger_out,
+			  struct iommu_domain *domain, struct device *dev,
+			  struct io_pgtable *iop)
+{
+	struct iommu_debug_attachment *logger;
+
+	if (!logger_out || !dev || !iop)
+		return -EINVAL;
+
+	logger = iommu_logger_init(domain, dev, iop);
+	if (IS_ERR(logger))
+		return PTR_ERR(logger);
+
+	mutex_lock(&iommu_debug_attachments_lock);
+	list_add(&logger->list, &iommu_debug_attachments);
+	mutex_unlock(&iommu_debug_attachments_lock);
+
+	*logger_out = logger;
+	return 0;
+}
+EXPORT_SYMBOL(iommu_logger_register);
+
+void iommu_logger_unregister(struct iommu_debug_attachment *logger)
+{
+	if (!logger)
+		return;
+
+	mutex_lock(&iommu_debug_attachments_lock);
+	list_del(&logger->list);
+	mutex_unlock(&iommu_debug_attachments_lock);
+	kfree(logger->client_name);
+	kfree(logger);
+}
+EXPORT_SYMBOL(iommu_logger_unregister);
+
+MODULE_DESCRIPTION("QTI IOMMU SUPPORT");
+MODULE_LICENSE("GPL v2");

drivers/iommu/iommu-logger.h

@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. */
+
+#ifndef __LINUX_QTI_IOMMU_LOGGER_H
+#define __LINUX_QTI_IOMMU_LOGGER_H
+
+#include <linux/io-pgtable.h>
+
+enum iommu_logger_pgtable_fmt {
+	IOMMU_LOGGER_ARM_32_LPAE_S1,
+	IOMMU_LOGGER_ARM_64_LPAE_S1,
+	IOMMU_LOGGER_MAX_PGTABLE_FMTS,
+};
+
+/*
+ * Each group may have more than one domain; but each domain may
+ * only have one group.
+ */
+struct iommu_debug_attachment {
+	struct iommu_domain *domain;
+	struct iommu_group *group;
+	char *client_name;
+	enum iommu_logger_pgtable_fmt fmt;
+	unsigned int levels;
+	/*
+	 * Virtual addresses of the top-level page tables are stored here,
+	 * as they are more useful for debug tools than physical addresses.
+	 */
+	void *ttbr0;
+	void *ttbr1;
+	struct list_head list;
+};
+
+#if IS_ENABLED(CONFIG_QTI_IOMMU_SUPPORT)
+
+int iommu_logger_register(struct iommu_debug_attachment **a,
+			  struct iommu_domain *domain, struct device *dev,
+			  struct io_pgtable *iop);
+void iommu_logger_unregister(struct iommu_debug_attachment *a);
+#else
+static inline int iommu_logger_register(struct iommu_debug_attachment **a,
+					struct iommu_domain *domain,
+					struct device *dev,
+					struct io_pgtable *iop)
+{
+	return 0;
+}
+
+static inline void iommu_logger_unregister(struct iommu_debug_attachment *a) {}
+#endif /* CONFIG_QTI_IOMMU_LOGGER */
+#endif /* __LINUX_QTI_IOMMU_LOGGER_H */

drivers/iommu/qcom-dma-iommu-generic.c

@@ -0,0 +1,814 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2014, 2020-2021, The Linux Foundation. All rights reserved.
+ * Contiguous Memory Allocator for DMA mapping framework
+ * Copyright (c) 2010-2011 by Samsung Electronics.
+ * Written by:
+ *	Marek Szyprowski <m.szyprowski@samsung.com>
+ *	Michal Nazarewicz <mina86@mina86.com>
+ * Copyright (C) 2012, 2014-2015 ARM Ltd.
+ */
+
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/platform_device.h>
+#include <linux/mutex.h>
+#include <linux/rbtree.h>
+#include <linux/genalloc.h>
+#include <linux/dma-direct.h>
+#include <linux/cma.h>
+#include <linux/iova.h>
+#include <linux/dma-map-ops.h>
+#include <linux/dma-mapping.h>
+#include <linux/qcom-dma-mapping.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/iommu.h>
+#include <linux/qcom-iommu-util.h>
+#include <linux/workqueue.h>
+#include <linux/debugfs.h>
+#include "qcom-dma-iommu-generic.h"
+
+static bool probe_finished;
+static struct device *qcom_dma_iommu_dev;
+static struct cma *qcom_dma_contiguous_default_area;
+
+struct pci_host_bridge *qcom_pci_find_host_bridge(struct pci_bus *bus)
+{
+	while (bus->parent)
+		bus = bus->parent;
+
+	return to_pci_host_bridge(bus->bridge);
+}
+
+/*
+ * This avoids arch-specific assembly, but may be slower since it calls
+ * back into the dma layer again.
+ */
+void qcom_arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
+		enum dma_data_direction dir)
+{
+	dma_addr_t dma_addr = phys_to_dma(qcom_dma_iommu_dev, paddr);
+
+	dma_sync_single_for_device(qcom_dma_iommu_dev,
+		dma_addr, size, dir);
+}
+
+void qcom_arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
+		enum dma_data_direction dir)
+{
+	dma_addr_t dma_addr = phys_to_dma(qcom_dma_iommu_dev, paddr);
+
+	dma_sync_single_for_cpu(qcom_dma_iommu_dev,
+		dma_addr, size, dir);
+}
+
+void qcom_arch_dma_prep_coherent(struct page *page, size_t size)
+{
+	phys_addr_t phys = page_to_phys(page);
+	dma_addr_t dma_addr = phys_to_dma(qcom_dma_iommu_dev, phys);
+
+	dma_sync_single_for_device(qcom_dma_iommu_dev,
+		dma_addr, size, DMA_TO_DEVICE);
+}
+
+static struct cma *qcom_dev_get_cma_area(struct device *dev)
+{
+	if (dev && dev->cma_area)
+		return dev->cma_area;
+	return qcom_dma_contiguous_default_area;
+}
+
+struct page *qcom_dma_alloc_from_contiguous(struct device *dev, size_t count,
+				       unsigned int align, bool no_warn)
+{
+	if (align > CONFIG_CMA_ALIGNMENT)
+		align = CONFIG_CMA_ALIGNMENT;
+
+	return cma_alloc(qcom_dev_get_cma_area(dev), count, align, no_warn);
+}
+
+bool qcom_dma_release_from_contiguous(struct device *dev, struct page *pages,
+				 int count)
+{
+	return cma_release(qcom_dev_get_cma_area(dev), pages, count);
+}
+
+static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp)
+{
+	unsigned int align = min(get_order(size), CONFIG_CMA_ALIGNMENT);
+
+	return cma_alloc(cma, size >> PAGE_SHIFT, align, gfp & __GFP_NOWARN);
+}
+
+struct page *qcom_dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
+{
+	/* CMA can be used only in the context which permits sleeping */
+	if (!gfpflags_allow_blocking(gfp))
+		return NULL;
+	if (dev->cma_area)
+		return cma_alloc_aligned(dev->cma_area, size, gfp);
+	if (size <= PAGE_SIZE || !qcom_dma_contiguous_default_area)
+		return NULL;
+	return cma_alloc_aligned(qcom_dma_contiguous_default_area, size, gfp);
+}
+
+void qcom_dma_free_contiguous(struct device *dev, struct page *page, size_t size)
+{
+	if (!cma_release(qcom_dev_get_cma_area(dev), page,
+			 PAGE_ALIGN(size) >> PAGE_SHIFT))
+		__free_pages(page, get_order(size));
+}
+
+
+/*
+ * find_vm_area is not exported. Some dma apis expect that an array of
+ * struct pages can be saved in the vm_area, and retrieved at a later time.
+ */
+struct rb_root _root;
+struct rb_root *root = &_root;
+DEFINE_MUTEX(rbtree_lock);
+
+struct qcom_iommu_dma_area {
+	struct rb_node node;
+	unsigned long addr;
+	struct page **pages;
+};
+
+static void qcom_insert_vm_area(struct qcom_iommu_dma_area *area)
+{
+	struct rb_node **new, *parent;
+
+	mutex_lock(&rbtree_lock);
+
+	parent = NULL;
+	new = &root->rb_node;
+	while (*new) {
+		struct qcom_iommu_dma_area *entry;
+
+		entry = rb_entry(*new,
+				struct qcom_iommu_dma_area,
+				node);
+
+		parent = *new;
+		if (area->addr < entry->addr)
+			new = &((*new)->rb_left);
+		else if (area->addr > entry->addr)
+			new = &((*new)->rb_right);
+		else {
+			mutex_unlock(&rbtree_lock);
+			WARN_ON(1);
+			return;
+		}
+	}
+
+	rb_link_node(&area->node, parent, new);
+	rb_insert_color(&area->node, root);
+	mutex_unlock(&rbtree_lock);
+}
+
+static struct qcom_iommu_dma_area *qcom_find_vm_area(const void *cpu_addr)
+{
+	struct rb_node *node;
+	struct qcom_iommu_dma_area *entry;
+	unsigned long addr = (unsigned long)cpu_addr;
+
+	mutex_lock(&rbtree_lock);
+	node = root->rb_node;
+	while (node) {
+		entry = rb_entry(node,
+				struct qcom_iommu_dma_area,
+				node);
+
+		if (addr < entry->addr)
+			node = node->rb_left;
+		else if (addr > entry->addr)
+			node = node->rb_right;
+		else {
+			mutex_unlock(&rbtree_lock);
+			return entry;
+		}
+	}
+
+	mutex_unlock(&rbtree_lock);
+
+	return NULL;
+}
+
+struct page **qcom_dma_common_find_pages(void *cpu_addr)
+{
+	struct qcom_iommu_dma_area *area = qcom_find_vm_area(cpu_addr);
+
+	if (!area)
+		return NULL;
+	return area->pages;
+}
+
+/*
+ * Remaps an array of PAGE_SIZE pages into another vm_area.
+ * Cannot be used in non-sleeping contexts
+ */
+void *qcom_dma_common_pages_remap(struct page **pages, size_t size,
+			 pgprot_t prot, const void *caller)
+{
+	struct qcom_iommu_dma_area *area;
+	void *vaddr;
+
+	area = kzalloc(sizeof(*area), GFP_KERNEL);
+	if (!area)
+		return NULL;
+
+	vaddr = vmap(pages, PAGE_ALIGN(size) >> PAGE_SHIFT,
+		     VM_DMA_COHERENT, prot);
+	if (!vaddr) {
+		kfree(area);
+		return NULL;
+	}
+
+	area->pages = pages;
+	area->addr = (unsigned long)vaddr;
+	qcom_insert_vm_area(area);
+
+	return vaddr;
+}
+
+/*
+ * Remaps an allocated contiguous region into another vm_area.
+ * Cannot be used in non-sleeping contexts
+ */
+void *qcom_dma_common_contiguous_remap(struct page *page, size_t size,
+			pgprot_t prot, const void *caller)
+{
+	int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+	struct page **pages;
+	void *vaddr;
+	int i;
+
+	pages = kmalloc_array(count, sizeof(struct page *), GFP_KERNEL);
+	if (!pages)
+		return NULL;
+	for (i = 0; i < count; i++)
+		pages[i] = nth_page(page, i);
+	vaddr = vmap(pages, count, VM_DMA_COHERENT, prot);
+	kfree(pages);
+
+	return vaddr;
+}
+
+/*
+ * Unmaps a range previously mapped by dma_common_contiguous_remap or
+ * dma_common_pages_remap. Note that dma_common_contiguous_remap does
+ * not insert an rb_tree entry since there is no pages array to save.
+ */
+void qcom_dma_common_free_remap(void *cpu_addr, size_t size)
+{
+	struct qcom_iommu_dma_area *area;
+
+	/* qcom_dma_common_contiguous_remap doesn't save the pages array */
+	area = qcom_find_vm_area(cpu_addr);
+	if (area) {
+		mutex_lock(&rbtree_lock);
+		rb_erase(&area->node, root);
+		mutex_unlock(&rbtree_lock);
+		kfree(area);
+	}
+
+	vunmap(cpu_addr);
+}
+
+static struct gen_pool *atomic_pool __ro_after_init;
+
+static size_t atomic_pool_size;
+static unsigned long current_pool_size;
+
+/* Dynamic background expansion when the atomic pool is near capacity */
+static struct work_struct atomic_pool_work;
+
+static void dma_atomic_pool_debugfs_init(void)
+{
+	struct dentry *root;
+
+	root = debugfs_create_dir("qcom_dma_pools", NULL);
+	if (IS_ERR_OR_NULL(root))
+		return;
+
+	debugfs_create_ulong("pool_size", 0400, root, &current_pool_size);
+}
+
+static void dma_atomic_pool_size_add(gfp_t gfp, size_t size)
+{
+	current_pool_size += size;
+}
+
+static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
+			      gfp_t gfp)
+{
+	unsigned int order;
+	struct page *page = NULL;
+	void *addr;
+	int ret = -ENOMEM;
+
+	/* Cannot allocate larger than MAX_ORDER - 1 */
+	order = min(get_order(pool_size), MAX_ORDER - 1);
+
+	do {
+		pool_size = 1 << (PAGE_SHIFT + order);
+
+		if (qcom_dev_get_cma_area(NULL))
+			page = qcom_dma_alloc_from_contiguous(NULL, 1 << order,
+							      order, false);
+		else
+			page = alloc_pages(gfp, order);
+	} while (!page && order-- > 0);
+	if (!page)
+		goto out;
+
+	qcom_arch_dma_prep_coherent(page, pool_size);
+
+	addr = qcom_dma_common_contiguous_remap(page, pool_size,
+						pgprot_dmacoherent(PAGE_KERNEL),
+						__builtin_return_address(0));
+	if (!addr)
+		goto free_page;
+
+	ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),
+				pool_size, NUMA_NO_NODE);
+	if (ret)
+		goto remove_mapping;
+
+	dma_atomic_pool_size_add(gfp, pool_size);
+	return 0;
+
+remove_mapping:
+	qcom_dma_common_free_remap(addr, pool_size);
+free_page:
+	if (!qcom_dma_release_from_contiguous(NULL, page, 1 << order))
+		__free_pages(page, order);
+out:
+	return ret;
+}
+
+static void atomic_pool_resize(struct gen_pool *pool, gfp_t gfp)
+{
+	if (pool && gen_pool_avail(pool) < atomic_pool_size)
+		atomic_pool_expand(pool, gen_pool_size(pool), gfp);
+}
+
+static void atomic_pool_work_fn(struct work_struct *work)
+{
+	atomic_pool_resize(atomic_pool, GFP_KERNEL);
+}
+
+static struct gen_pool *__dma_atomic_pool_init(size_t pool_size, gfp_t gfp)
+{
+	struct gen_pool *pool;
+	int ret;
+
+	pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
+	if (!pool)
+		return NULL;
+
+	gen_pool_set_algo(pool, gen_pool_first_fit_order_align, NULL);
+
+	ret = atomic_pool_expand(pool, pool_size, gfp);
+	if (ret) {
+		gen_pool_destroy(pool);
+		pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",
+		       pool_size >> 10, &gfp);
+		return NULL;
+	}
+
+	pr_info("DMA preallocated %zu KiB %pGg pool for atomic allocations\n",
+		gen_pool_size(pool) >> 10, &gfp);
+	return pool;
+}
+
+static int dma_atomic_pool_init(struct device *dev)
+{
+	int ret = 0;
+	unsigned long pages;
+
+	/* Default the pool size to 128KB per 1 GB of memory, min 128 KB, max MAX_ORDER - 1. */
+	pages = totalram_pages() / (SZ_1G / SZ_128K);
+	pages = min_t(unsigned long, pages, MAX_ORDER_NR_PAGES);
+	atomic_pool_size = max_t(size_t, pages << PAGE_SHIFT, SZ_128K);
+	INIT_WORK(&atomic_pool_work, atomic_pool_work_fn);
+
+	atomic_pool = __dma_atomic_pool_init(atomic_pool_size, GFP_KERNEL);
+	if (!atomic_pool)
+		return -ENOMEM;
+
+	dma_atomic_pool_debugfs_init();
+	return ret;
+}
+
+/*
+ * Couldn't implement this via dma_alloc_attrs(qcom_iommu_dma_dev, GFP_ATOMIC)
+ * due to dma_free_from_pool only passing in cpu_addr & not dma_handle.
+ */
+void *qcom_dma_alloc_from_pool(struct device *dev, size_t size,
+			struct page **ret_page, gfp_t flags)
+{
+	unsigned long val;
+	void *ptr = NULL;
+
+	if (!atomic_pool) {
+		WARN(1, "coherent pool not initialised!\n");
+		return NULL;
+	}
+
+	val = gen_pool_alloc(atomic_pool, size);
+	if (val) {
+		phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
+
+		*ret_page = pfn_to_page(__phys_to_pfn(phys));
+		ptr = (void *)val;
+		memset(ptr, 0, size);
+	}
+	if (gen_pool_avail(atomic_pool) < atomic_pool_size)
+		schedule_work(&atomic_pool_work);
+
+	return ptr;
+}
+
+bool qcom_dma_free_from_pool(struct device *dev, void *start, size_t size)
+{
+	if (!atomic_pool || !gen_pool_has_addr(atomic_pool, (unsigned long)start, size))
+		return false;
+	gen_pool_free(atomic_pool, (unsigned long)start, size);
+	return true;
+}
+
+static void qcom_dma_atomic_pool_exit(struct device *dev)
+{
+	unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
+	void *addr;
+	struct page *page;
+
+	/*
+	 * Find the starting address. Pool is expected to be unused.
+	 *
+	 * While the pool size can expand, it is okay to use the initial size
+	 * here, as this function can only ever be called prior to the pool
+	 * ever being used. The pool can only expand when an allocation is satisfied
+	 * from it, which would not be possible by the time this function is called.
+	 * Therefore the size of the pool will be the initial size.
+	 */
+	addr = (void *)gen_pool_alloc(atomic_pool, atomic_pool_size);
+	if (!addr) {
+		WARN_ON(1);
+		return;
+	}
+
+	gen_pool_free(atomic_pool, (unsigned long)addr, atomic_pool_size);
+	gen_pool_destroy(atomic_pool);
+	page = vmalloc_to_page(addr);
+	qcom_dma_common_free_remap(addr, atomic_pool_size);
+	qcom_dma_release_from_contiguous(dev, page, nr_pages);
+}
+
+/*
+ * struct dma_coherent_mem is private, so we cna't access it. 0 indicates
+ * an error condition for dma_mmap_from_dev_coherent.
+ */
+int qcom_dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
+			   void *vaddr, size_t size, int *ret)
+{
+	return 0;
+}
+
+/*
+ * Return the page attributes used for mapping dma_alloc_* memory, either in
+ * kernel space if remapping is needed, or to userspace through dma_mmap_*.
+ */
+pgprot_t qcom_dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs)
+{
+	if (dev_is_dma_coherent(dev))
+		return prot;
+#ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE
+	if (attrs & DMA_ATTR_WRITE_COMBINE)
+		return pgprot_writecombine(prot);
+#endif
+	return pgprot_dmacoherent(prot);
+}
+
+/**
+ * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API
+ *                    page flags.
+ * @dir: Direction of DMA transfer
+ * @coherent: Is the DMA master cache-coherent?
+ * @attrs: DMA attributes for the mapping
+ *
+ * Return: corresponding IOMMU API page protection flags
+ */
+int qcom_dma_info_to_prot(enum dma_data_direction dir, bool coherent,
+		     unsigned long attrs)
+{
+	int prot = coherent ? IOMMU_CACHE : 0;
+
+	if (attrs & DMA_ATTR_PRIVILEGED)
+		prot |= IOMMU_PRIV;
+
+	if (attrs & DMA_ATTR_SYS_CACHE_ONLY)
+		prot |= IOMMU_SYS_CACHE;
+
+	if (attrs & DMA_ATTR_SYS_CACHE_ONLY_NWA)
+		prot |= IOMMU_SYS_CACHE_NWA;
+
+	switch (dir) {
+	case DMA_BIDIRECTIONAL:
+		return prot | IOMMU_READ | IOMMU_WRITE;
+	case DMA_TO_DEVICE:
+		return prot | IOMMU_READ;
+	case DMA_FROM_DEVICE:
+		return prot | IOMMU_WRITE;
+	default:
+		return 0;
+	}
+}
+
+/*
+ * The DMA API client is passing in a scatterlist which could describe
+ * any old buffer layout, but the IOMMU API requires everything to be
+ * aligned to IOMMU pages. Hence the need for this complicated bit of
+ * impedance-matching, to be able to hand off a suitably-aligned list,
+ * but still preserve the original offsets and sizes for the caller.
+ */
+size_t qcom_iommu_dma_prepare_map_sg(struct device *dev, struct iova_domain *iovad,
+				struct scatterlist *sg, int nents)
+{
+	struct scatterlist *s, *prev = NULL;
+	size_t iova_len = 0;
+	unsigned long mask = dma_get_seg_boundary(dev);
+	int i;
+
+	/*
+	 * Work out how much IOVA space we need, and align the segments to
+	 * IOVA granules for the IOMMU driver to handle. With some clever
+	 * trickery we can modify the list in-place, but reversibly, by
+	 * stashing the unaligned parts in the as-yet-unused DMA fields.
+	 */
+	for_each_sg(sg, s, nents, i) {
+		size_t s_iova_off = iova_offset(iovad, s->offset);
+		size_t s_length = s->length;
+		size_t pad_len = (mask - iova_len + 1) & mask;
+
+		sg_dma_address(s) = s_iova_off;
+		sg_dma_len(s) = s_length;
+		s->offset -= s_iova_off;
+		s_length = iova_align(iovad, s_length + s_iova_off);
+		s->length = s_length;
+
+		/*
+		 * Due to the alignment of our single IOVA allocation, we can
+		 * depend on these assumptions about the segment boundary mask:
+		 * - If mask size >= IOVA size, then the IOVA range cannot
+		 *   possibly fall across a boundary, so we don't care.
+		 * - If mask size < IOVA size, then the IOVA range must start
+		 *   exactly on a boundary, therefore we can lay things out
+		 *   based purely on segment lengths without needing to know
+		 *   the actual addresses beforehand.
+		 * - The mask must be a power of 2, so pad_len == 0 if
+		 *   iova_len == 0, thus we cannot dereference prev the first
+		 *   time through here (i.e. before it has a meaningful value).
+		 */
+		if (pad_len && pad_len < s_length - 1) {
+			prev->length += pad_len;
+			iova_len += pad_len;
+		}
+
+		iova_len += s_length;
+		prev = s;
+	}
+
+	return iova_len;
+}
+
+/*
+ * Prepare a successfully-mapped scatterlist to give back to the caller.
+ *
+ * At this point the segments are already laid out by iommu_dma_map_sg() to
+ * avoid individually crossing any boundaries, so we merely need to check a
+ * segment's start address to avoid concatenating across one.
+ */
+int qcom_iommu_dma_finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
+		dma_addr_t dma_addr)
+{
+	struct scatterlist *s, *cur = sg;
+	unsigned long seg_mask = dma_get_seg_boundary(dev);
+	unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev);
+	int i, count = 0;
+
+	for_each_sg(sg, s, nents, i) {
+		/* Restore this segment's original unaligned fields first */
+		unsigned int s_iova_off = sg_dma_address(s);
+		unsigned int s_length = sg_dma_len(s);
+		unsigned int s_iova_len = s->length;
+
+		s->offset += s_iova_off;
+		s->length = s_length;
+		sg_dma_address(s) = DMA_MAPPING_ERROR;
+		sg_dma_len(s) = 0;
+
+		/*
+		 * Now fill in the real DMA data. If...
+		 * - there is a valid output segment to append to
+		 * - and this segment starts on an IOVA page boundary
+		 * - but doesn't fall at a segment boundary
+		 * - and wouldn't make the resulting output segment too long
+		 */
+		if (cur_len && !s_iova_off && (dma_addr & seg_mask) &&
+		    (max_len - cur_len >= s_length)) {
+			/* ...then concatenate it with the previous one */
+			cur_len += s_length;
+		} else {
+			/* Otherwise start the next output segment */
+			if (i > 0)
+				cur = sg_next(cur);
+			cur_len = s_length;
+			count++;
+
+			sg_dma_address(cur) = dma_addr + s_iova_off;
+		}
+
+		sg_dma_len(cur) = cur_len;
+		dma_addr += s_iova_len;
+
+		if (s_length + s_iova_off < s_iova_len)
+			cur_len = 0;
+	}
+	return count;
+}
+
+/*
+ * If mapping failed, then just restore the original list,
+ * but making sure the DMA fields are invalidated.
+ */
+void qcom_iommu_dma_invalidate_sg(struct scatterlist *sg, int nents)
+{
+	struct scatterlist *s;
+	int i;
+
+	for_each_sg(sg, s, nents, i) {
+		if (sg_dma_address(s) != DMA_MAPPING_ERROR)
+			s->offset += sg_dma_address(s);
+		if (sg_dma_len(s))
+			s->length = sg_dma_len(s);
+		sg_dma_address(s) = DMA_MAPPING_ERROR;
+		sg_dma_len(s) = 0;
+	}
+}
+
+/**
+ * __iommu_dma_mmap - Map a buffer into provided user VMA
+ * @pages: Array representing buffer from __iommu_dma_alloc()
+ * @size: Size of buffer in bytes
+ * @vma: VMA describing requested userspace mapping
+ *
+ * Maps the pages of the buffer in @pages into @vma. The caller is responsible
+ * for verifying the correct size and protection of @vma beforehand.
+ */
+static int __qcom_iommu_dma_mmap(struct page **pages, size_t size,
+		struct vm_area_struct *vma)
+{
+	return vm_map_pages(vma, pages, PAGE_ALIGN(size) >> PAGE_SHIFT);
+}
+
+int qcom_iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
+		void *cpu_addr, dma_addr_t dma_addr, size_t size,
+		unsigned long attrs)
+{
+	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
+	unsigned long pfn, off = vma->vm_pgoff;
+	int ret;
+
+	vma->vm_page_prot = qcom_dma_pgprot(dev, vma->vm_page_prot, attrs);
+
+	if (qcom_dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
+		return ret;
+
+	if (off >= nr_pages || vma_pages(vma) > nr_pages - off)
+		return -ENXIO;
+
+	if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {
+		struct page **pages = qcom_dma_common_find_pages(cpu_addr);
+
+		if (pages)
+			return __qcom_iommu_dma_mmap(pages, size, vma);
+		pfn = vmalloc_to_pfn(cpu_addr);
+	} else {
+		pfn = page_to_pfn(virt_to_page(cpu_addr));
+	}
+
+	return remap_pfn_range(vma, vma->vm_start, pfn + off,
+			       vma->vm_end - vma->vm_start,
+			       vma->vm_page_prot);
+}
+
+int qcom_iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
+		void *cpu_addr, dma_addr_t dma_addr, size_t size,
+		unsigned long attrs)
+{
+	struct page *page;
+	int ret;
+
+	if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {
+		struct page **pages = qcom_dma_common_find_pages(cpu_addr);
+
+		if (pages) {
+			return sg_alloc_table_from_pages(sgt, pages,
+					PAGE_ALIGN(size) >> PAGE_SHIFT,
+					0, size, GFP_KERNEL);
+		}
+
+		page = vmalloc_to_page(cpu_addr);
+	} else {
+		page = virt_to_page(cpu_addr);
+	}
+
+	ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
+	if (!ret)
+		sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
+	return ret;
+}
+
+static int qcom_dma_iommu_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct device *dev = &pdev->dev;
+
+	qcom_dma_iommu_dev = dev;
+	if (dev_is_dma_coherent(dev)) {
+		dev_err(dev, "Cannot be dma-coherent\n");
+		return -EINVAL;
+	}
+
+	/* Should be connected to linux,cma-default node */
+	ret = of_reserved_mem_device_init_by_idx(dev, dev->of_node, 0);
+	if (ret)
+		return ret;
+
+	qcom_dma_contiguous_default_area = dev->cma_area;
+	if (!qcom_dma_contiguous_default_area) {
+		dev_err(dev, "Unable to find cma area\n");
+		return -EINVAL;
+	}
+
+	ret = dma_atomic_pool_init(dev);
+	if (ret)
+		goto out_iova_cache;
+
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
+	if (ret)
+		goto out_atomic_pool;
+
+	probe_finished = true;
+	return 0;
+
+out_atomic_pool:
+	qcom_dma_atomic_pool_exit(dev);
+
+out_iova_cache:
+	return ret;
+}
+
+bool qcom_dma_iommu_is_ready(void)
+{
+	if (!probe_finished)
+		return false;
+	return true;
+}
+EXPORT_SYMBOL(qcom_dma_iommu_is_ready);
+
+static int qcom_dma_iommu_remove(struct platform_device *pdev)
+{
+	qcom_dma_atomic_pool_exit(&pdev->dev);
+	return 0;
+}
+
+static const struct of_device_id qcom_dma_iommu_of_match[] = {
+	{.compatible = "qcom,iommu-dma"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, qcom_dma_iommu_of_match);
+
+static struct platform_driver qcom_dma_iommu_driver = {
+	.probe = qcom_dma_iommu_probe,
+	.remove = qcom_dma_iommu_remove,
+	.driver = {
+		.name = "qcom_dma_iommu",
+		.of_match_table = qcom_dma_iommu_of_match,
+		.suppress_bind_attrs    = true,
+	},
+};
+
+int __init qcom_dma_iommu_generic_driver_init(void)
+{
+	return platform_driver_register(&qcom_dma_iommu_driver);
+}
+
+void qcom_dma_iommu_generic_driver_exit(void)
+{
+	platform_driver_unregister(&qcom_dma_iommu_driver);
+}

drivers/iommu/qcom-dma-iommu-generic.h

@@ -0,0 +1,85 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef __QCOM_DMA_IOMMU_GENERIC_H
+#define __QCOM_DMA_IOMMU_GENERIC_H
+
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/pci.h>
+
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
+
+bool qcom_dma_iommu_is_ready(void);
+extern int __init qcom_dma_iommu_generic_driver_init(void);
+extern void qcom_dma_iommu_generic_driver_exit(void);
+
+struct pci_host_bridge *qcom_pci_find_host_bridge(struct pci_bus *bus);
+
+void qcom_arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
+		enum dma_data_direction dir);
+void qcom_arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
+		enum dma_data_direction dir);
+void qcom_arch_dma_prep_coherent(struct page *page, size_t size);
+
+/* kernel/dma/contiguous.c */
+struct page *qcom_dma_alloc_from_contiguous(struct device *dev, size_t count,
+	unsigned int align, bool no_warn);
+bool qcom_dma_release_from_contiguous(struct device *dev, struct page *pages,
+	int count);
+struct page *qcom_dma_alloc_contiguous(struct device *dev, size_t size,
+	gfp_t gfp);
+void qcom_dma_free_contiguous(struct device *dev, struct page *page,
+	size_t size);
+
+/* kernel/dma/remap.c */
+struct page **qcom_dma_common_find_pages(void *cpu_addr);
+void *qcom_dma_common_pages_remap(struct page **pages, size_t size,
+	 pgprot_t prot, const void *caller);
+void *qcom_dma_common_contiguous_remap(struct page *page, size_t size,
+	pgprot_t prot, const void *caller);
+void qcom_dma_common_free_remap(void *cpu_addr, size_t size);
+void *qcom_dma_alloc_from_pool(struct device *dev, size_t size,
+	struct page **ret_page, gfp_t flags);
+bool qcom_dma_free_from_pool(struct device *dev, void *start, size_t size);
+
+int qcom_dma_mmap_from_dev_coherent(struct device *dev,
+	struct vm_area_struct *vma, void *vaddr, size_t size, int *ret);
+
+/* kernel/dma/mapping.c */
+pgprot_t qcom_dma_pgprot(struct device *dev, pgprot_t prot,
+	unsigned long attrs);
+
+/* DMA-IOMMU utilities */
+int qcom_dma_info_to_prot(enum dma_data_direction dir, bool coherent,
+		     unsigned long attrs);
+size_t qcom_iommu_dma_prepare_map_sg(struct device *dev, struct iova_domain *iovad,
+				struct scatterlist *sg, int nents);
+int qcom_iommu_dma_finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
+		dma_addr_t dma_addr);
+void qcom_iommu_dma_invalidate_sg(struct scatterlist *sg, int nents);
+int qcom_iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
+		void *cpu_addr, dma_addr_t dma_addr, size_t size,
+		unsigned long attrs);
+int qcom_iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
+		void *cpu_addr, dma_addr_t dma_addr, size_t size,
+		unsigned long attrs);
+
+#else /*CONFIG_IOMMU_IO_PGTABLE_FAST*/
+
+static inline bool qcom_dma_iommu_is_ready(void)
+{
+	return true;
+}
+
+static inline int __init qcom_dma_iommu_generic_driver_init(void)
+{
+	return 0;
+}
+
+static inline void qcom_dma_iommu_generic_driver_exit(void) {}
+
+#endif /*CONFIG_IOMMU_IO_PGTABLE_FAST*/
+#endif	/* __QCOM_DMA_IOMMU_GENERIC_H */

drivers/iommu/qcom-io-pgtable-alloc.c

@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#include <linux/module.h>
+#include <linux/shrinker.h>
+#include <linux/slab.h>
+
+#include <soc/qcom/secure_buffer.h>
+
+struct io_pgtable_pool {
+	u32 vmid;
+	struct kref ref;
+	spinlock_t pool_lock;
+	struct list_head page_pool;
+};
+
+static DEFINE_MUTEX(page_pool_xa_lock);
+static DEFINE_XARRAY(page_pool_xa);
+static atomic_long_t page_pool_count = ATOMIC_LONG_INIT(0);
+
+static bool is_secure_vmid(u32 vmid)
+{
+	return !!vmid;
+}
+
+static int io_pgtable_hyp_assign_page(u32 vmid, struct page *page)
+{
+	u32 src_vmid_list[] = {VMID_HLOS};
+	int dst_vmid_list[] = {VMID_HLOS, vmid};
+	int dest_perms[] = {PERM_READ | PERM_WRITE, PERM_READ};
+	int ret;
+
+	ret = hyp_assign_phys(page_to_phys(page), PAGE_SIZE, src_vmid_list,
+			      ARRAY_SIZE(src_vmid_list), dst_vmid_list, dest_perms,
+			      ARRAY_SIZE(dst_vmid_list));
+
+	WARN(ret, "failed to assign memory to VMID: %u rc:%d\n", vmid, ret);
+	return ret ? -EADDRNOTAVAIL : 0;
+}
+
+static int io_pgtable_hyp_unassign_page(u32 vmid, struct page *page)
+{
+	u32 src_vmid_list[] = {VMID_HLOS, vmid};
+	int dst_vmid_list[] = {VMID_HLOS};
+	int dest_perms[] = {PERM_READ | PERM_WRITE | PERM_EXEC};
+	int ret;
+
+	ret = hyp_assign_phys(page_to_phys(page), PAGE_SIZE, src_vmid_list,
+			      ARRAY_SIZE(src_vmid_list), dst_vmid_list, dest_perms,
+			      ARRAY_SIZE(dst_vmid_list));
+
+	WARN(ret, "failed to unassign memory from VMID: %u rc: %d\n", vmid, ret);
+	return ret ? -EADDRNOTAVAIL : 0;
+}
+
+static struct page *__alloc_page_from_pool(struct list_head *page_pool)
+{
+	struct page *page;
+
+	page = list_first_entry_or_null(page_pool, struct page, lru);
+	if (page) {
+		list_del(&page->lru);
+		atomic_long_dec(&page_pool_count);
+		dec_node_page_state(page, NR_KERNEL_MISC_RECLAIMABLE);
+	}
+
+	return page;
+}
+
+static struct page *alloc_page_from_pool(u32 vmid)
+{
+	struct io_pgtable_pool *pool = xa_load(&page_pool_xa, vmid);
+	struct page *page;
+	unsigned long flags;
+
+	spin_lock_irqsave(&pool->pool_lock, flags);
+	page = __alloc_page_from_pool(&pool->page_pool);
+	spin_unlock_irqrestore(&pool->pool_lock, flags);
+
+	return page;
+}
+
+static void free_page_to_pool(struct page *page)
+{
+	u32 vmid = page_private(page);
+	struct io_pgtable_pool *pool = xa_load(&page_pool_xa, vmid);
+	unsigned long flags;
+
+	clear_page(page_address(page));
+	spin_lock_irqsave(&pool->pool_lock, flags);
+	list_add(&page->lru, &pool->page_pool);
+	atomic_long_inc(&page_pool_count);
+	inc_node_page_state(page, NR_KERNEL_MISC_RECLAIMABLE);
+	spin_unlock_irqrestore(&pool->pool_lock, flags);
+}
+
+/* Assumes that page_pool_xa_lock is held. */
+static void io_pgtable_pool_release(struct kref *ref)
+{
+	struct io_pgtable_pool *pool = container_of(ref, struct io_pgtable_pool, ref);
+	struct page *page;
+	bool secure_vmid = is_secure_vmid(pool->vmid);
+
+	xa_erase(&page_pool_xa, pool->vmid);
+
+	/*
+	 * There's no need to take the pool lock, as the pool is no longer accessible to other
+	 * IOMMU clients. There's no possibility for concurrent access either as this
+	 * function is only invoked when the last reference is removed.
+	 */
+	page = __alloc_page_from_pool(&pool->page_pool);
+	while (page) {
+		if (!secure_vmid || !io_pgtable_hyp_unassign_page(pool->vmid, page))
+			__free_page(page);
+
+		page = __alloc_page_from_pool(&pool->page_pool);
+	}
+
+	kfree(pool);
+}
+
+/*
+ * qcom_io_pgtable_allocator_register: Register with the io-pgtable allocator interface.
+ *
+ * @vmid: The VMID that io-pgtable memory needs to be shared with when allocated. If VMID
+ *        is 0, then page table memory will not be shared with any other VMs.
+ *
+ * On success, 0 is returned and there will be a reference held for metadata associated with
+ * @vmid. Otherwise, an error code will be returned.
+ */
+int qcom_io_pgtable_allocator_register(u32 vmid)
+{
+	struct io_pgtable_pool *pool;
+	int ret = 0;
+
+	mutex_lock(&page_pool_xa_lock);
+	pool = xa_load(&page_pool_xa, vmid);
+	if (pool) {
+		kref_get(&pool->ref);
+		goto out;
+	}
+
+	pool = kmalloc(sizeof(*pool), GFP_KERNEL);
+	if (!pool) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	pool->vmid = vmid;
+	kref_init(&pool->ref);
+	spin_lock_init(&pool->pool_lock);
+	INIT_LIST_HEAD(&pool->page_pool);
+
+	ret = xa_err(xa_store(&page_pool_xa, vmid, pool, GFP_KERNEL));
+	if (ret < 0)
+		kfree(pool);
+out:
+	mutex_unlock(&page_pool_xa_lock);
+	return ret;
+}
+
+/*
+ * qcom_io_pgtable_allocator_unregister: Unregister with the io-pgtable allocator interface.
+ *
+ * @vmid: The VMID that was used when registering with the interface with
+ *        qcom_io_pgtable_allocator_register().
+ *
+ * Decrements the references to allocator metadata for @vmid.
+ *
+ * If this call results in references to @vmid dropping to 0, then all metadata and pages
+ * associated with @vmid are released.
+ */
+void qcom_io_pgtable_allocator_unregister(u32 vmid)
+{
+	struct io_pgtable_pool *pool;
+
+	mutex_lock(&page_pool_xa_lock);
+	pool = xa_load(&page_pool_xa, vmid);
+	kref_put(&pool->ref, io_pgtable_pool_release);
+	mutex_unlock(&page_pool_xa_lock);
+}
+
+/*
+ * qcom_io_pgtable_alloc_page: Allocate page table memory from the io-pgtable allocator.
+ *
+ * @vmid: The VMID that the page table memory should be shared with.
+ * @gfp: The GFP flags to be used for allocating the page table memory.
+ *
+ * This function may sleep if memory needs to be shared with other VMs.
+ *
+ * On success, a page will be returned. The page will also have been shared with other
+ * VMs--if any. In case of an error, this function returns NULL.
+ */
+struct page *qcom_io_pgtable_alloc_page(u32 vmid, gfp_t gfp)
+{
+	struct page *page;
+
+	/*
+	 * Mapping memory for secure domains may result in having to assign page table
+	 * memory to another VMID, which can sleep. Atomic and secure domains are
+	 * not a legal combination. We can use the GFP flags to detect atomic domains,
+	 * as they will have GFP_ATOMIC set.
+	 */
+	BUG_ON(!gfpflags_allow_blocking(gfp) && is_secure_vmid(vmid));
+
+	page = alloc_page_from_pool(vmid);
+	if (page)
+		return page;
+
+	page = alloc_page(gfp);
+	if (!page)
+		return NULL;
+	/* The page may be inaccessible if this is true, so leak it. */
+	else if (is_secure_vmid(vmid) && io_pgtable_hyp_assign_page(vmid, page))
+		return NULL;
+
+	set_page_private(page, (unsigned long)vmid);
+	return page;
+}
+
+/*
+ * qcom_io_pgtable_free_page: Frees page table memory.
+ *
+ * @page: The page to be freed.
+ *
+ * We cache pages in their respective page pools to improve performance
+ * for future allocations.
+ *
+ * Export this symbol for the IOMMU driver, since it decides when
+ * page table memory is freed after TLB maintenance.
+ */
+void qcom_io_pgtable_free_page(struct page *page)
+{
+	free_page_to_pool(page);
+}
+EXPORT_SYMBOL(qcom_io_pgtable_free_page);
+
+static unsigned long io_pgtable_alloc_count_objects(struct shrinker *shrinker,
+						    struct shrink_control *sc)
+{
+	unsigned long count = atomic_long_read(&page_pool_count);
+
+	return count ? count : SHRINK_EMPTY;
+}
+
+static unsigned long scan_page_pool(struct io_pgtable_pool *pool, struct list_head *freelist,
+				    unsigned long nr_to_scan)
+{
+	struct page *page;
+	unsigned long count = 0, flags;
+
+	spin_lock_irqsave(&pool->pool_lock, flags);
+	while (count < nr_to_scan) {
+		page = __alloc_page_from_pool(&pool->page_pool);
+		if (page) {
+			list_add(&page->lru, freelist);
+			count++;
+		} else {
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&pool->pool_lock, flags);
+
+	return count;
+}
+
+static unsigned long io_pgtable_alloc_scan_objects(struct shrinker *shrinker,
+						   struct shrink_control *sc)
+{
+	struct page *page, *tmp;
+	struct io_pgtable_pool *pool;
+	unsigned long index;
+	unsigned long nr_to_scan = sc->nr_to_scan, count = 0;
+	u32 vmid;
+	LIST_HEAD(freelist);
+
+	mutex_lock(&page_pool_xa_lock);
+	xa_for_each(&page_pool_xa, index, pool) {
+		count += scan_page_pool(pool, &freelist, nr_to_scan - count);
+		if (count >= nr_to_scan)
+			break;
+	}
+	mutex_unlock(&page_pool_xa_lock);
+
+	list_for_each_entry_safe(page, tmp, &freelist, lru) {
+		vmid = page_private(page);
+		list_del(&page->lru);
+
+		if (!is_secure_vmid(vmid) || !io_pgtable_hyp_unassign_page(vmid, page))
+			__free_page(page);
+		else
+			count--;
+	}
+
+	return count;
+}
+
+static struct shrinker io_pgtable_alloc_shrinker = {
+	.count_objects = io_pgtable_alloc_count_objects,
+	.scan_objects = io_pgtable_alloc_scan_objects,
+	.seeks = DEFAULT_SEEKS,
+};
+
+int qcom_io_pgtable_alloc_init(void)
+{
+	return register_shrinker(&io_pgtable_alloc_shrinker);
+}
+
+void qcom_io_pgtable_alloc_exit(void)
+{
+	unregister_shrinker(&io_pgtable_alloc_shrinker);
+}

drivers/iommu/qcom-io-pgtable-alloc.h

@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+#ifndef __QCOM_IO_PGTABLE_ALLOC_H
+#define __QCOM_IO_PGTABLE_ALLOC_H
+
+int qcom_io_pgtable_allocator_register(u32 vmid);
+void qcom_io_pgtable_allocator_unregister(u32 vmid);
+struct page *qcom_io_pgtable_alloc_page(u32 vmid, gfp_t gfp);
+void qcom_io_pgtable_free_page(struct page *page);
+int qcom_io_pgtable_alloc_init(void);
+void qcom_io_pgtable_alloc_exit(void);
+
+#endif /* __QCOM_IO_PGTABLE_ALLOC_H */
+

drivers/iommu/qcom-io-pgtable.h

@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+#ifndef __QCOM_QCOM_IO_PGTABLE_H
+#define __QCOM_QCOM_IO_PGTABLE_H
+
+#include <linux/io-pgtable.h>
+
+struct qcom_iommu_pgtable_log_ops {
+	void (*log_new_table)(void *cookie, void *virt, unsigned long iova, size_t granule);
+	void (*log_remove_table)(void *cookie, void *virt, unsigned long iova, size_t granule);
+};
+
+struct qcom_iommu_flush_ops {
+	void (*tlb_add_walk_page)(void *cookie, void *virt);
+	void (*tlb_add_inv)(void *cookie);
+	void (*tlb_sync)(void *cookie);
+};
+
+struct qcom_io_pgtable_info {
+	struct io_pgtable_cfg cfg;
+	const struct qcom_iommu_flush_ops *iommu_tlb_ops;
+	const struct qcom_iommu_pgtable_log_ops *pgtable_log_ops;
+	/* When set to 0, all page table memory is treated as non-secure. */
+	u32 vmid;
+	dma_addr_t iova_base;
+	dma_addr_t iova_end;
+};
+
+#define to_qcom_io_pgtable_info(x)\
+container_of((x), struct qcom_io_pgtable_info, cfg)
+
+#define IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA       BIT(31)
+
+#define ARM_V8L_FAST ((unsigned int)-1)
+#define QCOM_ARM_64_LPAE_S1 ((unsigned int)-2)
+
+struct io_pgtable_ops *qcom_alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
+				struct qcom_io_pgtable_info *pgtbl_info,
+				void *cookie);
+void qcom_free_io_pgtable_ops(struct io_pgtable_ops *ops);
+
+static inline void
+qcom_io_pgtable_tlb_add_walk_page(const struct qcom_iommu_flush_ops *tlb_ops, void *cookie,
+				  void *virt)
+{
+	tlb_ops->tlb_add_walk_page(cookie, virt);
+}
+
+static inline void
+qcom_io_pgtable_tlb_add_inv(const struct qcom_iommu_flush_ops *tlb_ops, void *cookie)
+{
+	tlb_ops->tlb_add_inv(cookie);
+}
+
+static inline void
+qcom_io_pgtable_tlb_sync(const struct qcom_iommu_flush_ops *tlb_ops, void *cookie)
+{
+	tlb_ops->tlb_sync(cookie);
+}
+
+static inline void
+qcom_io_pgtable_log_new_table(const struct qcom_iommu_pgtable_log_ops *ops, void *cookie,
+			      void *virt, unsigned long iova, size_t granule)
+{
+	ops->log_new_table(cookie, virt, iova, granule);
+}
+
+static inline void
+qcom_io_pgtable_log_remove_table(const struct qcom_iommu_pgtable_log_ops *ops, void *cookie,
+				 void *virt, unsigned long iova, size_t granule)
+{
+	ops->log_remove_table(cookie, virt, iova, granule);
+}
+
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
+extern struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns;
+#endif
+#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE
+extern struct io_pgtable_init_fns qcom_io_pgtable_arm_64_lpae_s1_init_fns;
+#endif
+
+#endif /* __QCOM_QCOM_IO_PGTABLE_H */

drivers/iommu/qcom-iommu-debug.c

@@ -0,0 +1,383 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2015-2021, The Linux Foundation. All rights reserved.
+ *
+ */
+
+#define pr_fmt(fmt) "iommu-debug: %s: " fmt, __func__
+
+#include <linux/debugfs.h>
+#include <linux/iommu.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/qcom-iommu-util.h>
+#include "qcom-iommu-debug.h"
+
+#define USECASE_SWITCH_TIMEOUT_MSECS (500)
+
+static int iommu_debug_nr_iters_set(void *data, u64 val)
+{
+	struct iommu_debug_device *ddev = data;
+
+	if (!val)
+		val = 1;
+
+	if (val > 10000)
+		val = 10000;
+
+	ddev->nr_iters = (u32)val;
+
+	return 0;
+}
+
+static int iommu_debug_nr_iters_get(void *data, u64 *val)
+{
+	struct iommu_debug_device *ddev = data;
+
+	*val = ddev->nr_iters;
+
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(iommu_debug_nr_iters_fops,
+			 iommu_debug_nr_iters_get,
+			 iommu_debug_nr_iters_set,
+			 "%llu\n");
+
+int iommu_debug_check_mapping_flags(struct device *dev, dma_addr_t iova, size_t size,
+				    phys_addr_t expected_pa, u32 flags)
+{
+	struct qcom_iommu_atos_txn txn;
+	struct iommu_fwspec *fwspec;
+	struct iommu_domain *domain;
+
+	domain = iommu_get_domain_for_dev(dev);
+	if (!domain) {
+		dev_err(dev, "iommu_get_domain_for_dev() failed\n");
+		return -EINVAL;
+	}
+
+	fwspec = dev_iommu_fwspec_get(dev);
+	if (!fwspec) {
+		dev_err(dev, "dev_iommu_fwspec_get() failed\n");
+		return -EINVAL;
+	}
+
+	txn.addr = iova;
+	txn.id = FIELD_GET(ARM_SMMU_SMR_ID, (fwspec->ids[0]));
+	txn.flags = flags;
+
+	size = PAGE_ALIGN(size);
+	while (size) {
+		phys_addr_t walk_pa, atos_pa;
+
+		atos_pa = qcom_iommu_iova_to_phys_hard(domain, &txn);
+		walk_pa = iommu_iova_to_phys(domain, iova);
+
+		if (expected_pa != atos_pa || expected_pa != walk_pa) {
+			dev_err_ratelimited(dev,
+				"Bad translation for %pad! Expected: %pa Got: %pa (ATOS) %pa (Table Walk) sid=%08x\n",
+				&iova, &expected_pa, &atos_pa, &walk_pa, txn.id);
+			return -EINVAL;
+		}
+
+		size -= PAGE_SIZE;
+		iova += PAGE_SIZE;
+		expected_pa += PAGE_SIZE;
+	}
+
+	return 0;
+}
+
+int iommu_debug_check_mapping_sg_flags(struct device *dev, struct scatterlist *sgl,
+				       unsigned int pgoffset, unsigned int dma_nents,
+				       unsigned int nents, u32 flags)
+{
+	int ret;
+	struct sg_page_iter piter;
+	struct sg_dma_page_iter diter;
+
+	for (__sg_page_iter_start(&piter, sgl, nents, pgoffset),
+	     __sg_page_iter_start(&diter.base, sgl, dma_nents, pgoffset);
+	     __sg_page_iter_next(&piter) && __sg_page_iter_dma_next(&diter);) {
+
+		struct page *page = sg_page_iter_page(&piter);
+		dma_addr_t dma_addr = sg_page_iter_dma_address(&diter);
+
+		ret = iommu_debug_check_mapping_flags(dev, dma_addr, PAGE_SIZE,
+						      page_to_phys(page), flags);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void iommu_debug_destroy_test_dev(struct iommu_debug_device *ddev)
+{
+	if (ddev->test_dev) {
+		of_platform_device_destroy(ddev->test_dev, NULL);
+		ddev->test_dev = NULL;
+		ddev->domain = NULL;
+	}
+}
+
+/*
+ * Returns struct device corresponding to the new usecase.
+ * ddev->test_dev will change - caller must not use old value!
+ * Caller must hold ddev->state_lock
+ */
+struct device *
+iommu_debug_switch_usecase(struct iommu_debug_device *ddev, u32 usecase_nr)
+{
+	struct platform_device *test_pdev;
+	struct device_node *child;
+	int child_nr = 0;
+	int ret;
+
+	if (ddev->test_dev)
+		iommu_debug_destroy_test_dev(ddev);
+
+	if (usecase_nr >= of_get_child_count(ddev->self->of_node)) {
+		dev_err(ddev->self, "Invalid usecase nr requested: %u\n",
+			usecase_nr);
+		return NULL;
+	}
+
+	reinit_completion(&ddev->probe_wait);
+	for_each_child_of_node(ddev->self->of_node, child) {
+		if (child_nr == usecase_nr)
+			break;
+		child_nr++;
+	}
+
+	test_pdev = of_platform_device_create(child, NULL, ddev->self);
+	if (!test_pdev) {
+		dev_err(ddev->self, "Creating platform device failed\n");
+		return NULL;
+	}
+
+	/*
+	 * Wait for child device's probe function to be called.
+	 * Its very unlikely to be asynchonrous...
+	 */
+	ret = wait_for_completion_interruptible_timeout(&ddev->probe_wait,
+						msecs_to_jiffies(USECASE_SWITCH_TIMEOUT_MSECS));
+	if (ret <= 0) {
+		dev_err(ddev->self, "Timed out waiting for usecase to register\n");
+		goto out;
+	}
+
+	ddev->usecase_nr = usecase_nr;
+	ddev->test_dev = &test_pdev->dev;
+	ddev->domain = iommu_get_domain_for_dev(ddev->test_dev);
+	if (!ddev->domain) {
+		dev_err(ddev->self, "Oops, usecase not associated with iommu\n");
+		goto out;
+	}
+
+	return ddev->test_dev;
+out:
+	iommu_debug_destroy_test_dev(ddev);
+	return NULL;
+}
+
+/*
+ * Caller must hold ddev->state_lock
+ */
+struct device *iommu_debug_usecase_reset(struct iommu_debug_device *ddev)
+{
+	return iommu_debug_switch_usecase(ddev, ddev->usecase_nr);
+}
+
+static int iommu_debug_usecase_register(struct device *dev)
+{
+	struct iommu_debug_device *ddev = dev_get_drvdata(dev->parent);
+
+	complete(&ddev->probe_wait);
+	return 0;
+}
+
+static ssize_t iommu_debug_usecase_read(struct file *file, char __user *ubuf,
+					size_t count, loff_t *offset)
+{
+	struct iommu_debug_device *ddev = file->private_data;
+
+	return simple_read_from_buffer(ubuf, count, offset, ddev->buffer,
+				       strnlen(ddev->buffer, PAGE_SIZE));
+}
+
+static ssize_t iommu_debug_usecase_write(struct file *file, const char __user *ubuf,
+					 size_t count, loff_t *offset)
+{
+	struct iommu_debug_device *ddev = file->private_data;
+	unsigned int usecase_nr;
+	int ret;
+
+	ret = kstrtouint_from_user(ubuf, count, 0, &usecase_nr);
+	if (ret || usecase_nr >= ddev->nr_children)
+		return -EINVAL;
+
+	mutex_lock(&ddev->state_lock);
+	if (!iommu_debug_switch_usecase(ddev, usecase_nr)) {
+		mutex_unlock(&ddev->state_lock);
+		return -EINVAL;
+	}
+	mutex_unlock(&ddev->state_lock);
+
+	return count;
+}
+
+static const struct file_operations iommu_debug_usecase_fops = {
+	.open	 = simple_open,
+	.read	 = iommu_debug_usecase_read,
+	.write   = iommu_debug_usecase_write,
+	.llseek	 = no_llseek,
+};
+
+static int iommu_debug_debugfs_setup(struct iommu_debug_device *ddev)
+{
+	struct dentry *dir;
+
+	dir = debugfs_create_dir("iommu-test", NULL);
+	if (IS_ERR(dir))
+		return -EINVAL;
+
+	ddev->root_dir = dir;
+
+	debugfs_create_file("usecase", 0600, dir, ddev, &iommu_debug_usecase_fops);
+	debugfs_create_file("functional_arm_dma_api", 0400, dir, ddev,
+			    &iommu_debug_functional_arm_dma_api_fops);
+	debugfs_create_file("functional_fast_dma_api", 0400, dir, ddev,
+			    &iommu_debug_functional_fast_dma_api_fops);
+	debugfs_create_file("atos", 0600, dir, ddev, &iommu_debug_atos_fops);
+	debugfs_create_file("map", 0200, dir, ddev, &iommu_debug_map_fops);
+	debugfs_create_file("unmap", 0200, dir, ddev, &iommu_debug_unmap_fops);
+	debugfs_create_file("dma_map", 0200, dir, ddev, &iommu_debug_dma_map_fops);
+	debugfs_create_file("dma_unmap", 0200, dir, ddev, &iommu_debug_dma_unmap_fops);
+	debugfs_create_file("nr_iters", 0600, dir, ddev, &iommu_debug_nr_iters_fops);
+	debugfs_create_file("test_virt_addr", 0400, dir, ddev, &iommu_debug_test_virt_addr_fops);
+	debugfs_create_file("profiling", 0400, dir, ddev, &iommu_debug_profiling_fops);
+
+	return 0;
+}
+
+static int iommu_debug_probe(struct platform_device *pdev)
+{
+	struct iommu_debug_device *ddev;
+	struct device *dev = &pdev->dev;
+	struct device_node *child;
+	int ret;
+	int offset = 0;
+
+	ddev = devm_kzalloc(dev, sizeof(*ddev), GFP_KERNEL);
+	if (!ddev)
+		return -ENOMEM;
+
+	ddev->self = dev;
+	ddev->usecase_nr = U32_MAX;
+	ddev->nr_iters = 1;
+	mutex_init(&ddev->state_lock);
+	init_completion(&ddev->probe_wait);
+
+	ddev->buffer = devm_kzalloc(dev, PAGE_SIZE, GFP_KERNEL);
+	if (!ddev->buffer) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ddev->nr_children = 0;
+	for_each_child_of_node(dev->of_node, child) {
+		offset += scnprintf(ddev->buffer + offset, PAGE_SIZE - offset,
+				"%d: %s\n", ddev->nr_children, child->name);
+		if (offset + 1 == PAGE_SIZE) {
+			dev_err(dev, "Too many testcases?\n");
+			break;
+		}
+		ddev->nr_children++;
+	}
+	dev_set_drvdata(dev, ddev);
+
+	ret = iommu_debug_debugfs_setup(ddev);
+	if (ret)
+		goto out;
+
+	return 0;
+
+out:
+	mutex_destroy(&ddev->state_lock);
+	return ret;
+}
+
+static int iommu_debug_remove(struct platform_device *pdev)
+{
+	struct iommu_debug_device *ddev = platform_get_drvdata(pdev);
+
+	debugfs_remove_recursive(ddev->root_dir);
+	if (ddev->test_dev)
+		of_platform_device_destroy(ddev->test_dev, NULL);
+
+	mutex_destroy(&ddev->state_lock);
+	return 0;
+}
+
+static const struct of_device_id iommu_debug_of_match[] = {
+	{ .compatible = "qcom,iommu-debug-test" },
+	{ },
+};
+
+static struct platform_driver iommu_debug_driver = {
+	.probe = iommu_debug_probe,
+	.remove = iommu_debug_remove,
+	.driver = {
+		.name = "qcom-iommu-debug",
+		.of_match_table = iommu_debug_of_match,
+	},
+};
+
+/*
+ * This isn't really a "driver", we just need something in the device tree
+ * to hook up to the `iommus' property.
+ */
+static int iommu_debug_usecase_probe(struct platform_device *pdev)
+{
+	return iommu_debug_usecase_register(&pdev->dev);
+}
+
+static const struct of_device_id iommu_debug_usecase_of_match[] = {
+	{ .compatible = "qcom,iommu-debug-usecase" },
+	{ },
+};
+
+static struct platform_driver iommu_debug_usecase_driver = {
+	.probe = iommu_debug_usecase_probe,
+	.driver = {
+		.name = "qcom-iommu-debug-usecase",
+		.of_match_table = iommu_debug_usecase_of_match,
+	},
+};
+
+static int iommu_debug_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&iommu_debug_driver);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&iommu_debug_usecase_driver);
+	if (ret)
+		platform_driver_unregister(&iommu_debug_driver);
+	return ret;
+}
+
+static void iommu_debug_exit(void)
+{
+	platform_driver_unregister(&iommu_debug_usecase_driver);
+	platform_driver_unregister(&iommu_debug_driver);
+}
+
+module_init(iommu_debug_init);
+module_exit(iommu_debug_exit);
+
+MODULE_LICENSE("GPL v2");

drivers/iommu/qcom-iommu-debug.h

@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2015-2021, The Linux Foundation. All rights reserved.
+ *
+ */
+
+#ifndef __DRIVERS_IOMMU_QCOM_IOMMU_DEBUG_H__
+#define __DRIVERS_IOMMU_QCOM_IOMMU_DEBUG_H__
+
+#include <linux/device.h>
+#include <linux/debugfs.h>
+#include <linux/iommu.h>
+#include <linux/completion.h>
+#include <linux/mutex.h>
+
+#define MSI_IOVA_BASE			0x8000000
+#define MSI_IOVA_LENGTH			0x100000
+#define ARM_SMMU_SMR_ID			GENMASK(15, 0)
+
+struct iommu_debug_device {
+	struct device *self;
+	u32 nr_children;
+	char *buffer;
+	struct dentry *root_dir;
+	/* for usecase under test */
+	struct device *test_dev;
+	struct iommu_domain *domain;
+	u32 usecase_nr;
+	/* Protects test_dev */
+	struct mutex state_lock;
+	/* For waiting for child probe to complete */
+	struct completion probe_wait;
+	/* Used for atos */
+	u64 iova;
+	/* number of iterations */
+	u32 nr_iters;
+};
+
+struct device *iommu_debug_usecase_reset(struct iommu_debug_device *ddev);
+struct device *iommu_debug_switch_usecase(struct iommu_debug_device *ddev, u32 usecase_nr);
+
+int iommu_debug_check_mapping_flags(struct device *dev, dma_addr_t iova, size_t size,
+				    phys_addr_t expected_pa, u32 flags);
+#define iommu_debug_check_mapping(d, i, s, p) \
+	iommu_debug_check_mapping_flags(d, i, s, p, 0)
+/* Only checks a single page */
+#define iommu_debug_check_mapping_fast(d, i, s, p) \
+	iommu_debug_check_mapping_flags(d, i, PAGE_SIZE, p, 0)
+
+int iommu_debug_check_mapping_sg_flags(struct device *dev, struct scatterlist *sgl,
+				       unsigned int pgoffset, unsigned int dma_nents,
+				       unsigned int nents, u32 flags);
+#define iommu_debug_check_mapping_sg(d, s, o, e1, e2) \
+	iommu_debug_check_mapping_sg_flags(d, s, o, e1, e2, 0)
+
+/* Only checks the last page of first sgl */
+static inline int iommu_debug_check_mapping_sg_fast(struct device *dev, struct scatterlist *sgl,
+						    unsigned int pgoffset, unsigned int dma_nents,
+						    unsigned int nents)
+{
+	pgoffset = PAGE_ALIGN(sgl->offset + sgl->length) >> PAGE_SHIFT;
+	return iommu_debug_check_mapping_sg_flags(dev, sgl, pgoffset - 1, dma_nents, 1, 0);
+}
+
+extern const struct file_operations iommu_debug_functional_arm_dma_api_fops;
+extern const struct file_operations iommu_debug_functional_fast_dma_api_fops;
+extern const struct file_operations iommu_debug_atos_fops;
+extern const struct file_operations iommu_debug_map_fops;
+extern const struct file_operations iommu_debug_unmap_fops;
+extern const struct file_operations iommu_debug_dma_map_fops;
+extern const struct file_operations iommu_debug_dma_unmap_fops;
+extern const struct file_operations iommu_debug_test_virt_addr_fops;
+extern const struct file_operations iommu_debug_profiling_fops;
+
+#endif

drivers/iommu/qcom-iommu-util.c

@@ -0,0 +1,529 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/dma-mapping-fast.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/qcom-iommu-util.h>
+#include "qcom-dma-iommu-generic.h"
+#include "qcom-io-pgtable.h"
+#include "qcom-io-pgtable-alloc.h"
+
+struct qcom_iommu_range_prop_cb_data {
+	int (*range_prop_entry_cb_fn)(const __be32 *p, int naddr, int nsize, void *arg);
+	void *arg;
+};
+
+struct iova_range {
+	u64 base;
+	u64 end;
+};
+
+struct device_node *qcom_iommu_group_parse_phandle(struct device *dev)
+{
+	struct device_node *np;
+
+	if (!dev->of_node)
+		return NULL;
+
+	np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0);
+	return np ? np : dev->of_node;
+}
+
+static int of_property_walk_each_entry(struct device *dev, const char *propname,
+				       struct qcom_iommu_range_prop_cb_data *cb_data)
+{
+	struct device_node *np;
+	const __be32 *p, *property_end;
+	int ret, len, naddr, nsize;
+
+	np = qcom_iommu_group_parse_phandle(dev);
+	if (!np)
+		return -EINVAL;
+
+	p = of_get_property(np, propname, &len);
+	if (!p)
+		return -ENODEV;
+
+	len /= sizeof(u32);
+	naddr = of_n_addr_cells(np);
+	nsize = of_n_size_cells(np);
+	if (!naddr || !nsize || len % (naddr + nsize)) {
+		dev_err(dev, "%s Invalid length %d. Address cells %d. Size cells %d\n",
+			propname, len, naddr, nsize);
+		return -EINVAL;
+	}
+	property_end = p + len;
+
+	while (p < property_end) {
+		ret = cb_data->range_prop_entry_cb_fn(p, naddr, nsize, cb_data->arg);
+		if (ret)
+			return ret;
+
+		p += naddr + nsize;
+	}
+
+	return 0;
+}
+
+static bool check_overlap(struct iommu_resv_region *region, u64 start, u64 end)
+{
+	u64 region_end = region->start + region->length - 1;
+
+	return end >= region->start && start <= region_end;
+}
+
+static int insert_range(const __be32 *p, int naddr, int nsize, void *arg)
+{
+	struct list_head *head = arg;
+	struct iommu_resv_region *region, *new;
+	u64 start = of_read_number(p, naddr);
+	u64 end = start + of_read_number(p + naddr, nsize) - 1;
+
+	list_for_each_entry(region, head, list) {
+		if (check_overlap(region, start, end))
+			return -EINVAL;
+
+		if (start < region->start)
+			break;
+	}
+
+	new = iommu_alloc_resv_region(start, end - start + 1,
+					0, IOMMU_RESV_RESERVED);
+	if (!new)
+		return -ENOMEM;
+	list_add_tail(&new->list, &region->list);
+	return 0;
+}
+
+/*
+ * Returns a sorted list of all regions described by the
+ * "qcom,iommu-dma-addr-pool" property.
+ *
+ * Caller is responsible for freeing the entries on the list via
+ * generic_iommu_put_resv_regions
+ */
+int qcom_iommu_generate_dma_regions(struct device *dev,
+		struct list_head *head)
+{
+	struct qcom_iommu_range_prop_cb_data insert_range_cb_data = {
+		.range_prop_entry_cb_fn = insert_range,
+		.arg = head,
+	};
+
+	return of_property_walk_each_entry(dev, "qcom,iommu-dma-addr-pool",
+					   &insert_range_cb_data);
+}
+EXPORT_SYMBOL(qcom_iommu_generate_dma_regions);
+
+static int invert_regions(struct list_head *head, struct list_head *inverted)
+{
+	struct iommu_resv_region *prev, *curr, *new;
+	phys_addr_t rsv_start;
+	size_t rsv_size;
+	int ret = 0;
+
+	/*
+	 * Since its not possible to express start 0, size 1<<64 return
+	 * an error instead. Also an iova allocator without any iovas doesn't
+	 * make sense.
+	 */
+	if (list_empty(head))
+		return -EINVAL;
+
+	/*
+	 * Handle case where there is a non-zero sized area between
+	 * iommu_resv_regions A & B.
+	 */
+	prev = NULL;
+	list_for_each_entry(curr, head, list) {
+		if (!prev)
+			goto next;
+
+		rsv_start = prev->start + prev->length;
+		rsv_size = curr->start - rsv_start;
+		if (!rsv_size)
+			goto next;
+
+		new = iommu_alloc_resv_region(rsv_start, rsv_size,
+						0, IOMMU_RESV_RESERVED);
+		if (!new) {
+			ret = -ENOMEM;
+			goto out_err;
+		}
+		list_add_tail(&new->list, inverted);
+next:
+		prev = curr;
+	}
+
+	/* Now handle the beginning */
+	curr = list_first_entry(head, struct iommu_resv_region, list);
+	rsv_start = 0;
+	rsv_size = curr->start;
+	if (rsv_size) {
+		new = iommu_alloc_resv_region(rsv_start, rsv_size,
+						0, IOMMU_RESV_RESERVED);
+		if (!new) {
+			ret = -ENOMEM;
+			goto out_err;
+		}
+		list_add(&new->list, inverted);
+	}
+
+	/* Handle the end - checking for overflow */
+	rsv_start = prev->start + prev->length;
+	rsv_size = -rsv_start;
+
+	if (rsv_size && (U64_MAX - prev->start > prev->length)) {
+		new = iommu_alloc_resv_region(rsv_start, rsv_size,
+						0, IOMMU_RESV_RESERVED);
+		if (!new) {
+			ret = -ENOMEM;
+			goto out_err;
+		}
+		list_add_tail(&new->list, inverted);
+	}
+
+	return 0;
+
+out_err:
+	list_for_each_entry_safe(curr, prev, inverted, list)
+		kfree(curr);
+	return ret;
+}
+
+/* Used by iommu drivers to generate reserved regions for qcom,iommu-dma-addr-pool property */
+void qcom_iommu_generate_resv_regions(struct device *dev,
+				      struct list_head *head)
+{
+	struct iommu_resv_region *region;
+	LIST_HEAD(dma_regions);
+	LIST_HEAD(resv_regions);
+	int ret;
+
+	ret = qcom_iommu_generate_dma_regions(dev, &dma_regions);
+	if (ret)
+		return;
+
+	ret = invert_regions(&dma_regions, &resv_regions);
+	generic_iommu_put_resv_regions(dev, &dma_regions);
+	if (ret)
+		return;
+
+	list_for_each_entry(region, &resv_regions, list) {
+		dev_dbg(dev, "Reserved region %llx-%llx\n",
+			(u64)region->start,
+			(u64)(region->start + region->length - 1));
+	}
+
+	list_splice(&resv_regions, head);
+}
+EXPORT_SYMBOL(qcom_iommu_generate_resv_regions);
+
+void qcom_iommu_get_resv_regions(struct device *dev, struct list_head *list)
+{
+	const struct iommu_ops *ops = dev->bus->iommu_ops;
+
+	if (ops && ops->get_resv_regions)
+		ops->get_resv_regions(dev, list);
+}
+EXPORT_SYMBOL(qcom_iommu_get_resv_regions);
+
+void qcom_iommu_put_resv_regions(struct device *dev, struct list_head *list)
+{
+	const struct iommu_ops *ops = dev->bus->iommu_ops;
+
+	if (ops && ops->put_resv_regions)
+		ops->put_resv_regions(dev, list);
+}
+EXPORT_SYMBOL(qcom_iommu_put_resv_regions);
+
+static int get_addr_range(const __be32 *p, int naddr, int nsize, void *arg)
+{
+
+	u64 start = of_read_number(p, naddr);
+	u64 end = start + of_read_number(p + naddr, nsize) - 1;
+	struct iova_range *range = arg;
+
+	if (start >= SZ_4G || end >= SZ_4G) {
+		pr_err("fastmap does not support IOVAs >= 4 GB\n");
+		return -EINVAL;
+	}
+
+	range->base = min_not_zero(range->base, start);
+	range->end = max(range->end, end);
+
+	return 0;
+}
+
+int qcom_iommu_get_fast_iova_range(struct device *dev, dma_addr_t *ret_iova_base,
+				   dma_addr_t *ret_iova_end)
+{
+	struct iova_range dma_range = {};
+	struct iova_range geometry_range = {};
+	struct qcom_iommu_range_prop_cb_data get_addr_range_cb_data = {
+		.range_prop_entry_cb_fn = get_addr_range,
+	};
+	int ret;
+
+	if (!dev || !ret_iova_base || !ret_iova_end)
+		return -EINVAL;
+
+	get_addr_range_cb_data.arg = &dma_range;
+	ret = of_property_walk_each_entry(dev, "qcom,iommu-dma-addr-pool",
+					  &get_addr_range_cb_data);
+	if (ret == -ENODEV) {
+		dma_range.base = 0;
+		dma_range.end = SZ_4G - 1;
+	} else if (ret) {
+		return ret;
+	}
+
+	get_addr_range_cb_data.arg = &geometry_range;
+	ret = of_property_walk_each_entry(dev, "qcom,iommu-geometry",
+					  &get_addr_range_cb_data);
+	if (ret == -ENODEV) {
+		geometry_range.base = 0;
+		geometry_range.end = SZ_4G - 1;
+	} else if (ret) {
+		return ret;
+	}
+
+	*ret_iova_base = min(geometry_range.base, dma_range.base);
+	*ret_iova_end =  max(geometry_range.end, dma_range.end);
+	return 0;
+}
+EXPORT_SYMBOL(qcom_iommu_get_fast_iova_range);
+
+phys_addr_t qcom_iommu_iova_to_phys_hard(struct iommu_domain *domain,
+				    struct qcom_iommu_atos_txn *txn)
+{
+	struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops);
+
+	if (unlikely(ops->iova_to_phys_hard == NULL))
+		return 0;
+
+	return ops->iova_to_phys_hard(domain, txn);
+}
+EXPORT_SYMBOL(qcom_iommu_iova_to_phys_hard);
+
+int qcom_iommu_sid_switch(struct device *dev, enum sid_switch_direction dir)
+{
+	struct qcom_iommu_ops *ops;
+	struct iommu_domain *domain;
+
+	domain = iommu_get_domain_for_dev(dev);
+	if (!domain)
+		return -EINVAL;
+
+	ops = to_qcom_iommu_ops(domain->ops);
+	if (unlikely(ops->sid_switch == NULL))
+		return -EINVAL;
+
+	return ops->sid_switch(dev, dir);
+}
+EXPORT_SYMBOL(qcom_iommu_sid_switch);
+
+int qcom_iommu_get_fault_ids(struct iommu_domain *domain,
+			     struct qcom_iommu_fault_ids *f_ids)
+{
+	struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops);
+
+	if (unlikely(ops->get_fault_ids == NULL))
+		return -EINVAL;
+
+	return ops->get_fault_ids(domain, f_ids);
+}
+EXPORT_SYMBOL(qcom_iommu_get_fault_ids);
+
+int qcom_iommu_get_msi_size(struct device *dev, u32 *msi_size)
+{
+	struct device_node *np = qcom_iommu_group_parse_phandle(dev);
+
+	if (!np)
+		return -EINVAL;
+
+	return of_property_read_u32(np, "qcom,iommu-msi-size", msi_size);
+}
+
+int qcom_iommu_get_context_bank_nr(struct iommu_domain *domain)
+{
+	struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops);
+
+	if (unlikely(ops->get_context_bank_nr == NULL))
+		return -EINVAL;
+
+	return ops->get_context_bank_nr(domain);
+}
+EXPORT_SYMBOL(qcom_iommu_get_context_bank_nr);
+
+int qcom_iommu_set_secure_vmid(struct iommu_domain *domain, enum vmid vmid)
+{
+	struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops);
+
+	if (unlikely(ops->set_secure_vmid == NULL))
+		return -EINVAL;
+
+	return ops->set_secure_vmid(domain, vmid);
+}
+EXPORT_SYMBOL(qcom_iommu_set_secure_vmid);
+
+int qcom_iommu_set_fault_model(struct iommu_domain *domain, int fault_model)
+{
+	struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops);
+
+	if (unlikely(ops->set_fault_model == NULL))
+		return -EINVAL;
+	else if (fault_model & ~(QCOM_IOMMU_FAULT_MODEL_NON_FATAL |
+				 QCOM_IOMMU_FAULT_MODEL_NO_CFRE |
+				 QCOM_IOMMU_FAULT_MODEL_NO_STALL |
+				 QCOM_IOMMU_FAULT_MODEL_HUPCF))
+		return -EINVAL;
+
+	return ops->set_fault_model(domain, fault_model);
+}
+EXPORT_SYMBOL(qcom_iommu_set_fault_model);
+
+int qcom_iommu_enable_s1_translation(struct iommu_domain *domain)
+{
+	struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops);
+
+	if (unlikely(ops->enable_s1_translation == NULL))
+		return -EINVAL;
+
+	return ops->enable_s1_translation(domain);
+}
+EXPORT_SYMBOL(qcom_iommu_enable_s1_translation);
+
+int qcom_iommu_get_mappings_configuration(struct iommu_domain *domain)
+{
+	struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops);
+
+	if (unlikely(ops->get_mappings_configuration == NULL))
+		return -EINVAL;
+
+	return ops->get_mappings_configuration(domain);
+}
+EXPORT_SYMBOL(qcom_iommu_get_mappings_configuration);
+
+struct io_pgtable_ops *qcom_alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
+				struct qcom_io_pgtable_info *pgtbl_info,
+				void *cookie)
+{
+	struct io_pgtable *iop;
+	const struct io_pgtable_init_fns *fns;
+	struct io_pgtable_cfg *cfg = &pgtbl_info->cfg;
+
+	if (fmt < IO_PGTABLE_NUM_FMTS)
+		return alloc_io_pgtable_ops(fmt, cfg, cookie);
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
+	else if (fmt == ARM_V8L_FAST)
+		fns = &io_pgtable_av8l_fast_init_fns;
+#endif
+#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE
+	else if (fmt == QCOM_ARM_64_LPAE_S1)
+		fns = &qcom_io_pgtable_arm_64_lpae_s1_init_fns;
+#endif
+	else {
+		pr_err("Invalid io-pgtable fmt %u\n", fmt);
+		return NULL;
+	}
+
+	iop = fns->alloc(cfg, cookie);
+	if (!iop)
+		return NULL;
+
+	iop->fmt	= fmt;
+	iop->cookie	= cookie;
+	iop->cfg	= *cfg;
+
+	return &iop->ops;
+}
+EXPORT_SYMBOL(qcom_alloc_io_pgtable_ops);
+
+void qcom_free_io_pgtable_ops(struct io_pgtable_ops *ops)
+{
+	struct io_pgtable *iop;
+	enum io_pgtable_fmt fmt;
+	const struct io_pgtable_init_fns *fns;
+
+	if (!ops)
+		return;
+
+	iop = io_pgtable_ops_to_pgtable(ops);
+	fmt = iop->fmt;
+	if (fmt < IO_PGTABLE_NUM_FMTS)
+		return free_io_pgtable_ops(ops);
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
+	else if (fmt == ARM_V8L_FAST)
+		fns = &io_pgtable_av8l_fast_init_fns;
+#endif
+#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE
+	else if (fmt == QCOM_ARM_64_LPAE_S1)
+		fns = &qcom_io_pgtable_arm_64_lpae_s1_init_fns;
+#endif
+	else {
+		pr_err("Invalid io-pgtable fmt %u\n", fmt);
+		return;
+	}
+
+	io_pgtable_tlb_flush_all(iop);
+	fns->free(iop);
+}
+EXPORT_SYMBOL(qcom_free_io_pgtable_ops);
+
+/*
+ * These tables must have the same length.
+ * It is allowed to have a NULL exitcall corresponding to a non-NULL initcall.
+ */
+static initcall_t init_table[] __initdata = {
+	dma_mapping_fast_init,
+	qcom_dma_iommu_generic_driver_init,
+	qcom_arm_lpae_do_selftests,
+	qcom_io_pgtable_alloc_init,
+	NULL
+};
+
+static exitcall_t exit_table[] = {
+	NULL, /* dma_mapping_fast_exit */
+	qcom_dma_iommu_generic_driver_exit,
+	NULL, /*qcom_arm_lpae_do_selftests */
+	qcom_io_pgtable_alloc_exit,
+	NULL,
+};
+
+static int __init qcom_iommu_util_init(void)
+{
+	initcall_t *init_fn;
+	exitcall_t *exit_fn;
+	int ret;
+
+	if (ARRAY_SIZE(init_table) != ARRAY_SIZE(exit_table)) {
+		pr_err("qcom-iommu-util: Invalid initcall/exitcall table\n");
+		return -EINVAL;
+	}
+
+	for (init_fn = init_table; *init_fn; init_fn++) {
+		ret = (**init_fn)();
+		if (ret) {
+			pr_err("%ps returned %d\n", *init_fn, ret);
+			goto out_undo;
+		}
+	}
+
+	return 0;
+
+out_undo:
+	exit_fn = exit_table + (init_fn - init_table);
+	for (exit_fn--; exit_fn >= exit_table; exit_fn--) {
+		if (!*exit_fn)
+			continue;
+		(**exit_fn)();
+	}
+	return ret;
+}
+module_init(qcom_iommu_util_init);
+
+MODULE_LICENSE("GPL v2");

include/linux/dma-mapping-fast.h

@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2016-2019, 2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef __LINUX_DMA_MAPPING_FAST_H
+#define __LINUX_DMA_MAPPING_FAST_H
+
+#include <linux/iommu.h>
+#include <linux/io-pgtable-fast.h>
+#include <linux/rbtree.h>
+
+struct dma_iommu_mapping;
+struct io_pgtable_ops;
+struct iova_domain;
+
+struct dma_fast_smmu_mapping {
+	struct device		*dev;
+	struct iommu_domain	*domain;
+	struct iova_domain	*iovad;
+
+	dma_addr_t	 base;
+	size_t		 size;
+	size_t		 num_4k_pages;
+
+	unsigned int	bitmap_size;
+	/* bitmap has 1s marked only valid mappings */
+	unsigned long	*bitmap;
+	/* clean_bitmap has 1s marked for both valid and stale tlb mappings */
+	unsigned long	*clean_bitmap;
+	unsigned long	next_start;
+	bool		have_stale_tlbs;
+
+	dma_addr_t	pgtbl_dma_handle;
+	struct io_pgtable_ops *pgtbl_ops;
+
+	spinlock_t	lock;
+	struct notifier_block notifier;
+	struct rb_node node;
+};
+
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
+int fast_smmu_init_mapping(struct device *dev, struct iommu_domain *domain,
+			   struct io_pgtable_ops *pgtable_ops);
+void fast_smmu_put_dma_cookie(struct iommu_domain *domain);
+void fast_smmu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size);
+int __init dma_mapping_fast_init(void);
+void dma_mapping_fast_exit(void);
+#else
+static inline int fast_smmu_init_mapping(struct device *dev,
+					 struct iommu_domain *domain,
+					 struct io_pgtable_ops *pgtable_ops)
+{
+	return -ENODEV;
+}
+
+static inline void fast_smmu_put_dma_cookie(struct iommu_domain *domain) {}
+static inline void fast_smmu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size) {}
+
+static inline int __init dma_mapping_fast_init(void)
+{
+	return 0;
+}
+
+static inline void dma_mapping_fast_exit(void) {}
+#endif
+
+#endif /* __LINUX_DMA_MAPPING_FAST_H */

include/linux/io-pgtable-fast.h

@@ -0,0 +1,126 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef __LINUX_IO_PGTABLE_FAST_H
+#define __LINUX_IO_PGTABLE_FAST_H
+
+#include <linux/notifier.h>
+#include <linux/io-pgtable.h>
+/*
+ * This ought to be private to io-pgtable-fast, but dma-mapping-fast
+ * currently requires it for a debug usecase.
+ */
+typedef u64 av8l_fast_iopte;
+
+struct io_pgtable_ops;
+struct scatterlist;
+
+struct av8l_fast_io_pgtable {
+	struct io_pgtable	  iop;
+	av8l_fast_iopte		 *pgd;
+	av8l_fast_iopte		 *puds[4];
+	av8l_fast_iopte		 *pmds;
+	struct page		**pages; /* page table memory */
+	int			  nr_pages;
+	dma_addr_t		  base;
+	dma_addr_t		  end;
+};
+
+/* Struct accessors */
+#define iof_pgtable_to_data(x)						\
+	container_of((x), struct av8l_fast_io_pgtable, iop)
+
+#define iof_pgtable_ops_to_pgtable(x)					\
+	container_of((x), struct io_pgtable, ops)
+
+#define iof_pgtable_ops_to_data(x)					\
+	iof_pgtable_to_data(iof_pgtable_ops_to_pgtable(x))
+
+
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST
+
+int av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova,
+			 phys_addr_t paddr, size_t size, int prot);
+
+void av8l_fast_unmap_public(struct io_pgtable_ops *ops, unsigned long iova,
+				size_t size);
+
+int av8l_fast_map_sg_public(struct io_pgtable_ops *ops,
+			unsigned long iova, struct scatterlist *sgl,
+			unsigned int nents, int prot, size_t *size);
+
+bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops,
+					unsigned long iova);
+
+phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops,
+					  unsigned long iova);
+#else
+static inline int
+av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova,
+		     phys_addr_t paddr, size_t size, int prot)
+{
+	return -EINVAL;
+}
+static inline void av8l_fast_unmap_public(struct io_pgtable_ops *ops,
+					  unsigned long iova, size_t size)
+{
+}
+
+static inline int av8l_fast_map_sg_public(struct io_pgtable_ops *ops,
+				unsigned long iova, struct scatterlist *sgl,
+				unsigned int nents, int prot, size_t *size)
+{
+	return 0;
+}
+
+static inline bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops,
+						  unsigned long iova)
+{
+	return false;
+}
+static inline phys_addr_t
+av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops,
+				  unsigned long iova)
+{
+	return 0;
+}
+#endif /* CONFIG_IOMMU_IO_PGTABLE_FAST */
+
+
+/* events for notifiers passed to av8l_register_notify */
+#define MAPPED_OVER_STALE_TLB 1
+
+
+#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB
+/*
+ * Doesn't matter what we use as long as bit 0 is unset.  The reason why we
+ * need a different value at all is that there are certain hardware
+ * platforms with erratum that require that a PTE actually be zero'd out
+ * and not just have its valid bit unset.
+ */
+#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0xa
+
+void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, u64 base, u64 end,
+				bool skip_sync);
+void av8l_register_notify(struct notifier_block *nb);
+
+#else  /* !CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */
+
+#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0
+
+static inline void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops,
+					      u64 base,
+					      u64 end,
+					      bool skip_sync)
+{
+}
+
+static inline void av8l_register_notify(struct notifier_block *nb)
+{
+}
+
+#endif	/* CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */
+
+#endif /* __LINUX_IO_PGTABLE_FAST_H */

include/linux/qcom-iommu-util.h

@@ -0,0 +1,143 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef __QCOM_IOMMU_UTIL_H
+#define __QCOM_IOMMU_UTIL_H
+
+#include <linux/iommu.h>
+#include <linux/dma-mapping.h>
+#include <linux/iova.h>
+
+#include <soc/qcom/secure_buffer.h>
+
+/* IOMMU fault behaviors */
+#define QCOM_IOMMU_FAULT_MODEL_NON_FATAL	BIT(0)
+#define QCOM_IOMMU_FAULT_MODEL_NO_CFRE		BIT(1)
+#define QCOM_IOMMU_FAULT_MODEL_NO_STALL		BIT(2)
+#define QCOM_IOMMU_FAULT_MODEL_HUPCF		BIT(3)
+
+/* IOMMU mapping configurations */
+#define QCOM_IOMMU_MAPPING_CONF_S1_BYPASS	BIT(0)
+#define QCOM_IOMMU_MAPPING_CONF_ATOMIC		BIT(1)
+#define QCOM_IOMMU_MAPPING_CONF_FAST		BIT(2)
+
+/* iommu transaction flags */
+/* 1 Write, 0 Read */
+#define QCOM_IOMMU_ATOS_TRANS_WRITE	BIT(0)
+/* 1 Privileged, 0 Unprivileged */
+#define QCOM_IOMMU_ATOS_TRANS_PRIV	BIT(1)
+/* 1 Instruction fetch, 0 Data access */
+#define QCOM_IOMMU_ATOS_TRANS_INST	BIT(2)
+/* Non secure unprivileged Data read operation */
+#define QCOM_IOMMU_ATOS_TRANS_DEFAULT	(0U)
+
+#ifndef IOMMU_SYS_CACHE
+/* Attributes are not supported, so render them ineffective. */
+#define IOMMU_SYS_CACHE		(0)
+#define IOMMU_SYS_CACHE_NWA	(0)
+#endif
+
+/* Use upstream device's bus attribute */
+#define IOMMU_USE_UPSTREAM_HINT	(IOMMU_SYS_CACHE)
+
+/* Use upstream device's bus attribute with no write-allocate cache policy */
+#define IOMMU_USE_LLC_NWA	(IOMMU_SYS_CACHE_NWA)
+
+/* vendor iommu fault flags */
+#define IOMMU_FAULT_TRANSLATION         (1 << 2)
+#define IOMMU_FAULT_PERMISSION          (1 << 3)
+#define IOMMU_FAULT_EXTERNAL            (1 << 4)
+#define IOMMU_FAULT_TRANSACTION_STALLED (1 << 5)
+
+/* iommu transaction flags */
+#define IOMMU_TRANS_WRITE	BIT(0)	/* 1 Write, 0 Read */
+#define IOMMU_TRANS_PRIV	BIT(1)	/* 1 Privileged, 0 Unprivileged */
+#define IOMMU_TRANS_INST	BIT(2)	/* 1 Instruction fetch, 0 Data access */
+#define IOMMU_TRANS_SEC	BIT(3)	/* 1 Secure, 0 Non-secure access*/
+
+/* Non secure unprivileged Data read operation */
+#define IOMMU_TRANS_DEFAULT	(0U)
+
+struct iommu_pgtbl_info {
+	void *ops;
+};
+
+struct qcom_iommu_atos_txn {
+	u64 addr;
+	u32 flags;
+	u32 id;
+};
+
+enum sid_switch_direction {
+	SID_ACQUIRE,
+	SID_RELEASE,
+};
+
+struct qcom_iommu_fault_ids {
+	u32 bid;
+	u32 pid;
+	u32 mid;
+};
+
+/*
+ * @sid_switch: add/remove all SIDS in the iommu domain containing dev from
+ *              iommu registers.
+ */
+struct qcom_iommu_ops {
+	phys_addr_t (*iova_to_phys_hard)(struct iommu_domain *domain,
+					struct qcom_iommu_atos_txn *txn);
+	int (*sid_switch)(struct device *dev, enum sid_switch_direction dir);
+	int (*get_fault_ids)(struct iommu_domain *domain,
+			struct qcom_iommu_fault_ids *ids);
+	int (*get_context_bank_nr)(struct iommu_domain *domain);
+	int (*set_secure_vmid)(struct iommu_domain *domain, enum vmid vmid);
+	int (*set_fault_model)(struct iommu_domain *domain, int fault_model);
+	int (*enable_s1_translation)(struct iommu_domain *domain);
+	int (*get_mappings_configuration)(struct iommu_domain *domain);
+	struct iommu_ops iommu_ops;
+};
+#define to_qcom_iommu_ops(x) (container_of(x, struct qcom_iommu_ops, iommu_ops))
+
+struct device_node *qcom_iommu_group_parse_phandle(struct device *dev);
+int qcom_iommu_generate_dma_regions(struct device *dev,
+				    struct list_head *head);
+void qcom_iommu_generate_resv_regions(struct device *dev,
+				      struct list_head *list);
+int qcom_iommu_get_fast_iova_range(struct device *dev,
+				   dma_addr_t *ret_iova_base,
+				   dma_addr_t *ret_iova_end);
+
+/* Remove once these functions are exported by upstream kernel */
+void qcom_iommu_get_resv_regions(struct device *dev, struct list_head *list);
+void qcom_iommu_put_resv_regions(struct device *dev, struct list_head *list);
+
+phys_addr_t qcom_iommu_iova_to_phys_hard(struct iommu_domain *domain,
+				    struct qcom_iommu_atos_txn *txn);
+
+int qcom_iommu_sid_switch(struct device *dev, enum sid_switch_direction dir);
+
+extern int qcom_iommu_get_fault_ids(struct iommu_domain *domain,
+				struct qcom_iommu_fault_ids *f_ids);
+extern int qcom_iommu_get_msi_size(struct device *dev, u32 *msi_size);
+
+int qcom_iommu_get_context_bank_nr(struct iommu_domain *domain);
+
+int qcom_iommu_set_secure_vmid(struct iommu_domain *domain, enum vmid vmid);
+
+int qcom_iommu_set_fault_model(struct iommu_domain *domain, int fault_model);
+
+int qcom_iommu_enable_s1_translation(struct iommu_domain *domain);
+
+int qcom_iommu_get_mappings_configuration(struct iommu_domain *domain);
+
+#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE
+int __init qcom_arm_lpae_do_selftests(void);
+#else
+static inline int __init qcom_arm_lpae_do_selftests(void)
+{
+	return 0;
+}
+#endif
+#endif /* __QCOM_IOMMU_UTIL_H */