From 9913ccceda618f5a8e99d2cb5d4c8b51d56a22c6 Mon Sep 17 00:00:00 2001 From: Chris Goldsworthy Date: Fri, 22 Apr 2022 13:52:01 -0700 Subject: [PATCH] qcom-iommu-util: Take a snapshot of the qcom-iommu-util module from msm-5.15 Take a snapshot of our qcom-iommu-util module as of commit f843a80d2c45 ("Merge "defconfig: gen3auto: Separated if_changed"") on msm-5.15. Update minor lint warnings with respect to header inclusions and spacing. Change-Id: Ib33f54f081eb029ade172254a6732c49dcf1f73d Signed-off-by: Chris Goldsworthy --- drivers/iommu/Kconfig | 96 ++ drivers/iommu/Makefile | 8 + drivers/iommu/dma-mapping-fast.c | 1101 +++++++++++++++ drivers/iommu/io-pgtable-fast.c | 844 ++++++++++++ drivers/iommu/iommu-logger.c | 172 +++ drivers/iommu/iommu-logger.h | 51 + drivers/iommu/qcom-dma-iommu-generic.c | 814 ++++++++++++ drivers/iommu/qcom-dma-iommu-generic.h | 85 ++ drivers/iommu/qcom-io-pgtable-alloc.c | 314 +++++ drivers/iommu/qcom-io-pgtable-alloc.h | 16 + drivers/iommu/qcom-io-pgtable-arm.c | 1702 ++++++++++++++++++++++++ drivers/iommu/qcom-io-pgtable.h | 84 ++ drivers/iommu/qcom-iommu-debug-user.c | 1373 +++++++++++++++++++ drivers/iommu/qcom-iommu-debug.c | 383 ++++++ drivers/iommu/qcom-iommu-debug.h | 75 ++ drivers/iommu/qcom-iommu-util.c | 529 ++++++++ include/linux/dma-mapping-fast.h | 68 + include/linux/io-pgtable-fast.h | 126 ++ include/linux/qcom-iommu-util.h | 143 ++ 19 files changed, 7984 insertions(+) create mode 100644 drivers/iommu/dma-mapping-fast.c create mode 100644 drivers/iommu/io-pgtable-fast.c create mode 100644 drivers/iommu/iommu-logger.c create mode 100644 drivers/iommu/iommu-logger.h create mode 100644 drivers/iommu/qcom-dma-iommu-generic.c create mode 100644 drivers/iommu/qcom-dma-iommu-generic.h create mode 100644 drivers/iommu/qcom-io-pgtable-alloc.c create mode 100644 drivers/iommu/qcom-io-pgtable-alloc.h create mode 100644 drivers/iommu/qcom-io-pgtable-arm.c create mode 100644 drivers/iommu/qcom-io-pgtable.h create mode 100644 drivers/iommu/qcom-iommu-debug-user.c create mode 100644 drivers/iommu/qcom-iommu-debug.c create mode 100644 drivers/iommu/qcom-iommu-debug.h create mode 100644 drivers/iommu/qcom-iommu-util.c create mode 100644 include/linux/dma-mapping-fast.h create mode 100644 include/linux/io-pgtable-fast.h create mode 100644 include/linux/qcom-iommu-util.h diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 3eb68fa1b8cc..a5217cc33a18 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -48,6 +48,43 @@ config IOMMU_IO_PGTABLE_LPAE_SELFTEST If unsure, say N here. +config IOMMU_IO_PGTABLE_FAST + bool "Fast ARMv7/v8 Long Descriptor Format" + depends on (ARM || ARM64) && IOMMU_DMA + help + Enable support for a subset of the ARM long descriptor pagetable + format. This allocator achieves fast performance by + pre-allocating and pre-populating page table memory up front. + only supports a 32 bit virtual address space. + + This implementation is mainly optimized for use cases where the + buffers are small (<= 64K) since it only supports 4K page sizes. + +config IOMMU_IO_PGTABLE_FAST_SELFTEST + bool "Fast IO pgtable selftests" + depends on IOMMU_IO_PGTABLE_FAST + help + Enable self-tests for "fast" page table allocator. + This performs a series of page-table consistency checks + during boot. + + If unsure, say N here. + +config IOMMU_IO_PGTABLE_FAST_PROVE_TLB + bool "Prove correctness of TLB maintenance in the Fast DMA mapper" + depends on IOMMU_IO_PGTABLE_FAST + help + Enables some debug features that help prove correctness of TLB + maintenance routines in the Fast DMA mapper. This option will + slow things down considerably, so should only be used in a debug + configuration. This relies on the ability to set bits in an + invalid page table entry, which is disallowed on some hardware + due to errata. If you're running on such a platform then this + option can only be used with unit tests. It will break real use + cases. + + If unsure, say N here. + config IOMMU_IO_PGTABLE_ARMV7S bool "ARMv7/v8 Short Descriptor Format" select IOMMU_IO_PGTABLE @@ -290,6 +327,14 @@ config SPAPR_TCE_IOMMU Enables bits of IOMMU API required by VFIO. The iommu_ops is not implemented as it is not necessary for VFIO. +config QTI_IOMMU_SUPPORT + tristate "Support for QTI iommu drivers" + help + The QTI GPU device may switch between multiple iommu domains, + depending on usecase. This introduces a need to track all such + domains in a non-driver specific manner. + If in doubt say N. + config APPLE_DART tristate "Apple DART IOMMU Support" depends on ARCH_APPLE || (COMPILE_TEST && !GENERIC_ATOMIC64) @@ -318,6 +363,15 @@ config ARM_SMMU Say Y here if your SoC includes an IOMMU device implementing the ARM SMMU architecture. +config ARM_SMMU_CONTEXT_FAULT_RETRY + bool "Context fault retry sequence" + depends on ARM_SMMU && ARCH_WAIPIO + help + In some cases, issuing a tlb invalidate operation after a + context fault may cause a subsequent retry of the failing + address to succeed. Only applies to clients which have + stall-on-fault enabled, like display. + config ARM_SMMU_LEGACY_DT_BINDINGS bool "Support the legacy \"mmu-masters\" devicetree bindings" depends on ARM_SMMU=y && OF @@ -355,6 +409,16 @@ config ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT 'arm-smmu.disable_bypass' will continue to override this config. +config ARM_SMMU_SELFTEST + bool "ARM SMMU self test support" + depends on ARM_SMMU + help + Enables self tests for arm smmu. Tests basic hardware + configurations like interrupts. Note that enabling this + option can marginally increase the boot time. + + If unsure, say N here. + config ARM_SMMU_QCOM def_tristate y depends on ARM_SMMU && ARCH_QCOM @@ -388,6 +452,18 @@ config ARM_SMMU_V3_SVA Say Y here if your system supports SVA extensions such as PCIe PASID and PRI. +config QCOM_LAZY_MAPPING + tristate "Reference counted iommu-mapping support" + depends on QCOM_DMABUF_HEAPS + depends on IOMMU_API + help + DMA-BUFs may be shared between several software clients. + Reference counting the mapping may simplify coordination between + these clients, and decrease latency by preventing multiple + map/unmaps of the same region. + + If unsure, say N here. + config S390_IOMMU def_bool y if S390 && PCI depends on S390 && PCI @@ -474,6 +550,16 @@ config VIRTIO_IOMMU Say Y here if you intend to run this kernel as a guest. +config QCOM_IOMMU_UTIL + tristate "Support for qcom additions to the iommu framework" + help + QCOM iommu drivers support a general set of functionality in + addition to the functions provided by the iommu framework. + This includes devicetree properties for configuring iommu + groups and iova ranges. + + Say N here if unsure. + config SPRD_IOMMU tristate "Unisoc IOMMU Support" depends on ARCH_SPRD || COMPILE_TEST @@ -486,4 +572,14 @@ config SPRD_IOMMU Say Y here if you want to use the multimedia devices listed above. +config QCOM_IOMMU_DEBUG + tristate "IOMMU debugging and testing" + depends on QCOM_IOMMU_UTIL + depends on DEBUG_FS + help + This option is used to enable profiling and debugging in + the IOMMU framework code. IOMMU profiling and debugging + can be done through the debugfs nodes which this option + makes available. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index bc7f730edbb0..295d77762568 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o obj-$(CONFIG_IOMMU_DEBUGFS) += iommu-debugfs.o obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o +obj-$(CONFIG_QCOM_LAZY_MAPPING) += msm_dma_iommu_mapping.o obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o @@ -29,4 +30,11 @@ obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o obj-$(CONFIG_IOMMU_SVA_LIB) += iommu-sva-lib.o io-pgfault.o obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o +obj-$(CONFIG_QCOM_IOMMU_UTIL) += qcom_iommu_util.o +qcom_iommu_util-y += qcom-iommu-util.o +qcom_iommu_util-$(CONFIG_IOMMU_IO_PGTABLE_FAST) += qcom-dma-iommu-generic.o io-pgtable-fast.o dma-mapping-fast.o +qcom_iommu_util-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += qcom-io-pgtable-arm.o qcom-io-pgtable-alloc.o +obj-$(CONFIG_QTI_IOMMU_SUPPORT) += iommu-logger.o +obj-$(CONFIG_QCOM_IOMMU_DEBUG) += qcom_iommu_debug.o +qcom_iommu_debug-y += qcom-iommu-debug.o qcom-iommu-debug-user.o obj-$(CONFIG_APPLE_DART) += apple-dart.o diff --git a/drivers/iommu/dma-mapping-fast.c b/drivers/iommu/dma-mapping-fast.c new file mode 100644 index 000000000000..aa777b60322c --- /dev/null +++ b/drivers/iommu/dma-mapping-fast.c @@ -0,0 +1,1101 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "qcom-dma-iommu-generic.h" + +/* some redundant definitions... :( TODO: move to io-pgtable-fast.h */ +#define FAST_PAGE_SHIFT 12 +#define FAST_PAGE_SIZE (1UL << FAST_PAGE_SHIFT) +#define FAST_PAGE_MASK (~(PAGE_SIZE - 1)) + +static struct rb_root mappings; +static DEFINE_RWLOCK(mappings_lock); + +static int fast_smmu_add_mapping(struct dma_fast_smmu_mapping *fast) +{ + struct rb_node **new = &mappings.rb_node, *parent = NULL; + struct dma_fast_smmu_mapping *entry; + int ret = 0; + unsigned long flags; + + write_lock_irqsave(&mappings_lock, flags); + while (*new) { + entry = rb_entry(*new, struct dma_fast_smmu_mapping, node); + + parent = *new; + if (fast->domain < entry->domain) { + new = &((*new)->rb_left); + } else if (fast->domain > entry->domain) { + new = &((*new)->rb_right); + } else { + ret = -EEXIST; + break; + } + } + + if (!ret) { + rb_link_node(&fast->node, parent, new); + rb_insert_color(&fast->node, &mappings); + } + write_unlock_irqrestore(&mappings_lock, flags); + + return ret; +} + +static struct dma_fast_smmu_mapping *__fast_smmu_lookup_mapping(struct iommu_domain *domain) +{ + struct rb_node *node = mappings.rb_node; + struct dma_fast_smmu_mapping *entry; + + while (node) { + entry = rb_entry(node, struct dma_fast_smmu_mapping, node); + + if (domain < entry->domain) + node = node->rb_left; + else if (domain > entry->domain) + node = node->rb_right; + else + return entry; + } + + return NULL; +} + +static struct dma_fast_smmu_mapping *fast_smmu_lookup_mapping(struct iommu_domain *domain) +{ + struct dma_fast_smmu_mapping *fast; + unsigned long flags; + + read_lock_irqsave(&mappings_lock, flags); + fast = __fast_smmu_lookup_mapping(domain); + read_unlock_irqrestore(&mappings_lock, flags); + return fast; +} + +static struct dma_fast_smmu_mapping *fast_smmu_remove_mapping(struct iommu_domain *domain) +{ + struct dma_fast_smmu_mapping *fast; + unsigned long flags; + + write_lock_irqsave(&mappings_lock, flags); + fast = __fast_smmu_lookup_mapping(domain); + if (fast) + rb_erase(&fast->node, &mappings); + write_unlock_irqrestore(&mappings_lock, flags); + return fast; +} + +static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot, + bool coherent) +{ + if (!coherent || (attrs & DMA_ATTR_WRITE_COMBINE)) + return pgprot_writecombine(prot); + return prot; +} + +static bool is_dma_coherent(struct device *dev, unsigned long attrs) +{ + bool is_coherent; + + if (attrs & DMA_ATTR_FORCE_COHERENT) + is_coherent = true; + else if (attrs & DMA_ATTR_FORCE_NON_COHERENT) + is_coherent = false; + else if (dev_is_dma_coherent(dev)) + is_coherent = true; + else + is_coherent = false; + + return is_coherent; +} + +static struct dma_fast_smmu_mapping *dev_get_mapping(struct device *dev) +{ + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return ERR_PTR(-EINVAL); + return fast_smmu_lookup_mapping(domain); +} + +static dma_addr_t __fast_smmu_alloc_iova(struct dma_fast_smmu_mapping *mapping, + unsigned long attrs, + size_t size) +{ + unsigned long bit, nbits = size >> FAST_PAGE_SHIFT; + unsigned long align = (1 << get_order(size)) - 1; + struct iommu_domain *domain = mapping->domain; + + bit = bitmap_find_next_zero_area(mapping->clean_bitmap, + mapping->num_4k_pages, + mapping->next_start, nbits, align); + if (unlikely(bit > mapping->num_4k_pages)) { + /* try wrapping */ + bit = bitmap_find_next_zero_area( + mapping->clean_bitmap, mapping->num_4k_pages, 0, nbits, + align); + if (unlikely(bit > mapping->num_4k_pages)) { + /* + * If we just re-allocated a VA whose TLB hasn't been + * invalidated since it was last used and unmapped, we + * need to invalidate it here. We actually invalidate + * the entire TLB so that we don't have to invalidate + * the TLB again until we wrap back around. + */ + if (mapping->have_stale_tlbs) { + bool skip_sync = (attrs & + DMA_ATTR_SKIP_CPU_SYNC); + + iommu_flush_iotlb_all(domain); + bitmap_copy(mapping->clean_bitmap, + mapping->bitmap, + mapping->num_4k_pages); + mapping->have_stale_tlbs = false; + av8l_fast_clear_stale_ptes(mapping->pgtbl_ops, + mapping->base, + mapping->base + + mapping->size - 1, + skip_sync); + bit = bitmap_find_next_zero_area( + mapping->clean_bitmap, + mapping->num_4k_pages, + 0, nbits, + align); + if (unlikely(bit > mapping->num_4k_pages)) + return DMA_MAPPING_ERROR; + + } else { + return DMA_MAPPING_ERROR; + } + } + } + + bitmap_set(mapping->bitmap, bit, nbits); + bitmap_set(mapping->clean_bitmap, bit, nbits); + mapping->next_start = bit + nbits; + if (unlikely(mapping->next_start >= mapping->num_4k_pages)) + mapping->next_start = 0; + + return (bit << FAST_PAGE_SHIFT) + mapping->base; +} + +static void __fast_smmu_free_iova(struct dma_fast_smmu_mapping *mapping, + dma_addr_t iova, size_t size) +{ + unsigned long start_bit = (iova - mapping->base) >> FAST_PAGE_SHIFT; + unsigned long nbits = size >> FAST_PAGE_SHIFT; + + /* + * We don't invalidate TLBs on unmap. We invalidate TLBs on map + * when we're about to re-allocate a VA that was previously + * unmapped but hasn't yet been invalidated. + */ + bitmap_clear(mapping->bitmap, start_bit, nbits); + mapping->have_stale_tlbs = true; +} + +static dma_addr_t fast_smmu_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + dma_addr_t iova; + unsigned long flags; + phys_addr_t phys_plus_off = page_to_phys(page) + offset; + phys_addr_t phys_to_map = round_down(phys_plus_off, FAST_PAGE_SIZE); + unsigned long offset_from_phys_to_map = phys_plus_off & ~FAST_PAGE_MASK; + size_t len = ALIGN(size + offset_from_phys_to_map, FAST_PAGE_SIZE); + bool skip_sync = (attrs & DMA_ATTR_SKIP_CPU_SYNC); + bool is_coherent = is_dma_coherent(dev, attrs); + int prot = qcom_dma_info_to_prot(dir, is_coherent, attrs); + + if (!skip_sync && !is_coherent) + qcom_arch_sync_dma_for_device(phys_plus_off, size, dir); + + spin_lock_irqsave(&mapping->lock, flags); + + iova = __fast_smmu_alloc_iova(mapping, attrs, len); + + if (unlikely(iova == DMA_MAPPING_ERROR)) + goto fail; + + if (unlikely(av8l_fast_map_public(mapping->pgtbl_ops, iova, + phys_to_map, len, prot))) + goto fail_free_iova; + + spin_unlock_irqrestore(&mapping->lock, flags); + + return iova + offset_from_phys_to_map; + +fail_free_iova: + __fast_smmu_free_iova(mapping, iova, size); +fail: + spin_unlock_irqrestore(&mapping->lock, flags); + return DMA_MAPPING_ERROR; +} + +static void fast_smmu_unmap_page(struct device *dev, dma_addr_t iova, + size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + unsigned long flags; + unsigned long offset = iova & ~FAST_PAGE_MASK; + size_t len = ALIGN(size + offset, FAST_PAGE_SIZE); + bool skip_sync = (attrs & DMA_ATTR_SKIP_CPU_SYNC); + bool is_coherent = is_dma_coherent(dev, attrs); + + if (!skip_sync && !is_coherent) { + phys_addr_t phys; + + phys = av8l_fast_iova_to_phys_public(mapping->pgtbl_ops, iova); + WARN_ON(!phys); + + qcom_arch_sync_dma_for_cpu(phys, size, dir); + } + + spin_lock_irqsave(&mapping->lock, flags); + av8l_fast_unmap_public(mapping->pgtbl_ops, iova, len); + __fast_smmu_free_iova(mapping, iova, len); + spin_unlock_irqrestore(&mapping->lock, flags); +} + +static void fast_smmu_sync_single_for_cpu(struct device *dev, + dma_addr_t iova, size_t size, enum dma_data_direction dir) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + + if (!av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova)) { + phys_addr_t phys; + + phys = av8l_fast_iova_to_phys_public(mapping->pgtbl_ops, iova); + WARN_ON(!phys); + + qcom_arch_sync_dma_for_cpu(phys, size, dir); + } +} + +static void fast_smmu_sync_single_for_device(struct device *dev, + dma_addr_t iova, size_t size, enum dma_data_direction dir) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + + if (!av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova)) { + phys_addr_t phys; + + phys = av8l_fast_iova_to_phys_public(mapping->pgtbl_ops, iova); + WARN_ON(!phys); + + qcom_arch_sync_dma_for_device(phys, size, dir); + } +} + +static void fast_smmu_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sgl, int nelems, + enum dma_data_direction dir) +{ + struct scatterlist *sg; + dma_addr_t iova = sg_dma_address(sgl); + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + int i; + + if (av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova)) + return; + + for_each_sg(sgl, sg, nelems, i) + qcom_arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); +} + +static void fast_smmu_sync_sg_for_device(struct device *dev, + struct scatterlist *sgl, int nelems, + enum dma_data_direction dir) +{ + struct scatterlist *sg; + dma_addr_t iova = sg_dma_address(sgl); + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + int i; + + if (av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova)) + return; + + for_each_sg(sgl, sg, nelems, i) + qcom_arch_sync_dma_for_device(sg_phys(sg), sg->length, dir); +} + +static int fast_smmu_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + size_t iova_len; + bool is_coherent = is_dma_coherent(dev, attrs); + int prot = qcom_dma_info_to_prot(dir, is_coherent, attrs); + int ret; + dma_addr_t iova; + unsigned long flags; + size_t unused = 0; + + iova_len = qcom_iommu_dma_prepare_map_sg(dev, mapping->iovad, sg, nents); + + spin_lock_irqsave(&mapping->lock, flags); + iova = __fast_smmu_alloc_iova(mapping, attrs, iova_len); + spin_unlock_irqrestore(&mapping->lock, flags); + + if (unlikely(iova == DMA_MAPPING_ERROR)) + goto fail; + + av8l_fast_map_sg_public(mapping->pgtbl_ops, iova, sg, nents, prot, + &unused); + + ret = qcom_iommu_dma_finalise_sg(dev, sg, nents, iova); + + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + fast_smmu_sync_sg_for_device(dev, sg, nents, dir); + + return ret; +fail: + qcom_iommu_dma_invalidate_sg(sg, nents); + return 0; +} + +static void fast_smmu_unmap_sg(struct device *dev, + struct scatterlist *sg, int nelems, + enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + unsigned long flags; + dma_addr_t start; + size_t len, offset; + struct scatterlist *tmp; + int i; + + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + fast_smmu_sync_sg_for_cpu(dev, sg, nelems, dir); + + /* + * The scatterlist segments are mapped into a single + * contiguous IOVA allocation, so this is incredibly easy. + */ + start = sg_dma_address(sg); + offset = start & ~FAST_PAGE_MASK; + for_each_sg(sg_next(sg), tmp, nelems - 1, i) { + if (sg_dma_len(tmp) == 0) + break; + sg = tmp; + } + len = ALIGN(sg_dma_address(sg) + sg_dma_len(sg) - (start - offset), + FAST_PAGE_SIZE); + + av8l_fast_unmap_public(mapping->pgtbl_ops, start, len); + + spin_lock_irqsave(&mapping->lock, flags); + __fast_smmu_free_iova(mapping, start, len); + spin_unlock_irqrestore(&mapping->lock, flags); +} + +static void __fast_smmu_free_pages(struct page **pages, int count) +{ + int i; + + if (!pages) + return; + for (i = 0; i < count; i++) + __free_page(pages[i]); + kvfree(pages); +} + +static void *fast_smmu_alloc_atomic(struct dma_fast_smmu_mapping *mapping, + size_t size, gfp_t gfp, unsigned long attrs, + dma_addr_t *handle, bool coherent) +{ + void *addr; + unsigned long flags; + struct page *page; + dma_addr_t dma_addr; + int prot = qcom_dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs); + + if (coherent) { + page = alloc_pages(gfp, get_order(size)); + addr = page ? page_address(page) : NULL; + } else + addr = qcom_dma_alloc_from_pool(mapping->dev, size, &page, gfp); + if (!addr) + return NULL; + + spin_lock_irqsave(&mapping->lock, flags); + dma_addr = __fast_smmu_alloc_iova(mapping, attrs, size); + if (dma_addr == DMA_MAPPING_ERROR) { + dev_err(mapping->dev, "no iova\n"); + spin_unlock_irqrestore(&mapping->lock, flags); + goto out_free_page; + } + if (unlikely(av8l_fast_map_public(mapping->pgtbl_ops, dma_addr, + page_to_phys(page), size, prot))) { + dev_err(mapping->dev, "no map public\n"); + goto out_free_iova; + } + spin_unlock_irqrestore(&mapping->lock, flags); + *handle = dma_addr; + return addr; + +out_free_iova: + __fast_smmu_free_iova(mapping, dma_addr, size); + spin_unlock_irqrestore(&mapping->lock, flags); +out_free_page: + if (coherent) + __free_pages(page, get_order(size)); + else + qcom_dma_free_from_pool(mapping->dev, addr, size); + return NULL; +} + +static struct page **__fast_smmu_alloc_pages(unsigned int count, gfp_t gfp) +{ + struct page **pages; + unsigned int i = 0, array_size = count * sizeof(*pages); + + if (array_size <= PAGE_SIZE) + pages = kzalloc(array_size, GFP_KERNEL); + else + pages = vzalloc(array_size); + if (!pages) + return NULL; + + /* IOMMU can map any pages, so himem can also be used here */ + gfp |= __GFP_NOWARN | __GFP_HIGHMEM; + + for (i = 0; i < count; ++i) { + struct page *page = alloc_page(gfp); + + if (!page) { + __fast_smmu_free_pages(pages, i); + return NULL; + } + pages[i] = page; + } + return pages; +} + +static void *__fast_smmu_alloc_contiguous(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + bool is_coherent = is_dma_coherent(dev, attrs); + int prot = qcom_dma_info_to_prot(DMA_BIDIRECTIONAL, is_coherent, attrs); + pgprot_t remap_prot = __get_dma_pgprot(attrs, PAGE_KERNEL, is_coherent); + struct page *page; + dma_addr_t iova; + unsigned long flags; + void *coherent_addr; + + page = qcom_dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, + get_order(size), gfp & __GFP_NOWARN); + if (!page) + return NULL; + + + spin_lock_irqsave(&mapping->lock, flags); + iova = __fast_smmu_alloc_iova(mapping, attrs, size); + spin_unlock_irqrestore(&mapping->lock, flags); + if (iova == DMA_MAPPING_ERROR) + goto release_page; + + if (av8l_fast_map_public(mapping->pgtbl_ops, iova, page_to_phys(page), + size, prot)) + goto release_iova; + + if (!is_coherent || PageHighMem(page)) { + coherent_addr = qcom_dma_common_contiguous_remap(page, size, + remap_prot, + __fast_smmu_alloc_contiguous); + if (!coherent_addr) + goto release_mapping; + + if (!is_coherent) + qcom_arch_dma_prep_coherent(page, size); + } else { + coherent_addr = page_address(page); + } + + memset(coherent_addr, 0, size); + *handle = iova; + return coherent_addr; + +release_mapping: + av8l_fast_unmap_public(mapping->pgtbl_ops, iova, size); +release_iova: + __fast_smmu_free_iova(mapping, iova, size); +release_page: + qcom_dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); + return NULL; +} + +static void *fast_smmu_alloc(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + struct sg_table sgt; + dma_addr_t dma_addr, iova_iter; + void *addr; + unsigned long flags; + struct sg_mapping_iter miter; + size_t count = ALIGN(size, SZ_4K) >> PAGE_SHIFT; + bool is_coherent = is_dma_coherent(dev, attrs); + int prot = qcom_dma_info_to_prot(DMA_BIDIRECTIONAL, is_coherent, attrs); + pgprot_t remap_prot = __get_dma_pgprot(attrs, PAGE_KERNEL, is_coherent); + struct page **pages; + + /* + * sg_alloc_table_from_pages accepts unsigned int value for count + * so check count doesn't exceed UINT_MAX. + */ + + if (count > UINT_MAX) { + dev_err(dev, "count: %zx exceeds UNIT_MAX\n", count); + return NULL; + } + + gfp |= __GFP_ZERO; + + *handle = DMA_MAPPING_ERROR; + size = ALIGN(size, SZ_4K); + + if (!gfpflags_allow_blocking(gfp)) + return fast_smmu_alloc_atomic(mapping, size, gfp, attrs, handle, + is_coherent); + else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) + return __fast_smmu_alloc_contiguous(dev, size, handle, gfp, + attrs); + + pages = __fast_smmu_alloc_pages(count, gfp); + if (!pages) { + dev_err(dev, "no pages\n"); + return NULL; + } + + if (sg_alloc_table_from_pages(&sgt, pages, count, 0, size, gfp)) { + dev_err(dev, "no sg tablen\n"); + goto out_free_pages; + } + + if (!is_coherent) { + /* + * The CPU-centric flushing implied by SG_MITER_TO_SG isn't + * sufficient here, so skip it by using the "wrong" direction. + */ + sg_miter_start(&miter, sgt.sgl, sgt.orig_nents, + SG_MITER_FROM_SG); + while (sg_miter_next(&miter)) + qcom_arch_dma_prep_coherent(miter.page, miter.length); + sg_miter_stop(&miter); + } + + spin_lock_irqsave(&mapping->lock, flags); + dma_addr = __fast_smmu_alloc_iova(mapping, attrs, size); + if (dma_addr == DMA_MAPPING_ERROR) { + dev_err(dev, "no iova\n"); + spin_unlock_irqrestore(&mapping->lock, flags); + goto out_free_sg; + } + iova_iter = dma_addr; + sg_miter_start(&miter, sgt.sgl, sgt.orig_nents, + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + while (sg_miter_next(&miter)) { + if (unlikely(av8l_fast_map_public( + mapping->pgtbl_ops, iova_iter, + page_to_phys(miter.page), + miter.length, prot))) { + dev_err(dev, "no map public\n"); + /* TODO: unwind previously successful mappings */ + goto out_free_iova; + } + iova_iter += miter.length; + } + sg_miter_stop(&miter); + spin_unlock_irqrestore(&mapping->lock, flags); + + addr = qcom_dma_common_pages_remap(pages, size, remap_prot, + __builtin_return_address(0)); + if (!addr) { + dev_err(dev, "no common pages\n"); + goto out_unmap; + } + + *handle = dma_addr; + sg_free_table(&sgt); + return addr; + +out_unmap: + /* need to take the lock again for page tables and iova */ + spin_lock_irqsave(&mapping->lock, flags); + av8l_fast_unmap_public(mapping->pgtbl_ops, dma_addr, size); +out_free_iova: + __fast_smmu_free_iova(mapping, dma_addr, size); + spin_unlock_irqrestore(&mapping->lock, flags); +out_free_sg: + sg_free_table(&sgt); +out_free_pages: + __fast_smmu_free_pages(pages, count); + return NULL; +} + +static void fast_smmu_free(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t dma_handle, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + struct page **pages = NULL; + struct page *page = NULL; + unsigned long flags; + + size = ALIGN(size, FAST_PAGE_SIZE); + + spin_lock_irqsave(&mapping->lock, flags); + av8l_fast_unmap_public(mapping->pgtbl_ops, dma_handle, size); + __fast_smmu_free_iova(mapping, dma_handle, size); + spin_unlock_irqrestore(&mapping->lock, flags); + + if (qcom_dma_free_from_pool(dev, cpu_addr, size)) + return; + + if (is_vmalloc_addr(cpu_addr)) { + pages = qcom_dma_common_find_pages(cpu_addr); + if (!pages) + page = vmalloc_to_page(cpu_addr); + qcom_dma_common_free_remap(cpu_addr, size); + } else { + page = virt_to_page(cpu_addr); + } + + if (pages) + __fast_smmu_free_pages(pages, size >> FAST_PAGE_SHIFT); + + if (page) + qcom_dma_free_contiguous(dev, page, size); +} + +static int fast_smmu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, + size_t size, unsigned long attrs) +{ + return qcom_iommu_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); +} + +static int fast_smmu_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, + size_t size, unsigned long attrs) +{ + return qcom_iommu_dma_get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs); +} + +static dma_addr_t fast_smmu_dma_map_resource( + struct device *dev, phys_addr_t phys_addr, + size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + size_t offset = phys_addr & ~FAST_PAGE_MASK; + size_t len = round_up(size + offset, FAST_PAGE_SIZE); + dma_addr_t dma_addr; + int prot; + unsigned long flags; + + spin_lock_irqsave(&mapping->lock, flags); + dma_addr = __fast_smmu_alloc_iova(mapping, attrs, len); + spin_unlock_irqrestore(&mapping->lock, flags); + + if (dma_addr == DMA_MAPPING_ERROR) + return dma_addr; + + prot = qcom_dma_info_to_prot(dir, false, attrs); + prot |= IOMMU_MMIO; + + if (iommu_map(mapping->domain, dma_addr, phys_addr - offset, + len, prot)) { + spin_lock_irqsave(&mapping->lock, flags); + __fast_smmu_free_iova(mapping, dma_addr, len); + spin_unlock_irqrestore(&mapping->lock, flags); + return DMA_MAPPING_ERROR; + } + return dma_addr + offset; +} + +static void fast_smmu_dma_unmap_resource( + struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + size_t offset = addr & ~FAST_PAGE_MASK; + size_t len = round_up(size + offset, FAST_PAGE_SIZE); + unsigned long flags; + + iommu_unmap(mapping->domain, addr - offset, len); + spin_lock_irqsave(&mapping->lock, flags); + __fast_smmu_free_iova(mapping, addr, len); + spin_unlock_irqrestore(&mapping->lock, flags); +} + +static void __fast_smmu_mapped_over_stale(struct dma_fast_smmu_mapping *fast, + void *priv) +{ + av8l_fast_iopte *pmds, *ptep = priv; + dma_addr_t iova; + unsigned long bitmap_idx; + struct av8l_fast_io_pgtable *data; + + data = iof_pgtable_ops_to_data(fast->pgtbl_ops); + pmds = data->pmds; + + bitmap_idx = (unsigned long)(ptep - pmds); + iova = bitmap_idx << FAST_PAGE_SHIFT; + dev_err(fast->dev, "Mapped over stale tlb at %pa\n", &iova); + dev_err(fast->dev, "bitmap (failure at idx %lu):\n", bitmap_idx); + dev_err(fast->dev, "ptep: %pK pmds: %pK diff: %lu\n", ptep, + pmds, bitmap_idx); + print_hex_dump(KERN_ERR, "bmap: ", DUMP_PREFIX_ADDRESS, + 32, 8, fast->bitmap, fast->bitmap_size, false); +} + +static int fast_smmu_notify(struct notifier_block *self, + unsigned long action, void *data) +{ + struct dma_fast_smmu_mapping *fast = container_of( + self, struct dma_fast_smmu_mapping, notifier); + + switch (action) { + case MAPPED_OVER_STALE_TLB: + __fast_smmu_mapped_over_stale(fast, data); + return NOTIFY_OK; + default: + WARN(1, "Unhandled notifier action"); + return NOTIFY_DONE; + } +} + +static const struct dma_map_ops fast_smmu_dma_ops = { + .alloc = fast_smmu_alloc, + .free = fast_smmu_free, + .mmap = fast_smmu_mmap_attrs, + .get_sgtable = fast_smmu_get_sgtable, + .map_page = fast_smmu_map_page, + .unmap_page = fast_smmu_unmap_page, + .sync_single_for_cpu = fast_smmu_sync_single_for_cpu, + .sync_single_for_device = fast_smmu_sync_single_for_device, + .map_sg = fast_smmu_map_sg, + .unmap_sg = fast_smmu_unmap_sg, + .sync_sg_for_cpu = fast_smmu_sync_sg_for_cpu, + .sync_sg_for_device = fast_smmu_sync_sg_for_device, + .map_resource = fast_smmu_dma_map_resource, + .unmap_resource = fast_smmu_dma_unmap_resource, +}; + +/** + * __fast_smmu_create_mapping_sized + * @base: bottom of the VA range + * @size: size of the VA range in bytes + * + * Creates a mapping structure which holds information about used/unused IO + * address ranges, which is required to perform mapping with IOMMU aware + * functions. The only VA range supported is [0, 4GB). + * + * The client device need to be attached to the mapping with + * fast_smmu_attach_device function. + */ +static struct dma_fast_smmu_mapping *__fast_smmu_create_mapping_sized( + dma_addr_t base, u64 size) +{ + struct dma_fast_smmu_mapping *fast; + + fast = kzalloc(sizeof(struct dma_fast_smmu_mapping), GFP_KERNEL); + if (!fast) + goto err; + + fast->base = base; + fast->size = size; + fast->num_4k_pages = size >> FAST_PAGE_SHIFT; + fast->bitmap_size = BITS_TO_LONGS(fast->num_4k_pages) * sizeof(long); + + fast->bitmap = kzalloc(fast->bitmap_size, GFP_KERNEL | __GFP_NOWARN | + __GFP_NORETRY); + if (!fast->bitmap) + fast->bitmap = vzalloc(fast->bitmap_size); + + if (!fast->bitmap) + goto err2; + + fast->clean_bitmap = kzalloc(fast->bitmap_size, GFP_KERNEL | + __GFP_NOWARN | __GFP_NORETRY); + if (!fast->clean_bitmap) + fast->clean_bitmap = vzalloc(fast->bitmap_size); + + if (!fast->clean_bitmap) + goto err3; + + spin_lock_init(&fast->lock); + + fast->iovad = kzalloc(sizeof(*fast->iovad), GFP_KERNEL); + if (!fast->iovad) + goto err_free_bitmap; + init_iova_domain(fast->iovad, FAST_PAGE_SIZE, + base >> FAST_PAGE_SHIFT); + + return fast; + +err_free_bitmap: + kvfree(fast->clean_bitmap); +err3: + kvfree(fast->bitmap); +err2: + kfree(fast); +err: + return ERR_PTR(-ENOMEM); +} + +/* + * Based off of similar code from dma-iommu.c, but modified to use a different + * iova allocator + */ +static void fast_smmu_reserve_pci_windows(struct device *dev, + struct dma_fast_smmu_mapping *mapping) +{ + struct pci_host_bridge *bridge; + struct resource_entry *window; + phys_addr_t start, end; + struct pci_dev *pci_dev; + unsigned long flags; + + if (!dev_is_pci(dev)) + return; + + pci_dev = to_pci_dev(dev); + bridge = qcom_pci_find_host_bridge(pci_dev->bus); + + spin_lock_irqsave(&mapping->lock, flags); + resource_list_for_each_entry(window, &bridge->windows) { + if (resource_type(window->res) != IORESOURCE_MEM && + resource_type(window->res) != IORESOURCE_IO) + continue; + + start = round_down(window->res->start - window->offset, + FAST_PAGE_SIZE); + end = round_up(window->res->end - window->offset, + FAST_PAGE_SIZE); + start = max_t(unsigned long, mapping->base, start); + end = min_t(unsigned long, mapping->base + mapping->size, end); + if (start >= end) + continue; + + dev_dbg(dev, "iova allocator reserved 0x%pa-0x%pa\n", + &start, &end); + + start = (start - mapping->base) >> FAST_PAGE_SHIFT; + end = (end - mapping->base) >> FAST_PAGE_SHIFT; + bitmap_set(mapping->bitmap, start, end - start); + bitmap_set(mapping->clean_bitmap, start, end - start); + } + spin_unlock_irqrestore(&mapping->lock, flags); +} + +static void fast_smmu_reserve_msi_iova(struct device *dev, struct dma_fast_smmu_mapping *fast) +{ + dma_addr_t msi_iova_base; + u32 msi_size; + int ret; + unsigned long flags; + + spin_lock_irqsave(&fast->lock, flags); + + /* MSI cookie has already been setup. */ + if (fast->domain->iova_cookie) + goto out; + + if (qcom_iommu_get_msi_size(dev, &msi_size) < 0) + goto out; + + msi_iova_base = __fast_smmu_alloc_iova(fast, 0, msi_size); + if (msi_iova_base == DMA_MAPPING_ERROR) { + dev_err(dev, "iova allocator failed to reserve MSI range of size: 0x%x\n", + msi_size); + goto out; + } + dev_dbg(dev, "iova allocator reserved 0x%lx-0x%lx for MSI\n", msi_iova_base, + msi_iova_base + msi_size); + + ret = iommu_get_msi_cookie(fast->domain, msi_iova_base); + if (ret < 0) { + dev_err(dev, "failed to obtain MSI iova cookie rc: %d\n", ret); + __fast_smmu_free_iova(fast, msi_iova_base, msi_size); + } + +out: + spin_unlock_irqrestore(&fast->lock, flags); +} + +static void fast_smmu_reserve_iommu_regions(struct device *dev, + struct dma_fast_smmu_mapping *fast) +{ + struct iommu_resv_region *region; + unsigned long flags; + struct dma_fast_smmu_mapping *mapping = dev_get_mapping(dev); + LIST_HEAD(resv_regions); + + if (dev_is_pci(dev)) + fast_smmu_reserve_pci_windows(dev, fast); + + qcom_iommu_get_resv_regions(dev, &resv_regions); + spin_lock_irqsave(&mapping->lock, flags); + list_for_each_entry(region, &resv_regions, list) { + unsigned long lo, hi; + + lo = max(fast->base, region->start); + hi = min(fast->base + fast->size - 1, + region->start + region->length - 1); + + lo = (lo - fast->base) >> FAST_PAGE_SHIFT; + hi = (hi - fast->base) >> FAST_PAGE_SHIFT; + bitmap_set(fast->bitmap, lo, hi - lo + 1); + bitmap_set(fast->clean_bitmap, lo, hi - lo + 1); + } + spin_unlock_irqrestore(&mapping->lock, flags); + qcom_iommu_put_resv_regions(dev, &resv_regions); + + fast_smmu_reserve_msi_iova(dev, fast); +} + +void fast_smmu_put_dma_cookie(struct iommu_domain *domain) +{ + struct dma_fast_smmu_mapping *fast = fast_smmu_remove_mapping(domain); + + if (!fast) + return; + + iommu_put_dma_cookie(domain); + + if (fast->iovad) { + put_iova_domain(fast->iovad); + kfree(fast->iovad); + } + + if (fast->bitmap) + kvfree(fast->bitmap); + + if (fast->clean_bitmap) + kvfree(fast->clean_bitmap); + + kfree(fast); +} +EXPORT_SYMBOL(fast_smmu_put_dma_cookie); + +/** + * fast_smmu_init_mapping + * @dev: valid struct device pointer + * @domain: valid IOMMU domain pointer + * @pgtable_ops: The page table ops associated with this domain + * + * Called the first time a device is attached to this mapping. + * Not for dma client use. + */ +int fast_smmu_init_mapping(struct device *dev, struct iommu_domain *domain, + struct io_pgtable_ops *pgtable_ops) +{ + u64 dma_base, dma_end, size; + struct dma_fast_smmu_mapping *fast = fast_smmu_lookup_mapping(domain); + + if (fast) { + dev_err(dev, "Iova cookie already present\n"); + return -EINVAL; + } + + if (!pgtable_ops) + return -EINVAL; + + dma_base = max_t(u64, domain->geometry.aperture_start, 0); + dma_end = min_t(u64, domain->geometry.aperture_end, + (SZ_1G * 4ULL - 1)); + size = dma_end - dma_base + 1; + if (dma_base >= dma_end) { + dev_err(dev, "Invalid domain geometry\n"); + return -EINVAL; + } + + fast = __fast_smmu_create_mapping_sized(dma_base, size); + if (IS_ERR(fast)) + return -ENOMEM; + + fast->domain = domain; + fast->dev = dev; + fast_smmu_add_mapping(fast); + + fast->pgtbl_ops = pgtable_ops; + + fast->notifier.notifier_call = fast_smmu_notify; + av8l_register_notify(&fast->notifier); + + return 0; +} +EXPORT_SYMBOL(fast_smmu_init_mapping); + +static void __fast_smmu_setup_dma_ops(void *data, struct device *dev, + u64 dma_base, u64 dma_limit) +{ + struct dma_fast_smmu_mapping *fast; + struct iommu_domain *domain; + int ret; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return; + + ret = qcom_iommu_get_mappings_configuration(domain); + if (ret < 0 || !(ret & QCOM_IOMMU_MAPPING_CONF_FAST)) + return; + + fast = dev_get_mapping(dev); + if (!fast) { + dev_err(dev, "Missing fastmap iova cookie\n"); + return; + } + + fast_smmu_reserve_iommu_regions(dev, fast); + dev->dma_ops = &fast_smmu_dma_ops; +} + +/* + * Called by drivers who create their own iommu domains via + * iommu_domain_alloc(). + */ +void fast_smmu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) +{ + __fast_smmu_setup_dma_ops(NULL, dev, dma_base, dma_limit); +} +EXPORT_SYMBOL(fast_smmu_setup_dma_ops); + +int __init dma_mapping_fast_init(void) +{ + return register_trace_android_rvh_iommu_setup_dma_ops( + __fast_smmu_setup_dma_ops, NULL); +} + diff --git a/drivers/iommu/io-pgtable-fast.c b/drivers/iommu/io-pgtable-fast.c new file mode 100644 index 000000000000..92277f965c97 --- /dev/null +++ b/drivers/iommu/io-pgtable-fast.c @@ -0,0 +1,844 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "io-pgtable-fast: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "qcom-io-pgtable.h" + +#define AV8L_FAST_MAX_ADDR_BITS 48 + +/* Page table bits */ +#define AV8L_FAST_PTE_TYPE_SHIFT 0 +#define AV8L_FAST_PTE_TYPE_MASK 0x3 + +#define AV8L_FAST_PTE_TYPE_BLOCK 1 +#define AV8L_FAST_PTE_TYPE_TABLE 3 +#define AV8L_FAST_PTE_TYPE_PAGE 3 + +#define AV8L_FAST_PTE_NSTABLE (((av8l_fast_iopte)1) << 63) +#define AV8L_FAST_PTE_XN (((av8l_fast_iopte)3) << 53) +#define AV8L_FAST_PTE_AF (((av8l_fast_iopte)1) << 10) +#define AV8L_FAST_PTE_SH_NS (((av8l_fast_iopte)0) << 8) +#define AV8L_FAST_PTE_SH_OS (((av8l_fast_iopte)2) << 8) +#define AV8L_FAST_PTE_SH_IS (((av8l_fast_iopte)3) << 8) +#define AV8L_FAST_PTE_SH_MASK (((av8l_fast_iopte)3) << 8) +#define AV8L_FAST_PTE_NS (((av8l_fast_iopte)1) << 5) +#define AV8L_FAST_PTE_VALID (((av8l_fast_iopte)1) << 0) + +#define AV8L_FAST_PTE_ATTR_LO_MASK (((av8l_fast_iopte)0x3ff) << 2) +/* Ignore the contiguous bit for block splitting */ +#define AV8L_FAST_PTE_ATTR_HI_MASK (((av8l_fast_iopte)6) << 52) +#define AV8L_FAST_PTE_ATTR_MASK (AV8L_FAST_PTE_ATTR_LO_MASK | \ + AV8L_FAST_PTE_ATTR_HI_MASK) +#define AV8L_FAST_PTE_ADDR_MASK ((av8l_fast_iopte)0xfffffffff000) + + +/* Stage-1 PTE */ +#define AV8L_FAST_PTE_AP_PRIV_RW (((av8l_fast_iopte)0) << 6) +#define AV8L_FAST_PTE_AP_RW (((av8l_fast_iopte)1) << 6) +#define AV8L_FAST_PTE_AP_PRIV_RO (((av8l_fast_iopte)2) << 6) +#define AV8L_FAST_PTE_AP_RO (((av8l_fast_iopte)3) << 6) +#define AV8L_FAST_PTE_ATTRINDX_SHIFT 2 +#define AV8L_FAST_PTE_ATTRINDX_MASK 0x7 +#define AV8L_FAST_PTE_nG (((av8l_fast_iopte)1) << 11) + +/* Stage-2 PTE */ +#define AV8L_FAST_PTE_HAP_FAULT (((av8l_fast_iopte)0) << 6) +#define AV8L_FAST_PTE_HAP_READ (((av8l_fast_iopte)1) << 6) +#define AV8L_FAST_PTE_HAP_WRITE (((av8l_fast_iopte)2) << 6) +#define AV8L_FAST_PTE_MEMATTR_OIWB (((av8l_fast_iopte)0xf) << 2) +#define AV8L_FAST_PTE_MEMATTR_NC (((av8l_fast_iopte)0x5) << 2) +#define AV8L_FAST_PTE_MEMATTR_DEV (((av8l_fast_iopte)0x1) << 2) + +/* Register bits */ +#define ARM_32_LPAE_TCR_EAE (1 << 31) +#define ARM_64_LPAE_S2_TCR_RES1 (1 << 31) + +#define AV8L_FAST_TCR_TG0_4K (0 << 14) +#define AV8L_FAST_TCR_TG0_64K (1 << 14) +#define AV8L_FAST_TCR_TG0_16K (2 << 14) + +#define AV8L_FAST_TCR_SH0_SHIFT 12 +#define AV8L_FAST_TCR_SH0_MASK 0x3 +#define AV8L_FAST_TCR_SH_NS 0 +#define AV8L_FAST_TCR_SH_OS 2 +#define AV8L_FAST_TCR_SH_IS 3 + +#define AV8L_FAST_TCR_ORGN0_SHIFT 10 +#define AV8L_FAST_TCR_IRGN0_SHIFT 8 +#define AV8L_FAST_TCR_RGN_MASK 0x3 +#define AV8L_FAST_TCR_RGN_NC 0 +#define AV8L_FAST_TCR_RGN_WBWA 1 +#define AV8L_FAST_TCR_RGN_WT 2 +#define AV8L_FAST_TCR_RGN_WB 3 + +#define AV8L_FAST_TCR_SL0_SHIFT 6 +#define AV8L_FAST_TCR_SL0_MASK 0x3 + +#define AV8L_FAST_TCR_T0SZ_SHIFT 0 +#define AV8L_FAST_TCR_SZ_MASK 0xf + +#define AV8L_FAST_TCR_PS_SHIFT 16 +#define AV8L_FAST_TCR_PS_MASK 0x7 + +#define AV8L_FAST_TCR_IPS_SHIFT 32 +#define AV8L_FAST_TCR_IPS_MASK 0x7 + +#define AV8L_FAST_TCR_PS_32_BIT 0x0ULL +#define AV8L_FAST_TCR_PS_36_BIT 0x1ULL +#define AV8L_FAST_TCR_PS_40_BIT 0x2ULL +#define AV8L_FAST_TCR_PS_42_BIT 0x3ULL +#define AV8L_FAST_TCR_PS_44_BIT 0x4ULL +#define AV8L_FAST_TCR_PS_48_BIT 0x5ULL + +#define AV8L_FAST_TCR_EPD1_SHIFT 23 +#define AV8L_FAST_TCR_EPD1_FAULT 1 + +#define AV8L_FAST_MAIR_ATTR_SHIFT(n) ((n) << 3) +#define AV8L_FAST_MAIR_ATTR_MASK 0xff +#define AV8L_FAST_MAIR_ATTR_DEVICE 0x04 +#define AV8L_FAST_MAIR_ATTR_NC 0x44 +#define AV8L_FAST_MAIR_ATTR_WBRWA 0xff +#define AV8L_FAST_MAIR_ATTR_UPSTREAM 0xf4 +#define AV8L_FAST_MAIR_ATTR_IDX_NC 0 +#define AV8L_FAST_MAIR_ATTR_IDX_CACHE 1 +#define AV8L_FAST_MAIR_ATTR_IDX_DEV 2 +#define AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM 3 + +#define AV8L_FAST_PAGE_SHIFT 12 + +#define PTE_MAIR_IDX(pte) \ + ((pte >> AV8L_FAST_PTE_ATTRINDX_SHIFT) & \ + AV8L_FAST_PTE_ATTRINDX_MASK) + +#define PTE_SH_IDX(pte) (pte & AV8L_FAST_PTE_SH_MASK) + +#define iopte_pmd_offset(pmds, base, iova) (pmds + ((iova - base) >> 12)) + +static inline dma_addr_t av8l_dma_addr(void *addr) +{ + if (is_vmalloc_addr(addr)) + return page_to_phys(vmalloc_to_page(addr)) + + offset_in_page(addr); + return virt_to_phys(addr); +} + +static void __av8l_clean_range(struct device *dev, void *start, void *end) +{ + size_t size; + void *region_end; + unsigned long page_end; + + if (is_vmalloc_addr(start)) { + while (start < end) { + page_end = round_down((unsigned long)start + PAGE_SIZE, + PAGE_SIZE); + region_end = min_t(void *, end, page_end); + size = region_end - start; + dma_sync_single_for_device(dev, av8l_dma_addr(start), + size, DMA_TO_DEVICE); + start = region_end; + } + } else { + size = end - start; + dma_sync_single_for_device(dev, av8l_dma_addr(start), size, + DMA_TO_DEVICE); + } +} + +static void av8l_clean_range(struct io_pgtable_cfg *cfg, av8l_fast_iopte *start, + av8l_fast_iopte *end) +{ + if (!cfg->coherent_walk) + __av8l_clean_range(cfg->iommu_dev, start, end); +} + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB + +#include + +static ATOMIC_NOTIFIER_HEAD(av8l_notifier_list); + +void av8l_register_notify(struct notifier_block *nb) +{ + atomic_notifier_chain_register(&av8l_notifier_list, nb); +} +EXPORT_SYMBOL(av8l_register_notify); + +static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep) +{ + if (unlikely(*ptep)) { + atomic_notifier_call_chain( + &av8l_notifier_list, MAPPED_OVER_STALE_TLB, + (void *) ptep); + pr_err("Tried to map over a non-vacant pte: 0x%llx @ %p\n", + *ptep, ptep); + pr_err("Nearby memory:\n"); + print_hex_dump(KERN_ERR, "pgtbl: ", DUMP_PREFIX_ADDRESS, + 32, 8, ptep - 16, 32 * sizeof(*ptep), false); + } +} + +void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, u64 base, + u64 end, bool skip_sync) +{ + int i; + struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); + struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops); + av8l_fast_iopte *pmdp = iopte_pmd_offset(data->pmds, data->base, base); + + for (i = base >> AV8L_FAST_PAGE_SHIFT; + i <= (end >> AV8L_FAST_PAGE_SHIFT); ++i) { + if (!(*pmdp & AV8L_FAST_PTE_VALID)) { + *pmdp = 0; + if (!skip_sync) + av8l_clean_range(&iop->cfg, pmdp, pmdp + 1); + } + pmdp++; + } +} +#else +static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep) +{ +} +#endif + +static av8l_fast_iopte +av8l_fast_prot_to_pte(struct av8l_fast_io_pgtable *data, int prot) +{ + av8l_fast_iopte pte = AV8L_FAST_PTE_XN + | AV8L_FAST_PTE_TYPE_PAGE + | AV8L_FAST_PTE_AF + | AV8L_FAST_PTE_nG + | AV8L_FAST_PTE_SH_OS; + + if (prot & IOMMU_MMIO) + pte |= (AV8L_FAST_MAIR_ATTR_IDX_DEV + << AV8L_FAST_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_CACHE) + pte |= (AV8L_FAST_MAIR_ATTR_IDX_CACHE + << AV8L_FAST_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_SYS_CACHE) + pte |= (AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM + << AV8L_FAST_PTE_ATTRINDX_SHIFT); + if (!(prot & IOMMU_WRITE)) + pte |= AV8L_FAST_PTE_AP_RO; + else + pte |= AV8L_FAST_PTE_AP_RW; + + return pte; +} + +static int av8l_fast_map(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +{ + struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); + struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops); + av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova); + unsigned long i, nptes = size >> AV8L_FAST_PAGE_SHIFT; + av8l_fast_iopte pte; + + pte = av8l_fast_prot_to_pte(data, prot); + paddr &= AV8L_FAST_PTE_ADDR_MASK; + for (i = 0; i < nptes; i++, paddr += SZ_4K) { + __av8l_check_for_stale_tlb(ptep + i); + *(ptep + i) = pte | paddr; + } + av8l_clean_range(&iop->cfg, ptep, ptep + nptes); + + return 0; +} + +int av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + return av8l_fast_map(ops, iova, paddr, size, prot, GFP_ATOMIC); +} + +static int av8l_fast_map_pages(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, + size_t pgsize, size_t pgcount, int prot, gfp_t gfp, + size_t *mapped) +{ + int ret = av8l_fast_map(ops, iova, paddr, pgsize * pgcount, prot, gfp); + + if (!ret) + *mapped = pgsize * pgcount; + + return ret; +} + +static size_t +__av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova, + size_t size, bool allow_stale_tlb) +{ + struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); + struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops); + unsigned long nptes; + av8l_fast_iopte *ptep; + int val = allow_stale_tlb + ? AV8L_FAST_PTE_UNMAPPED_NEED_TLBI + : 0; + + ptep = iopte_pmd_offset(data->pmds, data->base, iova); + nptes = size >> AV8L_FAST_PAGE_SHIFT; + + memset(ptep, val, sizeof(*ptep) * nptes); + av8l_clean_range(&iop->cfg, ptep, ptep + nptes); + if (!allow_stale_tlb) + io_pgtable_tlb_flush_all(&data->iop); + + return size; +} + +/* caller must take care of tlb cache maintenance */ +void av8l_fast_unmap_public(struct io_pgtable_ops *ops, unsigned long iova, + size_t size) +{ + __av8l_fast_unmap(ops, iova, size, true); +} + +static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova, + size_t size, struct iommu_iotlb_gather *gather) +{ + return __av8l_fast_unmap(ops, iova, size, false); +} + +static size_t av8l_fast_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, size_t pgsize, + size_t pgcount, struct iommu_iotlb_gather *gather) +{ + return __av8l_fast_unmap(ops, iova, pgsize * pgcount, false); +} + +/* TODO: Add this back in android-mainline */ +static int __maybe_unused av8l_fast_map_sg(struct io_pgtable_ops *ops, + unsigned long iova, struct scatterlist *sgl, + unsigned int nents, int prot, gfp_t gfp, size_t *mapped) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nents, i) { + av8l_fast_map(ops, iova, sg_phys(sg), sg->length, prot, gfp); + iova += sg->length; + *mapped += sg->length; + } + + return 0; +} + +int av8l_fast_map_sg_public(struct io_pgtable_ops *ops, + unsigned long iova, struct scatterlist *sgl, + unsigned int nents, int prot, size_t *mapped) +{ + return av8l_fast_map_sg(ops, iova, sgl, nents, prot, GFP_ATOMIC, mapped); +} + +#if defined(CONFIG_ARM64) +#define FAST_PGDNDX(va) (((va) & 0x7fc0000000) >> 27) +#elif defined(CONFIG_ARM) +#define FAST_PGDNDX(va) (((va) & 0xc0000000) >> 27) +#endif + +static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); + av8l_fast_iopte pte, *pgdp, *pudp, *pmdp; + unsigned long pgd; + phys_addr_t phys; + const unsigned long pts = AV8L_FAST_PTE_TYPE_SHIFT; + const unsigned long ptm = AV8L_FAST_PTE_TYPE_MASK; + const unsigned long ptt = AV8L_FAST_PTE_TYPE_TABLE; + const unsigned long ptp = AV8L_FAST_PTE_TYPE_PAGE; + const av8l_fast_iopte am = AV8L_FAST_PTE_ADDR_MASK; + + /* TODO: clean up some of these magic numbers... */ + + pgd = (unsigned long)data->pgd | FAST_PGDNDX(iova); + pgdp = (av8l_fast_iopte *)pgd; + + pte = *pgdp; + if (((pte >> pts) & ptm) != ptt) + return 0; + pudp = phys_to_virt((pte & am) | ((iova & 0x3fe00000) >> 18)); + + pte = *pudp; + if (((pte >> pts) & ptm) != ptt) + return 0; + pmdp = phys_to_virt((pte & am) | ((iova & 0x1ff000) >> 9)); + + pte = *pmdp; + if (((pte >> pts) & ptm) != ptp) + return 0; + phys = pte & am; + + return phys | (iova & 0xfff); +} + +phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops, + unsigned long iova) +{ + return av8l_fast_iova_to_phys(ops, iova); +} + +static bool av8l_fast_iova_coherent(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops); + av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova); + + return ((PTE_MAIR_IDX(*ptep) == AV8L_FAST_MAIR_ATTR_IDX_CACHE) && + ((PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_OS) || + (PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_IS))); +} + +bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops, + unsigned long iova) +{ + return av8l_fast_iova_coherent(ops, iova); +} + +static struct av8l_fast_io_pgtable * +av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg) +{ + struct av8l_fast_io_pgtable *data; + + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + + data->iop.ops = (struct io_pgtable_ops) { + .map = av8l_fast_map, + .map_pages = av8l_fast_map_pages, + .unmap = av8l_fast_unmap, + .unmap_pages = av8l_fast_unmap_pages, + .iova_to_phys = av8l_fast_iova_to_phys, + }; + + return data; +} + +/* + * We need max 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and + * 2048 pages for pmds (each pud page contains 512 table entries, each + * pointing to a pmd). + */ +#define NUM_PGD_PAGES 1 +#define NUM_PUD_PAGES 4 +#define NUM_PMD_PAGES 2048 +#define NUM_PGTBL_PAGES (NUM_PGD_PAGES + NUM_PUD_PAGES + NUM_PMD_PAGES) + +/* undefine arch specific definitions which depends on page table format */ +#undef pud_index +#undef pud_mask +#undef pud_next +#undef pmd_index +#undef pmd_mask +#undef pmd_next + +#define pud_index(addr) (((addr) >> 30) & 0x3) +#define pud_mask(addr) ((addr) & ~((1UL << 30) - 1)) +#define pud_next(addr, end) \ +({ unsigned long __boundary = pud_mask(addr + (1UL << 30));\ + (__boundary - 1 < (end) - 1) ? __boundary : (end); \ +}) + +#define pmd_index(addr) (((addr) >> 21) & 0x1ff) +#define pmd_mask(addr) ((addr) & ~((1UL << 21) - 1)) +#define pmd_next(addr, end) \ +({ unsigned long __boundary = pmd_mask(addr + (1UL << 21));\ + (__boundary - 1 < (end) - 1) ? __boundary : (end); \ +}) + +static int +av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data, + struct io_pgtable_cfg *cfg, void *cookie) +{ + int i, j, pg = 0; + struct page **pages, *page; + struct qcom_io_pgtable_info *pgtbl_info = to_qcom_io_pgtable_info(cfg); + dma_addr_t pud, pmd; + int pmd_pg_index; + dma_addr_t base = pgtbl_info->iova_base; + dma_addr_t end = pgtbl_info->iova_end; + + pages = kmalloc(sizeof(*pages) * NUM_PGTBL_PAGES, __GFP_NOWARN | + __GFP_NORETRY); + + if (!pages) + pages = vmalloc(sizeof(*pages) * NUM_PGTBL_PAGES); + + if (!pages) + return -ENOMEM; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + goto err_free_pages_arr; + pages[pg++] = page; + data->pgd = page_address(page); + + /* + * We need max 2048 entries at level 2 to map 4GB of VA space. A page + * can hold 512 entries, so we need max 4 pages. + */ + for (i = pud_index(base), pud = base; pud < end; + ++i, pud = pud_next(pud, end)) { + av8l_fast_iopte pte, *ptep; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + goto err_free_pages; + pages[pg++] = page; + data->puds[i] = page_address(page); + pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE; + ptep = ((av8l_fast_iopte *)data->pgd) + i; + *ptep = pte; + } + av8l_clean_range(cfg, data->pgd, data->pgd + 4); + + /* + * We have max 4 puds, each of which can point to 512 pmds, so we'll + * have max 2048 pmds, each of which can hold 512 ptes, for a grand + * total of 2048*512=1048576 PTEs. + */ + pmd_pg_index = pg; + for (i = pud_index(base), pud = base; pud < end; + ++i, pud = pud_next(pud, end)) { + for (j = pmd_index(pud), pmd = pud; pmd < pud_next(pud, end); + ++j, pmd = pmd_next(pmd, end)) { + av8l_fast_iopte pte, *pudp; + void *addr; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + goto err_free_pages; + pages[pg++] = page; + + addr = page_address(page); + av8l_clean_range(cfg, addr, addr + SZ_4K); + + pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE; + pudp = data->puds[i] + j; + *pudp = pte; + } + av8l_clean_range(cfg, data->puds[i], data->puds[i] + 512); + } + + /* + * We map the pmds into a virtually contiguous space so that we + * don't have to traverse the first two levels of the page tables + * to find the appropriate pud. Instead, it will be a simple + * offset from the virtual base of the pmds. + */ + data->pmds = vmap(&pages[pmd_pg_index], pg - pmd_pg_index, + VM_IOREMAP, PAGE_KERNEL); + if (!data->pmds) + goto err_free_pages; + + data->pages = pages; + data->base = base; + data->end = end; + data->nr_pages = pg; + return 0; + +err_free_pages: + for (i = 0; i < pg; ++i) + __free_page(pages[i]); +err_free_pages_arr: + kvfree(pages); + return -ENOMEM; +} + +static struct io_pgtable * +av8l_fast_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) +{ + u64 reg; + struct av8l_fast_io_pgtable *data = + av8l_fast_alloc_pgtable_data(cfg); + typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr; + + if (!data) + return NULL; + + /* restrict according to the fast map requirements */ + cfg->ias = 32; + cfg->pgsize_bitmap = SZ_4K; + + /* TCR */ + if (cfg->coherent_walk) { + tcr->sh = AV8L_FAST_TCR_SH_IS; + tcr->irgn = AV8L_FAST_TCR_RGN_WBWA; + tcr->orgn = AV8L_FAST_TCR_RGN_WBWA; + if (WARN_ON(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)) + goto out_free_data; + } else { + tcr->sh = AV8L_FAST_TCR_SH_OS; + tcr->irgn = AV8L_FAST_TCR_RGN_NC; + if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)) + tcr->orgn = AV8L_FAST_TCR_RGN_NC; + else + tcr->orgn = AV8L_FAST_TCR_RGN_WBWA; + } + + tcr->tg = AV8L_FAST_TCR_TG0_4K; + + switch (cfg->oas) { + case 32: + tcr->ips = AV8L_FAST_TCR_PS_32_BIT; + break; + case 36: + tcr->ips = AV8L_FAST_TCR_PS_36_BIT; + break; + case 40: + tcr->ips = AV8L_FAST_TCR_PS_40_BIT; + break; + case 42: + tcr->ips = AV8L_FAST_TCR_PS_42_BIT; + break; + case 44: + tcr->ips = AV8L_FAST_TCR_PS_44_BIT; + break; + case 48: + tcr->ips = AV8L_FAST_TCR_PS_48_BIT; + break; + default: + goto out_free_data; + } + + tcr->tsz = 64ULL - cfg->ias; + + /* MAIRs */ + reg = (AV8L_FAST_MAIR_ATTR_NC + << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_NC)) | + (AV8L_FAST_MAIR_ATTR_WBRWA + << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_CACHE)) | + (AV8L_FAST_MAIR_ATTR_DEVICE + << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_DEV)) | + (AV8L_FAST_MAIR_ATTR_UPSTREAM + << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_UPSTREAM)); + + cfg->arm_lpae_s1_cfg.mair = reg; + + /* Allocate all page table memory! */ + if (av8l_fast_prepopulate_pgtables(data, cfg, cookie)) + goto out_free_data; + + /* TTBRs */ + cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd); + return &data->iop; + +out_free_data: + kfree(data); + return NULL; +} + +static void av8l_fast_free_pgtable(struct io_pgtable *iop) +{ + int i; + struct av8l_fast_io_pgtable *data = iof_pgtable_to_data(iop); + + vunmap(data->pmds); + for (i = 0; i < data->nr_pages; ++i) + __free_page(data->pages[i]); + kvfree(data->pages); + kfree(data); +} + +struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns = { + .alloc = av8l_fast_alloc_pgtable, + .free = av8l_fast_free_pgtable, +}; + + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_SELFTEST + +#include + +static struct io_pgtable_cfg *cfg_cookie; + +static void dummy_tlb_flush_all(void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static void dummy_tlb_flush(unsigned long iova, size_t size, size_t granule, + void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static void dummy_tlb_add_page(struct iommu_iotlb_gather *gather, + unsigned long iova, size_t granule, void *cookie) +{ + dummy_tlb_flush(iova, granule, granule, cookie); +} + +static struct iommu_flush_ops dummy_tlb_ops __initdata = { + .tlb_flush_all = dummy_tlb_flush_all, + .tlb_flush_walk = dummy_tlb_flush, + .tlb_add_page = dummy_tlb_add_page, +}; + +/* + * Returns true if the iova range is successfully mapped to the contiguous + * phys range in ops. + */ +static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops, + const unsigned long iova_start, + const phys_addr_t phys_start, + const size_t size) +{ + u64 iova = iova_start; + phys_addr_t phys = phys_start; + + while (iova < (iova_start + size)) { + /* + 42 just to make sure offsetting is working */ + if (ops->iova_to_phys(ops, iova + 42) != (phys + 42)) + return false; + iova += SZ_4K; + phys += SZ_4K; + } + return true; +} + +static int __init av8l_fast_positive_testing(void) +{ + int failed = 0; + u64 iova; + struct io_pgtable_ops *ops; + struct qcom_io_pgtable_info pgtable_info; + struct av8l_fast_io_pgtable *data; + av8l_fast_iopte *pmds; + u64 max = SZ_1G * 4ULL - 1; + u64 base = 0; + + pgtable_info.iova_base = base; + pgtable_info.iova_end = max; + pgtable_info.cfg = (struct io_pgtable_cfg) { + .quirks = 0, + .tlb = &dummy_tlb_ops, + .ias = 32, + .oas = 32, + .pgsize_bitmap = SZ_4K, + .coherent_walk = true, + }; + + cfg_cookie = &pgtable_info.pgtbl_cfg; + ops = alloc_io_pgtable_ops(ARM_V8L_FAST, &pgtable_info.pgtbl_cfg, + &pgtable_info.pgtbl_cfg); + + if (WARN_ON(!ops)) + return 1; + + data = iof_pgtable_ops_to_data(ops); + pmds = data->pmds; + + /* map the entire 4GB VA space with 4K map calls */ + for (iova = base; iova < max; iova += SZ_4K) { + if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) { + failed++; + continue; + } + } + if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base, + base, max - base))) + failed++; + + /* unmap it all */ + for (iova = base; iova < max; iova += SZ_4K) { + if (WARN_ON(ops->unmap(ops, iova, SZ_4K, NULL) != SZ_4K)) + failed++; + } + + /* sweep up TLB proving PTEs */ + av8l_fast_clear_stale_ptes(ops, base, max, false); + + /* map the entire 4GB VA space with 8K map calls */ + for (iova = base; iova < max; iova += SZ_8K) { + if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) { + failed++; + continue; + } + } + + if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base, + base, max - base))) + failed++; + + /* unmap it all with 8K unmap calls */ + for (iova = base; iova < max; iova += SZ_8K) { + if (WARN_ON(ops->unmap(ops, iova, SZ_8K, NULL) != SZ_8K)) + failed++; + } + + /* sweep up TLB proving PTEs */ + av8l_fast_clear_stale_ptes(ops, base, max, false); + + /* map the entire 4GB VA space with 16K map calls */ + for (iova = base; iova < max; iova += SZ_16K) { + if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) { + failed++; + continue; + } + } + + if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base, + base, max - base))) + failed++; + + /* unmap it all */ + for (iova = base; iova < max; iova += SZ_16K) { + if (WARN_ON(ops->unmap(ops, iova, SZ_16K, NULL) != SZ_16K)) + failed++; + } + + /* sweep up TLB proving PTEs */ + av8l_fast_clear_stale_ptes(ops, base, max, false); + + /* map the entire 4GB VA space with 64K map calls */ + for (iova = base; iova < max; iova += SZ_64K) { + if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) { + failed++; + continue; + } + } + + if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base, + base, max - base))) + failed++; + + /* unmap it all at once */ + if (WARN_ON(ops->unmap(ops, base, max - base, NULL) != (max - base))) + failed++; + + free_io_pgtable_ops(ops); + return failed; +} + +static int __init av8l_fast_do_selftests(void) +{ + int failed = 0; + + failed += av8l_fast_positive_testing(); + + pr_err("selftest: completed with %d failures\n", failed); + + return 0; +} +subsys_initcall(av8l_fast_do_selftests); +#endif diff --git a/drivers/iommu/iommu-logger.c b/drivers/iommu/iommu-logger.c new file mode 100644 index 000000000000..99e2824da2a1 --- /dev/null +++ b/drivers/iommu/iommu-logger.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include "qcom-io-pgtable.h" + +#include "iommu-logger.h" + +static DEFINE_MUTEX(iommu_debug_attachments_lock); +static LIST_HEAD(iommu_debug_attachments); + +static unsigned int iommu_logger_pgtable_levels(struct io_pgtable *iop) +{ + unsigned int va_bits, pte_size, bits_per_level, pg_shift; + unsigned long ias = iop->cfg.ias; + + switch ((u32)iop->fmt) { + case ARM_32_LPAE_S1: + case ARM_64_LPAE_S1: +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + case ARM_V8L_FAST: +#endif + case QCOM_ARM_64_LPAE_S1: + pte_size = sizeof(u64); + break; + default: + return 0; + } + + pg_shift = __ffs(iop->cfg.pgsize_bitmap); + bits_per_level = pg_shift - ilog2(pte_size); + va_bits = ias - pg_shift; + return DIV_ROUND_UP(va_bits, bits_per_level); +} + +static enum iommu_logger_pgtable_fmt iommu_logger_pgtable_fmt_lut( + enum io_pgtable_fmt fmt) +{ + switch ((u32)fmt) { + case ARM_32_LPAE_S1: + return IOMMU_LOGGER_ARM_32_LPAE_S1; + case ARM_64_LPAE_S1: +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + case ARM_V8L_FAST: +#endif + case QCOM_ARM_64_LPAE_S1: + return IOMMU_LOGGER_ARM_64_LPAE_S1; + default: + return IOMMU_LOGGER_MAX_PGTABLE_FMTS; + } +} + +static int iommu_logger_domain_ttbrs(struct io_pgtable *iop, void **ttbr0_ptr, + void **ttbr1_ptr) +{ + int ret; + u64 ttbr0; + + switch ((u32)iop->fmt) { + case ARM_32_LPAE_S1: + case ARM_64_LPAE_S1: +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + case ARM_V8L_FAST: +#endif + case QCOM_ARM_64_LPAE_S1: + ttbr0 = iop->cfg.arm_lpae_s1_cfg.ttbr; + ret = 0; + break; + default: + ret = -EINVAL; + } + + if (!ret) { + *ttbr0_ptr = phys_to_virt(ttbr0); + /* + * FIXME - fix ttbr1 retrieval later. In this kernel version + * struct io_pgtable no longer contains this information. + */ + *ttbr1_ptr = NULL; + } + + return ret; +} + +static struct iommu_debug_attachment *iommu_logger_init( + struct iommu_domain *domain, + struct device *dev, + struct io_pgtable *iop) +{ + struct iommu_debug_attachment *logger; + char *client_name; + struct iommu_group *group; + unsigned int levels = iommu_logger_pgtable_levels(iop); + enum iommu_logger_pgtable_fmt fmt = iommu_logger_pgtable_fmt_lut( + iop->fmt); + void *ttbr0, *ttbr1; + int ret; + + if (!levels || fmt == IOMMU_LOGGER_MAX_PGTABLE_FMTS) + return ERR_PTR(-EINVAL); + + ret = iommu_logger_domain_ttbrs(iop, &ttbr0, &ttbr1); + if (ret) + return ERR_PTR(ret); + + logger = kzalloc(sizeof(*logger), GFP_KERNEL); + if (!logger) + return ERR_PTR(-ENOMEM); + + client_name = kasprintf(GFP_KERNEL, "%s", kobject_name(&dev->kobj)); + if (!client_name) { + kfree(logger); + return ERR_PTR(-ENOMEM); + } + + group = iommu_group_get(dev); + iommu_group_put(group); + + INIT_LIST_HEAD(&logger->list); + logger->domain = domain; + logger->group = group; + logger->client_name = client_name; + logger->fmt = fmt; + logger->levels = levels; + logger->ttbr0 = ttbr0; + logger->ttbr1 = ttbr1; + + return logger; +} + +int iommu_logger_register(struct iommu_debug_attachment **logger_out, + struct iommu_domain *domain, struct device *dev, + struct io_pgtable *iop) +{ + struct iommu_debug_attachment *logger; + + if (!logger_out || !dev || !iop) + return -EINVAL; + + logger = iommu_logger_init(domain, dev, iop); + if (IS_ERR(logger)) + return PTR_ERR(logger); + + mutex_lock(&iommu_debug_attachments_lock); + list_add(&logger->list, &iommu_debug_attachments); + mutex_unlock(&iommu_debug_attachments_lock); + + *logger_out = logger; + return 0; +} +EXPORT_SYMBOL(iommu_logger_register); + +void iommu_logger_unregister(struct iommu_debug_attachment *logger) +{ + if (!logger) + return; + + mutex_lock(&iommu_debug_attachments_lock); + list_del(&logger->list); + mutex_unlock(&iommu_debug_attachments_lock); + kfree(logger->client_name); + kfree(logger); +} +EXPORT_SYMBOL(iommu_logger_unregister); + +MODULE_DESCRIPTION("QTI IOMMU SUPPORT"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/iommu-logger.h b/drivers/iommu/iommu-logger.h new file mode 100644 index 000000000000..d750b7a1c92c --- /dev/null +++ b/drivers/iommu/iommu-logger.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. */ + +#ifndef __LINUX_QTI_IOMMU_LOGGER_H +#define __LINUX_QTI_IOMMU_LOGGER_H + +#include + +enum iommu_logger_pgtable_fmt { + IOMMU_LOGGER_ARM_32_LPAE_S1, + IOMMU_LOGGER_ARM_64_LPAE_S1, + IOMMU_LOGGER_MAX_PGTABLE_FMTS, +}; + +/* + * Each group may have more than one domain; but each domain may + * only have one group. + */ +struct iommu_debug_attachment { + struct iommu_domain *domain; + struct iommu_group *group; + char *client_name; + enum iommu_logger_pgtable_fmt fmt; + unsigned int levels; + /* + * Virtual addresses of the top-level page tables are stored here, + * as they are more useful for debug tools than physical addresses. + */ + void *ttbr0; + void *ttbr1; + struct list_head list; +}; + +#if IS_ENABLED(CONFIG_QTI_IOMMU_SUPPORT) + +int iommu_logger_register(struct iommu_debug_attachment **a, + struct iommu_domain *domain, struct device *dev, + struct io_pgtable *iop); +void iommu_logger_unregister(struct iommu_debug_attachment *a); +#else +static inline int iommu_logger_register(struct iommu_debug_attachment **a, + struct iommu_domain *domain, + struct device *dev, + struct io_pgtable *iop) +{ + return 0; +} + +static inline void iommu_logger_unregister(struct iommu_debug_attachment *a) {} +#endif /* CONFIG_QTI_IOMMU_LOGGER */ +#endif /* __LINUX_QTI_IOMMU_LOGGER_H */ diff --git a/drivers/iommu/qcom-dma-iommu-generic.c b/drivers/iommu/qcom-dma-iommu-generic.c new file mode 100644 index 000000000000..ef500714ff52 --- /dev/null +++ b/drivers/iommu/qcom-dma-iommu-generic.c @@ -0,0 +1,814 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2014, 2020-2021, The Linux Foundation. All rights reserved. + * Contiguous Memory Allocator for DMA mapping framework + * Copyright (c) 2010-2011 by Samsung Electronics. + * Written by: + * Marek Szyprowski + * Michal Nazarewicz + * Copyright (C) 2012, 2014-2015 ARM Ltd. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "qcom-dma-iommu-generic.h" + +static bool probe_finished; +static struct device *qcom_dma_iommu_dev; +static struct cma *qcom_dma_contiguous_default_area; + +struct pci_host_bridge *qcom_pci_find_host_bridge(struct pci_bus *bus) +{ + while (bus->parent) + bus = bus->parent; + + return to_pci_host_bridge(bus->bridge); +} + +/* + * This avoids arch-specific assembly, but may be slower since it calls + * back into the dma layer again. + */ +void qcom_arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) +{ + dma_addr_t dma_addr = phys_to_dma(qcom_dma_iommu_dev, paddr); + + dma_sync_single_for_device(qcom_dma_iommu_dev, + dma_addr, size, dir); +} + +void qcom_arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) +{ + dma_addr_t dma_addr = phys_to_dma(qcom_dma_iommu_dev, paddr); + + dma_sync_single_for_cpu(qcom_dma_iommu_dev, + dma_addr, size, dir); +} + +void qcom_arch_dma_prep_coherent(struct page *page, size_t size) +{ + phys_addr_t phys = page_to_phys(page); + dma_addr_t dma_addr = phys_to_dma(qcom_dma_iommu_dev, phys); + + dma_sync_single_for_device(qcom_dma_iommu_dev, + dma_addr, size, DMA_TO_DEVICE); +} + +static struct cma *qcom_dev_get_cma_area(struct device *dev) +{ + if (dev && dev->cma_area) + return dev->cma_area; + return qcom_dma_contiguous_default_area; +} + +struct page *qcom_dma_alloc_from_contiguous(struct device *dev, size_t count, + unsigned int align, bool no_warn) +{ + if (align > CONFIG_CMA_ALIGNMENT) + align = CONFIG_CMA_ALIGNMENT; + + return cma_alloc(qcom_dev_get_cma_area(dev), count, align, no_warn); +} + +bool qcom_dma_release_from_contiguous(struct device *dev, struct page *pages, + int count) +{ + return cma_release(qcom_dev_get_cma_area(dev), pages, count); +} + +static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp) +{ + unsigned int align = min(get_order(size), CONFIG_CMA_ALIGNMENT); + + return cma_alloc(cma, size >> PAGE_SHIFT, align, gfp & __GFP_NOWARN); +} + +struct page *qcom_dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) +{ + /* CMA can be used only in the context which permits sleeping */ + if (!gfpflags_allow_blocking(gfp)) + return NULL; + if (dev->cma_area) + return cma_alloc_aligned(dev->cma_area, size, gfp); + if (size <= PAGE_SIZE || !qcom_dma_contiguous_default_area) + return NULL; + return cma_alloc_aligned(qcom_dma_contiguous_default_area, size, gfp); +} + +void qcom_dma_free_contiguous(struct device *dev, struct page *page, size_t size) +{ + if (!cma_release(qcom_dev_get_cma_area(dev), page, + PAGE_ALIGN(size) >> PAGE_SHIFT)) + __free_pages(page, get_order(size)); +} + + +/* + * find_vm_area is not exported. Some dma apis expect that an array of + * struct pages can be saved in the vm_area, and retrieved at a later time. + */ +struct rb_root _root; +struct rb_root *root = &_root; +DEFINE_MUTEX(rbtree_lock); + +struct qcom_iommu_dma_area { + struct rb_node node; + unsigned long addr; + struct page **pages; +}; + +static void qcom_insert_vm_area(struct qcom_iommu_dma_area *area) +{ + struct rb_node **new, *parent; + + mutex_lock(&rbtree_lock); + + parent = NULL; + new = &root->rb_node; + while (*new) { + struct qcom_iommu_dma_area *entry; + + entry = rb_entry(*new, + struct qcom_iommu_dma_area, + node); + + parent = *new; + if (area->addr < entry->addr) + new = &((*new)->rb_left); + else if (area->addr > entry->addr) + new = &((*new)->rb_right); + else { + mutex_unlock(&rbtree_lock); + WARN_ON(1); + return; + } + } + + rb_link_node(&area->node, parent, new); + rb_insert_color(&area->node, root); + mutex_unlock(&rbtree_lock); +} + +static struct qcom_iommu_dma_area *qcom_find_vm_area(const void *cpu_addr) +{ + struct rb_node *node; + struct qcom_iommu_dma_area *entry; + unsigned long addr = (unsigned long)cpu_addr; + + mutex_lock(&rbtree_lock); + node = root->rb_node; + while (node) { + entry = rb_entry(node, + struct qcom_iommu_dma_area, + node); + + if (addr < entry->addr) + node = node->rb_left; + else if (addr > entry->addr) + node = node->rb_right; + else { + mutex_unlock(&rbtree_lock); + return entry; + } + } + + mutex_unlock(&rbtree_lock); + + return NULL; +} + +struct page **qcom_dma_common_find_pages(void *cpu_addr) +{ + struct qcom_iommu_dma_area *area = qcom_find_vm_area(cpu_addr); + + if (!area) + return NULL; + return area->pages; +} + +/* + * Remaps an array of PAGE_SIZE pages into another vm_area. + * Cannot be used in non-sleeping contexts + */ +void *qcom_dma_common_pages_remap(struct page **pages, size_t size, + pgprot_t prot, const void *caller) +{ + struct qcom_iommu_dma_area *area; + void *vaddr; + + area = kzalloc(sizeof(*area), GFP_KERNEL); + if (!area) + return NULL; + + vaddr = vmap(pages, PAGE_ALIGN(size) >> PAGE_SHIFT, + VM_DMA_COHERENT, prot); + if (!vaddr) { + kfree(area); + return NULL; + } + + area->pages = pages; + area->addr = (unsigned long)vaddr; + qcom_insert_vm_area(area); + + return vaddr; +} + +/* + * Remaps an allocated contiguous region into another vm_area. + * Cannot be used in non-sleeping contexts + */ +void *qcom_dma_common_contiguous_remap(struct page *page, size_t size, + pgprot_t prot, const void *caller) +{ + int count = PAGE_ALIGN(size) >> PAGE_SHIFT; + struct page **pages; + void *vaddr; + int i; + + pages = kmalloc_array(count, sizeof(struct page *), GFP_KERNEL); + if (!pages) + return NULL; + for (i = 0; i < count; i++) + pages[i] = nth_page(page, i); + vaddr = vmap(pages, count, VM_DMA_COHERENT, prot); + kfree(pages); + + return vaddr; +} + +/* + * Unmaps a range previously mapped by dma_common_contiguous_remap or + * dma_common_pages_remap. Note that dma_common_contiguous_remap does + * not insert an rb_tree entry since there is no pages array to save. + */ +void qcom_dma_common_free_remap(void *cpu_addr, size_t size) +{ + struct qcom_iommu_dma_area *area; + + /* qcom_dma_common_contiguous_remap doesn't save the pages array */ + area = qcom_find_vm_area(cpu_addr); + if (area) { + mutex_lock(&rbtree_lock); + rb_erase(&area->node, root); + mutex_unlock(&rbtree_lock); + kfree(area); + } + + vunmap(cpu_addr); +} + +static struct gen_pool *atomic_pool __ro_after_init; + +static size_t atomic_pool_size; +static unsigned long current_pool_size; + +/* Dynamic background expansion when the atomic pool is near capacity */ +static struct work_struct atomic_pool_work; + +static void dma_atomic_pool_debugfs_init(void) +{ + struct dentry *root; + + root = debugfs_create_dir("qcom_dma_pools", NULL); + if (IS_ERR_OR_NULL(root)) + return; + + debugfs_create_ulong("pool_size", 0400, root, ¤t_pool_size); +} + +static void dma_atomic_pool_size_add(gfp_t gfp, size_t size) +{ + current_pool_size += size; +} + +static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size, + gfp_t gfp) +{ + unsigned int order; + struct page *page = NULL; + void *addr; + int ret = -ENOMEM; + + /* Cannot allocate larger than MAX_ORDER - 1 */ + order = min(get_order(pool_size), MAX_ORDER - 1); + + do { + pool_size = 1 << (PAGE_SHIFT + order); + + if (qcom_dev_get_cma_area(NULL)) + page = qcom_dma_alloc_from_contiguous(NULL, 1 << order, + order, false); + else + page = alloc_pages(gfp, order); + } while (!page && order-- > 0); + if (!page) + goto out; + + qcom_arch_dma_prep_coherent(page, pool_size); + + addr = qcom_dma_common_contiguous_remap(page, pool_size, + pgprot_dmacoherent(PAGE_KERNEL), + __builtin_return_address(0)); + if (!addr) + goto free_page; + + ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page), + pool_size, NUMA_NO_NODE); + if (ret) + goto remove_mapping; + + dma_atomic_pool_size_add(gfp, pool_size); + return 0; + +remove_mapping: + qcom_dma_common_free_remap(addr, pool_size); +free_page: + if (!qcom_dma_release_from_contiguous(NULL, page, 1 << order)) + __free_pages(page, order); +out: + return ret; +} + +static void atomic_pool_resize(struct gen_pool *pool, gfp_t gfp) +{ + if (pool && gen_pool_avail(pool) < atomic_pool_size) + atomic_pool_expand(pool, gen_pool_size(pool), gfp); +} + +static void atomic_pool_work_fn(struct work_struct *work) +{ + atomic_pool_resize(atomic_pool, GFP_KERNEL); +} + +static struct gen_pool *__dma_atomic_pool_init(size_t pool_size, gfp_t gfp) +{ + struct gen_pool *pool; + int ret; + + pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE); + if (!pool) + return NULL; + + gen_pool_set_algo(pool, gen_pool_first_fit_order_align, NULL); + + ret = atomic_pool_expand(pool, pool_size, gfp); + if (ret) { + gen_pool_destroy(pool); + pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n", + pool_size >> 10, &gfp); + return NULL; + } + + pr_info("DMA preallocated %zu KiB %pGg pool for atomic allocations\n", + gen_pool_size(pool) >> 10, &gfp); + return pool; +} + +static int dma_atomic_pool_init(struct device *dev) +{ + int ret = 0; + unsigned long pages; + + /* Default the pool size to 128KB per 1 GB of memory, min 128 KB, max MAX_ORDER - 1. */ + pages = totalram_pages() / (SZ_1G / SZ_128K); + pages = min_t(unsigned long, pages, MAX_ORDER_NR_PAGES); + atomic_pool_size = max_t(size_t, pages << PAGE_SHIFT, SZ_128K); + INIT_WORK(&atomic_pool_work, atomic_pool_work_fn); + + atomic_pool = __dma_atomic_pool_init(atomic_pool_size, GFP_KERNEL); + if (!atomic_pool) + return -ENOMEM; + + dma_atomic_pool_debugfs_init(); + return ret; +} + +/* + * Couldn't implement this via dma_alloc_attrs(qcom_iommu_dma_dev, GFP_ATOMIC) + * due to dma_free_from_pool only passing in cpu_addr & not dma_handle. + */ +void *qcom_dma_alloc_from_pool(struct device *dev, size_t size, + struct page **ret_page, gfp_t flags) +{ + unsigned long val; + void *ptr = NULL; + + if (!atomic_pool) { + WARN(1, "coherent pool not initialised!\n"); + return NULL; + } + + val = gen_pool_alloc(atomic_pool, size); + if (val) { + phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val); + + *ret_page = pfn_to_page(__phys_to_pfn(phys)); + ptr = (void *)val; + memset(ptr, 0, size); + } + if (gen_pool_avail(atomic_pool) < atomic_pool_size) + schedule_work(&atomic_pool_work); + + return ptr; +} + +bool qcom_dma_free_from_pool(struct device *dev, void *start, size_t size) +{ + if (!atomic_pool || !gen_pool_has_addr(atomic_pool, (unsigned long)start, size)) + return false; + gen_pool_free(atomic_pool, (unsigned long)start, size); + return true; +} + +static void qcom_dma_atomic_pool_exit(struct device *dev) +{ + unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT; + void *addr; + struct page *page; + + /* + * Find the starting address. Pool is expected to be unused. + * + * While the pool size can expand, it is okay to use the initial size + * here, as this function can only ever be called prior to the pool + * ever being used. The pool can only expand when an allocation is satisfied + * from it, which would not be possible by the time this function is called. + * Therefore the size of the pool will be the initial size. + */ + addr = (void *)gen_pool_alloc(atomic_pool, atomic_pool_size); + if (!addr) { + WARN_ON(1); + return; + } + + gen_pool_free(atomic_pool, (unsigned long)addr, atomic_pool_size); + gen_pool_destroy(atomic_pool); + page = vmalloc_to_page(addr); + qcom_dma_common_free_remap(addr, atomic_pool_size); + qcom_dma_release_from_contiguous(dev, page, nr_pages); +} + +/* + * struct dma_coherent_mem is private, so we cna't access it. 0 indicates + * an error condition for dma_mmap_from_dev_coherent. + */ +int qcom_dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma, + void *vaddr, size_t size, int *ret) +{ + return 0; +} + +/* + * Return the page attributes used for mapping dma_alloc_* memory, either in + * kernel space if remapping is needed, or to userspace through dma_mmap_*. + */ +pgprot_t qcom_dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) +{ + if (dev_is_dma_coherent(dev)) + return prot; +#ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE + if (attrs & DMA_ATTR_WRITE_COMBINE) + return pgprot_writecombine(prot); +#endif + return pgprot_dmacoherent(prot); +} + +/** + * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API + * page flags. + * @dir: Direction of DMA transfer + * @coherent: Is the DMA master cache-coherent? + * @attrs: DMA attributes for the mapping + * + * Return: corresponding IOMMU API page protection flags + */ +int qcom_dma_info_to_prot(enum dma_data_direction dir, bool coherent, + unsigned long attrs) +{ + int prot = coherent ? IOMMU_CACHE : 0; + + if (attrs & DMA_ATTR_PRIVILEGED) + prot |= IOMMU_PRIV; + + if (attrs & DMA_ATTR_SYS_CACHE_ONLY) + prot |= IOMMU_SYS_CACHE; + + if (attrs & DMA_ATTR_SYS_CACHE_ONLY_NWA) + prot |= IOMMU_SYS_CACHE_NWA; + + switch (dir) { + case DMA_BIDIRECTIONAL: + return prot | IOMMU_READ | IOMMU_WRITE; + case DMA_TO_DEVICE: + return prot | IOMMU_READ; + case DMA_FROM_DEVICE: + return prot | IOMMU_WRITE; + default: + return 0; + } +} + +/* + * The DMA API client is passing in a scatterlist which could describe + * any old buffer layout, but the IOMMU API requires everything to be + * aligned to IOMMU pages. Hence the need for this complicated bit of + * impedance-matching, to be able to hand off a suitably-aligned list, + * but still preserve the original offsets and sizes for the caller. + */ +size_t qcom_iommu_dma_prepare_map_sg(struct device *dev, struct iova_domain *iovad, + struct scatterlist *sg, int nents) +{ + struct scatterlist *s, *prev = NULL; + size_t iova_len = 0; + unsigned long mask = dma_get_seg_boundary(dev); + int i; + + /* + * Work out how much IOVA space we need, and align the segments to + * IOVA granules for the IOMMU driver to handle. With some clever + * trickery we can modify the list in-place, but reversibly, by + * stashing the unaligned parts in the as-yet-unused DMA fields. + */ + for_each_sg(sg, s, nents, i) { + size_t s_iova_off = iova_offset(iovad, s->offset); + size_t s_length = s->length; + size_t pad_len = (mask - iova_len + 1) & mask; + + sg_dma_address(s) = s_iova_off; + sg_dma_len(s) = s_length; + s->offset -= s_iova_off; + s_length = iova_align(iovad, s_length + s_iova_off); + s->length = s_length; + + /* + * Due to the alignment of our single IOVA allocation, we can + * depend on these assumptions about the segment boundary mask: + * - If mask size >= IOVA size, then the IOVA range cannot + * possibly fall across a boundary, so we don't care. + * - If mask size < IOVA size, then the IOVA range must start + * exactly on a boundary, therefore we can lay things out + * based purely on segment lengths without needing to know + * the actual addresses beforehand. + * - The mask must be a power of 2, so pad_len == 0 if + * iova_len == 0, thus we cannot dereference prev the first + * time through here (i.e. before it has a meaningful value). + */ + if (pad_len && pad_len < s_length - 1) { + prev->length += pad_len; + iova_len += pad_len; + } + + iova_len += s_length; + prev = s; + } + + return iova_len; +} + +/* + * Prepare a successfully-mapped scatterlist to give back to the caller. + * + * At this point the segments are already laid out by iommu_dma_map_sg() to + * avoid individually crossing any boundaries, so we merely need to check a + * segment's start address to avoid concatenating across one. + */ +int qcom_iommu_dma_finalise_sg(struct device *dev, struct scatterlist *sg, int nents, + dma_addr_t dma_addr) +{ + struct scatterlist *s, *cur = sg; + unsigned long seg_mask = dma_get_seg_boundary(dev); + unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev); + int i, count = 0; + + for_each_sg(sg, s, nents, i) { + /* Restore this segment's original unaligned fields first */ + unsigned int s_iova_off = sg_dma_address(s); + unsigned int s_length = sg_dma_len(s); + unsigned int s_iova_len = s->length; + + s->offset += s_iova_off; + s->length = s_length; + sg_dma_address(s) = DMA_MAPPING_ERROR; + sg_dma_len(s) = 0; + + /* + * Now fill in the real DMA data. If... + * - there is a valid output segment to append to + * - and this segment starts on an IOVA page boundary + * - but doesn't fall at a segment boundary + * - and wouldn't make the resulting output segment too long + */ + if (cur_len && !s_iova_off && (dma_addr & seg_mask) && + (max_len - cur_len >= s_length)) { + /* ...then concatenate it with the previous one */ + cur_len += s_length; + } else { + /* Otherwise start the next output segment */ + if (i > 0) + cur = sg_next(cur); + cur_len = s_length; + count++; + + sg_dma_address(cur) = dma_addr + s_iova_off; + } + + sg_dma_len(cur) = cur_len; + dma_addr += s_iova_len; + + if (s_length + s_iova_off < s_iova_len) + cur_len = 0; + } + return count; +} + +/* + * If mapping failed, then just restore the original list, + * but making sure the DMA fields are invalidated. + */ +void qcom_iommu_dma_invalidate_sg(struct scatterlist *sg, int nents) +{ + struct scatterlist *s; + int i; + + for_each_sg(sg, s, nents, i) { + if (sg_dma_address(s) != DMA_MAPPING_ERROR) + s->offset += sg_dma_address(s); + if (sg_dma_len(s)) + s->length = sg_dma_len(s); + sg_dma_address(s) = DMA_MAPPING_ERROR; + sg_dma_len(s) = 0; + } +} + +/** + * __iommu_dma_mmap - Map a buffer into provided user VMA + * @pages: Array representing buffer from __iommu_dma_alloc() + * @size: Size of buffer in bytes + * @vma: VMA describing requested userspace mapping + * + * Maps the pages of the buffer in @pages into @vma. The caller is responsible + * for verifying the correct size and protection of @vma beforehand. + */ +static int __qcom_iommu_dma_mmap(struct page **pages, size_t size, + struct vm_area_struct *vma) +{ + return vm_map_pages(vma, pages, PAGE_ALIGN(size) >> PAGE_SHIFT); +} + +int qcom_iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned long pfn, off = vma->vm_pgoff; + int ret; + + vma->vm_page_prot = qcom_dma_pgprot(dev, vma->vm_page_prot, attrs); + + if (qcom_dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) + return ret; + + if (off >= nr_pages || vma_pages(vma) > nr_pages - off) + return -ENXIO; + + if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) { + struct page **pages = qcom_dma_common_find_pages(cpu_addr); + + if (pages) + return __qcom_iommu_dma_mmap(pages, size, vma); + pfn = vmalloc_to_pfn(cpu_addr); + } else { + pfn = page_to_pfn(virt_to_page(cpu_addr)); + } + + return remap_pfn_range(vma, vma->vm_start, pfn + off, + vma->vm_end - vma->vm_start, + vma->vm_page_prot); +} + +int qcom_iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + struct page *page; + int ret; + + if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) { + struct page **pages = qcom_dma_common_find_pages(cpu_addr); + + if (pages) { + return sg_alloc_table_from_pages(sgt, pages, + PAGE_ALIGN(size) >> PAGE_SHIFT, + 0, size, GFP_KERNEL); + } + + page = vmalloc_to_page(cpu_addr); + } else { + page = virt_to_page(cpu_addr); + } + + ret = sg_alloc_table(sgt, 1, GFP_KERNEL); + if (!ret) + sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); + return ret; +} + +static int qcom_dma_iommu_probe(struct platform_device *pdev) +{ + int ret; + struct device *dev = &pdev->dev; + + qcom_dma_iommu_dev = dev; + if (dev_is_dma_coherent(dev)) { + dev_err(dev, "Cannot be dma-coherent\n"); + return -EINVAL; + } + + /* Should be connected to linux,cma-default node */ + ret = of_reserved_mem_device_init_by_idx(dev, dev->of_node, 0); + if (ret) + return ret; + + qcom_dma_contiguous_default_area = dev->cma_area; + if (!qcom_dma_contiguous_default_area) { + dev_err(dev, "Unable to find cma area\n"); + return -EINVAL; + } + + ret = dma_atomic_pool_init(dev); + if (ret) + goto out_iova_cache; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) + goto out_atomic_pool; + + probe_finished = true; + return 0; + +out_atomic_pool: + qcom_dma_atomic_pool_exit(dev); + +out_iova_cache: + return ret; +} + +bool qcom_dma_iommu_is_ready(void) +{ + if (!probe_finished) + return false; + return true; +} +EXPORT_SYMBOL(qcom_dma_iommu_is_ready); + +static int qcom_dma_iommu_remove(struct platform_device *pdev) +{ + qcom_dma_atomic_pool_exit(&pdev->dev); + return 0; +} + +static const struct of_device_id qcom_dma_iommu_of_match[] = { + {.compatible = "qcom,iommu-dma"}, + {} +}; +MODULE_DEVICE_TABLE(of, qcom_dma_iommu_of_match); + +static struct platform_driver qcom_dma_iommu_driver = { + .probe = qcom_dma_iommu_probe, + .remove = qcom_dma_iommu_remove, + .driver = { + .name = "qcom_dma_iommu", + .of_match_table = qcom_dma_iommu_of_match, + .suppress_bind_attrs = true, + }, +}; + +int __init qcom_dma_iommu_generic_driver_init(void) +{ + return platform_driver_register(&qcom_dma_iommu_driver); +} + +void qcom_dma_iommu_generic_driver_exit(void) +{ + platform_driver_unregister(&qcom_dma_iommu_driver); +} diff --git a/drivers/iommu/qcom-dma-iommu-generic.h b/drivers/iommu/qcom-dma-iommu-generic.h new file mode 100644 index 000000000000..773215ee7825 --- /dev/null +++ b/drivers/iommu/qcom-dma-iommu-generic.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef __QCOM_DMA_IOMMU_GENERIC_H +#define __QCOM_DMA_IOMMU_GENERIC_H + +#include +#include +#include + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + +bool qcom_dma_iommu_is_ready(void); +extern int __init qcom_dma_iommu_generic_driver_init(void); +extern void qcom_dma_iommu_generic_driver_exit(void); + +struct pci_host_bridge *qcom_pci_find_host_bridge(struct pci_bus *bus); + +void qcom_arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir); +void qcom_arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir); +void qcom_arch_dma_prep_coherent(struct page *page, size_t size); + +/* kernel/dma/contiguous.c */ +struct page *qcom_dma_alloc_from_contiguous(struct device *dev, size_t count, + unsigned int align, bool no_warn); +bool qcom_dma_release_from_contiguous(struct device *dev, struct page *pages, + int count); +struct page *qcom_dma_alloc_contiguous(struct device *dev, size_t size, + gfp_t gfp); +void qcom_dma_free_contiguous(struct device *dev, struct page *page, + size_t size); + +/* kernel/dma/remap.c */ +struct page **qcom_dma_common_find_pages(void *cpu_addr); +void *qcom_dma_common_pages_remap(struct page **pages, size_t size, + pgprot_t prot, const void *caller); +void *qcom_dma_common_contiguous_remap(struct page *page, size_t size, + pgprot_t prot, const void *caller); +void qcom_dma_common_free_remap(void *cpu_addr, size_t size); +void *qcom_dma_alloc_from_pool(struct device *dev, size_t size, + struct page **ret_page, gfp_t flags); +bool qcom_dma_free_from_pool(struct device *dev, void *start, size_t size); + +int qcom_dma_mmap_from_dev_coherent(struct device *dev, + struct vm_area_struct *vma, void *vaddr, size_t size, int *ret); + +/* kernel/dma/mapping.c */ +pgprot_t qcom_dma_pgprot(struct device *dev, pgprot_t prot, + unsigned long attrs); + +/* DMA-IOMMU utilities */ +int qcom_dma_info_to_prot(enum dma_data_direction dir, bool coherent, + unsigned long attrs); +size_t qcom_iommu_dma_prepare_map_sg(struct device *dev, struct iova_domain *iovad, + struct scatterlist *sg, int nents); +int qcom_iommu_dma_finalise_sg(struct device *dev, struct scatterlist *sg, int nents, + dma_addr_t dma_addr); +void qcom_iommu_dma_invalidate_sg(struct scatterlist *sg, int nents); +int qcom_iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs); +int qcom_iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs); + +#else /*CONFIG_IOMMU_IO_PGTABLE_FAST*/ + +static inline bool qcom_dma_iommu_is_ready(void) +{ + return true; +} + +static inline int __init qcom_dma_iommu_generic_driver_init(void) +{ + return 0; +} + +static inline void qcom_dma_iommu_generic_driver_exit(void) {} + +#endif /*CONFIG_IOMMU_IO_PGTABLE_FAST*/ +#endif /* __QCOM_DMA_IOMMU_GENERIC_H */ diff --git a/drivers/iommu/qcom-io-pgtable-alloc.c b/drivers/iommu/qcom-io-pgtable-alloc.c new file mode 100644 index 000000000000..872865d40a47 --- /dev/null +++ b/drivers/iommu/qcom-io-pgtable-alloc.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include + +#include + +struct io_pgtable_pool { + u32 vmid; + struct kref ref; + spinlock_t pool_lock; + struct list_head page_pool; +}; + +static DEFINE_MUTEX(page_pool_xa_lock); +static DEFINE_XARRAY(page_pool_xa); +static atomic_long_t page_pool_count = ATOMIC_LONG_INIT(0); + +static bool is_secure_vmid(u32 vmid) +{ + return !!vmid; +} + +static int io_pgtable_hyp_assign_page(u32 vmid, struct page *page) +{ + u32 src_vmid_list[] = {VMID_HLOS}; + int dst_vmid_list[] = {VMID_HLOS, vmid}; + int dest_perms[] = {PERM_READ | PERM_WRITE, PERM_READ}; + int ret; + + ret = hyp_assign_phys(page_to_phys(page), PAGE_SIZE, src_vmid_list, + ARRAY_SIZE(src_vmid_list), dst_vmid_list, dest_perms, + ARRAY_SIZE(dst_vmid_list)); + + WARN(ret, "failed to assign memory to VMID: %u rc:%d\n", vmid, ret); + return ret ? -EADDRNOTAVAIL : 0; +} + +static int io_pgtable_hyp_unassign_page(u32 vmid, struct page *page) +{ + u32 src_vmid_list[] = {VMID_HLOS, vmid}; + int dst_vmid_list[] = {VMID_HLOS}; + int dest_perms[] = {PERM_READ | PERM_WRITE | PERM_EXEC}; + int ret; + + ret = hyp_assign_phys(page_to_phys(page), PAGE_SIZE, src_vmid_list, + ARRAY_SIZE(src_vmid_list), dst_vmid_list, dest_perms, + ARRAY_SIZE(dst_vmid_list)); + + WARN(ret, "failed to unassign memory from VMID: %u rc: %d\n", vmid, ret); + return ret ? -EADDRNOTAVAIL : 0; +} + +static struct page *__alloc_page_from_pool(struct list_head *page_pool) +{ + struct page *page; + + page = list_first_entry_or_null(page_pool, struct page, lru); + if (page) { + list_del(&page->lru); + atomic_long_dec(&page_pool_count); + dec_node_page_state(page, NR_KERNEL_MISC_RECLAIMABLE); + } + + return page; +} + +static struct page *alloc_page_from_pool(u32 vmid) +{ + struct io_pgtable_pool *pool = xa_load(&page_pool_xa, vmid); + struct page *page; + unsigned long flags; + + spin_lock_irqsave(&pool->pool_lock, flags); + page = __alloc_page_from_pool(&pool->page_pool); + spin_unlock_irqrestore(&pool->pool_lock, flags); + + return page; +} + +static void free_page_to_pool(struct page *page) +{ + u32 vmid = page_private(page); + struct io_pgtable_pool *pool = xa_load(&page_pool_xa, vmid); + unsigned long flags; + + clear_page(page_address(page)); + spin_lock_irqsave(&pool->pool_lock, flags); + list_add(&page->lru, &pool->page_pool); + atomic_long_inc(&page_pool_count); + inc_node_page_state(page, NR_KERNEL_MISC_RECLAIMABLE); + spin_unlock_irqrestore(&pool->pool_lock, flags); +} + +/* Assumes that page_pool_xa_lock is held. */ +static void io_pgtable_pool_release(struct kref *ref) +{ + struct io_pgtable_pool *pool = container_of(ref, struct io_pgtable_pool, ref); + struct page *page; + bool secure_vmid = is_secure_vmid(pool->vmid); + + xa_erase(&page_pool_xa, pool->vmid); + + /* + * There's no need to take the pool lock, as the pool is no longer accessible to other + * IOMMU clients. There's no possibility for concurrent access either as this + * function is only invoked when the last reference is removed. + */ + page = __alloc_page_from_pool(&pool->page_pool); + while (page) { + if (!secure_vmid || !io_pgtable_hyp_unassign_page(pool->vmid, page)) + __free_page(page); + + page = __alloc_page_from_pool(&pool->page_pool); + } + + kfree(pool); +} + +/* + * qcom_io_pgtable_allocator_register: Register with the io-pgtable allocator interface. + * + * @vmid: The VMID that io-pgtable memory needs to be shared with when allocated. If VMID + * is 0, then page table memory will not be shared with any other VMs. + * + * On success, 0 is returned and there will be a reference held for metadata associated with + * @vmid. Otherwise, an error code will be returned. + */ +int qcom_io_pgtable_allocator_register(u32 vmid) +{ + struct io_pgtable_pool *pool; + int ret = 0; + + mutex_lock(&page_pool_xa_lock); + pool = xa_load(&page_pool_xa, vmid); + if (pool) { + kref_get(&pool->ref); + goto out; + } + + pool = kmalloc(sizeof(*pool), GFP_KERNEL); + if (!pool) { + ret = -ENOMEM; + goto out; + } + + pool->vmid = vmid; + kref_init(&pool->ref); + spin_lock_init(&pool->pool_lock); + INIT_LIST_HEAD(&pool->page_pool); + + ret = xa_err(xa_store(&page_pool_xa, vmid, pool, GFP_KERNEL)); + if (ret < 0) + kfree(pool); +out: + mutex_unlock(&page_pool_xa_lock); + return ret; +} + +/* + * qcom_io_pgtable_allocator_unregister: Unregister with the io-pgtable allocator interface. + * + * @vmid: The VMID that was used when registering with the interface with + * qcom_io_pgtable_allocator_register(). + * + * Decrements the references to allocator metadata for @vmid. + * + * If this call results in references to @vmid dropping to 0, then all metadata and pages + * associated with @vmid are released. + */ +void qcom_io_pgtable_allocator_unregister(u32 vmid) +{ + struct io_pgtable_pool *pool; + + mutex_lock(&page_pool_xa_lock); + pool = xa_load(&page_pool_xa, vmid); + kref_put(&pool->ref, io_pgtable_pool_release); + mutex_unlock(&page_pool_xa_lock); +} + +/* + * qcom_io_pgtable_alloc_page: Allocate page table memory from the io-pgtable allocator. + * + * @vmid: The VMID that the page table memory should be shared with. + * @gfp: The GFP flags to be used for allocating the page table memory. + * + * This function may sleep if memory needs to be shared with other VMs. + * + * On success, a page will be returned. The page will also have been shared with other + * VMs--if any. In case of an error, this function returns NULL. + */ +struct page *qcom_io_pgtable_alloc_page(u32 vmid, gfp_t gfp) +{ + struct page *page; + + /* + * Mapping memory for secure domains may result in having to assign page table + * memory to another VMID, which can sleep. Atomic and secure domains are + * not a legal combination. We can use the GFP flags to detect atomic domains, + * as they will have GFP_ATOMIC set. + */ + BUG_ON(!gfpflags_allow_blocking(gfp) && is_secure_vmid(vmid)); + + page = alloc_page_from_pool(vmid); + if (page) + return page; + + page = alloc_page(gfp); + if (!page) + return NULL; + /* The page may be inaccessible if this is true, so leak it. */ + else if (is_secure_vmid(vmid) && io_pgtable_hyp_assign_page(vmid, page)) + return NULL; + + set_page_private(page, (unsigned long)vmid); + return page; +} + +/* + * qcom_io_pgtable_free_page: Frees page table memory. + * + * @page: The page to be freed. + * + * We cache pages in their respective page pools to improve performance + * for future allocations. + * + * Export this symbol for the IOMMU driver, since it decides when + * page table memory is freed after TLB maintenance. + */ +void qcom_io_pgtable_free_page(struct page *page) +{ + free_page_to_pool(page); +} +EXPORT_SYMBOL(qcom_io_pgtable_free_page); + +static unsigned long io_pgtable_alloc_count_objects(struct shrinker *shrinker, + struct shrink_control *sc) +{ + unsigned long count = atomic_long_read(&page_pool_count); + + return count ? count : SHRINK_EMPTY; +} + +static unsigned long scan_page_pool(struct io_pgtable_pool *pool, struct list_head *freelist, + unsigned long nr_to_scan) +{ + struct page *page; + unsigned long count = 0, flags; + + spin_lock_irqsave(&pool->pool_lock, flags); + while (count < nr_to_scan) { + page = __alloc_page_from_pool(&pool->page_pool); + if (page) { + list_add(&page->lru, freelist); + count++; + } else { + break; + } + } + spin_unlock_irqrestore(&pool->pool_lock, flags); + + return count; +} + +static unsigned long io_pgtable_alloc_scan_objects(struct shrinker *shrinker, + struct shrink_control *sc) +{ + struct page *page, *tmp; + struct io_pgtable_pool *pool; + unsigned long index; + unsigned long nr_to_scan = sc->nr_to_scan, count = 0; + u32 vmid; + LIST_HEAD(freelist); + + mutex_lock(&page_pool_xa_lock); + xa_for_each(&page_pool_xa, index, pool) { + count += scan_page_pool(pool, &freelist, nr_to_scan - count); + if (count >= nr_to_scan) + break; + } + mutex_unlock(&page_pool_xa_lock); + + list_for_each_entry_safe(page, tmp, &freelist, lru) { + vmid = page_private(page); + list_del(&page->lru); + + if (!is_secure_vmid(vmid) || !io_pgtable_hyp_unassign_page(vmid, page)) + __free_page(page); + else + count--; + } + + return count; +} + +static struct shrinker io_pgtable_alloc_shrinker = { + .count_objects = io_pgtable_alloc_count_objects, + .scan_objects = io_pgtable_alloc_scan_objects, + .seeks = DEFAULT_SEEKS, +}; + +int qcom_io_pgtable_alloc_init(void) +{ + return register_shrinker(&io_pgtable_alloc_shrinker); +} + +void qcom_io_pgtable_alloc_exit(void) +{ + unregister_shrinker(&io_pgtable_alloc_shrinker); +} diff --git a/drivers/iommu/qcom-io-pgtable-alloc.h b/drivers/iommu/qcom-io-pgtable-alloc.h new file mode 100644 index 000000000000..f900fa2d15f8 --- /dev/null +++ b/drivers/iommu/qcom-io-pgtable-alloc.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. All rights reserved. + */ +#ifndef __QCOM_IO_PGTABLE_ALLOC_H +#define __QCOM_IO_PGTABLE_ALLOC_H + +int qcom_io_pgtable_allocator_register(u32 vmid); +void qcom_io_pgtable_allocator_unregister(u32 vmid); +struct page *qcom_io_pgtable_alloc_page(u32 vmid, gfp_t gfp); +void qcom_io_pgtable_free_page(struct page *page); +int qcom_io_pgtable_alloc_init(void); +void qcom_io_pgtable_alloc_exit(void); + +#endif /* __QCOM_IO_PGTABLE_ALLOC_H */ + diff --git a/drivers/iommu/qcom-io-pgtable-arm.c b/drivers/iommu/qcom-io-pgtable-arm.c new file mode 100644 index 000000000000..a9144d756d6a --- /dev/null +++ b/drivers/iommu/qcom-io-pgtable-arm.c @@ -0,0 +1,1702 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CPU-agnostic ARM page table allocator. + * + * Copyright (C) 2014 ARM Limited + * + * Author: Will Deacon + * + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "qcom-io-pgtable.h" +#include "qcom-io-pgtable-alloc.h" + +#include "io-pgtable-arm.h" + +#define ARM_LPAE_MAX_ADDR_BITS 52 +#define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 +#define ARM_LPAE_MAX_LEVELS 4 + +/* Struct accessors */ +#define io_pgtable_to_data(x) \ + container_of((x), struct arm_lpae_io_pgtable, iop) + +#define io_pgtable_ops_to_data(x) \ + io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) + +/* + * Calculate the right shift amount to get to the portion describing level l + * in a virtual address mapped by the pagetable in d. + */ +#define ARM_LPAE_LVL_SHIFT(l, d) \ + (((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \ + ilog2(sizeof(arm_lpae_iopte))) + +#define ARM_LPAE_GRANULE(d) \ + (sizeof(arm_lpae_iopte) << (d)->bits_per_level) +#define ARM_LPAE_PGD_SIZE(d) \ + (sizeof(arm_lpae_iopte) << (d)->pgd_bits) + +#define ARM_LPAE_PTES_PER_TABLE(d) \ + (ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte))) + +/* + * Calculate the index at level l used to map virtual address a using the + * pagetable in d. + */ +#define ARM_LPAE_PGD_IDX(l, d) \ + ((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0) + +#define ARM_LPAE_LVL_IDX(a, l, d) \ + (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l, d)) & \ + ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l, d))) - 1)) + +/* Calculate the block/page mapping size at level l for pagetable in d. */ +#define ARM_LPAE_BLOCK_SIZE(l, d) (1ULL << ARM_LPAE_LVL_SHIFT(l, d)) + +/* Page table bits */ +#define ARM_LPAE_PTE_TYPE_SHIFT 0 +#define ARM_LPAE_PTE_TYPE_MASK 0x3 + +#define ARM_LPAE_PTE_TYPE_BLOCK 1 +#define ARM_LPAE_PTE_TYPE_TABLE 3 +#define ARM_LPAE_PTE_TYPE_PAGE 3 + +#define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47, 12) + +#define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) +#define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) +#define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) +#define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8) +#define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8) +#define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8) +#define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5) +#define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0) + +#define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2) +/* Ignore the contiguous bit for block splitting */ +#define ARM_LPAE_PTE_ATTR_HI_MASK (((arm_lpae_iopte)6) << 52) +#define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK | \ + ARM_LPAE_PTE_ATTR_HI_MASK) +/* Software bit for solving coherency races */ +#define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55) + +/* Stage-1 PTE */ +#define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) +#define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6) +#define ARM_LPAE_PTE_ATTRINDX_SHIFT 2 +#define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11) + +/* Stage-2 PTE */ +#define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6) +#define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6) +#define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6) +#define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2) +#define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2) +#define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2) + +/* Register bits */ +#define ARM_LPAE_VTCR_SL0_MASK 0x3 + +#define ARM_LPAE_TCR_T0SZ_SHIFT 0 + +#define ARM_LPAE_VTCR_PS_SHIFT 16 +#define ARM_LPAE_VTCR_PS_MASK 0x7 + +#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) +#define ARM_LPAE_MAIR_ATTR_MASK 0xff +#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04ULL +#define ARM_LPAE_MAIR_ATTR_NC 0x44ULL +#define ARM_LPAE_MAIR_ATTR_INC_OWBRANWA 0xe4ULL +#define ARM_LPAE_MAIR_ATTR_IWBRWA_OWBRANWA 0xefULL +#define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4ULL +#define ARM_LPAE_MAIR_ATTR_WBRWA 0xffULL +#define ARM_LPAE_MAIR_ATTR_IDX_NC 0 +#define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 +#define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 +#define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3 +#define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE_NWA 4 +#define ARM_LPAE_MAIR_ATTR_IDX_ICACHE_OCACHE_NWA 5 + +#define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0) +#define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2) +#define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4) + +#define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL +#define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL + +/* IOPTE accessors */ +#define iopte_deref(pte, d) __va(iopte_to_paddr(pte, d)) + +#define iopte_type(pte) \ + (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) + +#define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK) + +struct arm_lpae_io_pgtable { + struct io_pgtable iop; + + int pgd_bits; + int start_level; + int bits_per_level; + + void *pgd; + u32 vmid; + const struct qcom_iommu_flush_ops *iommu_tlb_ops; + const struct qcom_iommu_pgtable_log_ops *pgtable_log_ops; + /* Protects table refcounts */ + spinlock_t lock; +}; + +typedef u64 arm_lpae_iopte; + +/* + * We'll use some ignored bits in table entries to keep track of the number + * of page mappings beneath the table. The maximum number of entries + * beneath any table mapping in armv8 is 8192 (which is possible at the + * 2nd and 3rd level when using a 64K granule size). The bits at our + * disposal are: + * + * 4k granule: [54..52], [11..2] + * 64k granule: [54..52], [15..2] + * + * [54..52], [11..2] is enough bits for tracking table mappings at any + * level for any granule, so we'll use those. + * + * If iopte_tblcnt reaches zero for a last-level pagetable, the pagetable + * will be freed. + */ +#define BOTTOM_IGNORED_MASK GENMASK(11, 2) +#define TOP_IGNORED_MASK GENMASK_ULL(54, 52) +#define IOPTE_RESERVED_MASK (TOP_IGNORED_MASK | BOTTOM_IGNORED_MASK) +#define BOTTOM_VAL_BITS GENMASK(9, 0) +#define TOP_VAL_BITS GENMASK(12, 10) + +static arm_lpae_iopte iopte_val(arm_lpae_iopte table_pte) +{ + return table_pte & ~IOPTE_RESERVED_MASK; +} + +static int iopte_tblcnt(arm_lpae_iopte table_pte) +{ + int top_cnt = FIELD_GET(TOP_IGNORED_MASK, table_pte); + int bottom_cnt = FIELD_GET(BOTTOM_IGNORED_MASK, table_pte); + + return FIELD_PREP(TOP_VAL_BITS, top_cnt) | + FIELD_PREP(BOTTOM_VAL_BITS, bottom_cnt); +} + +static void iopte_tblcnt_set(arm_lpae_iopte *table_pte, int val) +{ + arm_lpae_iopte pte = iopte_val(*table_pte); + int top_val = FIELD_GET(TOP_VAL_BITS, val); + int bottom_val = FIELD_GET(BOTTOM_VAL_BITS, val); + + pte |= FIELD_PREP(TOP_IGNORED_MASK, top_val) | + FIELD_PREP(BOTTOM_IGNORED_MASK, bottom_val); + *table_pte = pte; +} + +static void iopte_tblcnt_sub(arm_lpae_iopte *table_ptep, int cnt) +{ + int current_cnt = iopte_tblcnt(*table_ptep); + + current_cnt -= cnt; + iopte_tblcnt_set(table_ptep, current_cnt); +} + +static void iopte_tblcnt_add(arm_lpae_iopte *table_ptep, int cnt) +{ + int current_cnt = iopte_tblcnt(*table_ptep); + + current_cnt += cnt; + iopte_tblcnt_set(table_ptep, current_cnt); +} + +static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl, + enum io_pgtable_fmt fmt) +{ + if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE) + return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE; + + return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK; +} + +static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr, + struct arm_lpae_io_pgtable *data) +{ + arm_lpae_iopte pte = paddr; + + /* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */ + return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; +} + +static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte, + struct arm_lpae_io_pgtable *data) +{ + u64 paddr = iopte_val(pte) & ARM_LPAE_PTE_ADDR_MASK; + + if (ARM_LPAE_GRANULE(data) < SZ_64K) + return paddr; + + /* Rotate the packed high-order bits back to the top */ + return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); +} + +static bool selftest_running; + +static dma_addr_t __arm_lpae_dma_addr(void *pages) +{ + return (dma_addr_t)virt_to_phys(pages); +} + +static void *__arm_lpae_alloc_pages(struct arm_lpae_io_pgtable *data, + size_t size, gfp_t gfp, + struct io_pgtable_cfg *cfg, void *cookie) +{ + struct device *dev = cfg->iommu_dev; + dma_addr_t dma; + struct page *p; + void *pages; + + VM_BUG_ON((gfp & __GFP_HIGHMEM)); + p = qcom_io_pgtable_alloc_page(data->vmid, gfp | __GFP_ZERO); + if (!p) + return NULL; + + pages = page_address(p); + if (!cfg->coherent_walk) { + dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma)) + goto out_free; + /* + * We depend on the IOMMU being able to work with any physical + * address directly, so if the DMA layer suggests otherwise by + * translating or truncating them, that bodes very badly... + */ + if (dma != virt_to_phys(pages)) + goto out_unmap; + } + + return pages; + +out_unmap: + dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); + dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); +out_free: + qcom_io_pgtable_free_page(p); + return NULL; +} + +static void __arm_lpae_free_pages(struct arm_lpae_io_pgtable *data, + void *pages, size_t size, + struct io_pgtable_cfg *cfg, void *cookie, + bool deferred_free) +{ + if (!cfg->coherent_walk) + dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), + size, DMA_TO_DEVICE); + + if (deferred_free) + qcom_io_pgtable_tlb_add_walk_page(data->iommu_tlb_ops, cookie, pages); + else + qcom_io_pgtable_free_page(virt_to_page(pages)); +} + +static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, + struct io_pgtable_cfg *cfg) +{ + dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), + sizeof(*ptep) * num_entries, DMA_TO_DEVICE); +} + +static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, + int num_entries, struct io_pgtable_cfg *cfg) +{ + int i; + + for (i = 0; i < num_entries; i++) + ptep[i] = pte; + + if (!cfg->coherent_walk) + __arm_lpae_sync_pte(ptep, num_entries, cfg); +} + +static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, + struct iommu_iotlb_gather *gather, + unsigned long iova, size_t size, size_t pgcount, + int lvl, arm_lpae_iopte *ptep); + +static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, + phys_addr_t paddr, arm_lpae_iopte prot, + int lvl, int num_entries, arm_lpae_iopte *ptep, + bool flush) +{ + arm_lpae_iopte pte = prot; + size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + int i; + + if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1) + pte |= ARM_LPAE_PTE_TYPE_PAGE; + else + pte |= ARM_LPAE_PTE_TYPE_BLOCK; + + for (i = 0; i < num_entries; i++) + ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data); + + if (flush && !cfg->coherent_walk) + __arm_lpae_sync_pte(ptep, num_entries, cfg); +} + +static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, + unsigned long iova, phys_addr_t paddr, + arm_lpae_iopte prot, int lvl, int num_entries, + arm_lpae_iopte *ptep, arm_lpae_iopte *prev_ptep, + bool flush) +{ + int i; + + for (i = 0; i < num_entries; i++) + if (ptep[i] & ARM_LPAE_PTE_VALID) { + /* We require an unmap first */ + WARN_ON(!selftest_running); + return -EEXIST; + } + + __arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep, flush); + + if (prev_ptep) + iopte_tblcnt_add(prev_ptep, num_entries); + + return 0; +} + +static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, + arm_lpae_iopte *ptep, + arm_lpae_iopte curr, + struct io_pgtable_cfg *cfg, + int refcount, + struct arm_lpae_io_pgtable *data) +{ + arm_lpae_iopte old, new; + + /* Due to tlb maintenance in unmap being deferred */ + qcom_io_pgtable_tlb_sync(data->iommu_tlb_ops, data->iop.cookie); + + new = __pa(table) | ARM_LPAE_PTE_TYPE_TABLE; + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) + new |= ARM_LPAE_PTE_NSTABLE; + iopte_tblcnt_set(&new, refcount); + + /* + * Ensure the table itself is visible before its PTE can be. + * Whilst we could get away with cmpxchg64_release below, this + * doesn't have any ordering semantics when !CONFIG_SMP. + */ + dma_wmb(); + + old = cmpxchg64_relaxed(ptep, curr, new); + + if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC)) + return old; + + /* Even if it's not ours, there's no point waiting; just kick it */ + __arm_lpae_sync_pte(ptep, 1, cfg); + if (old == curr) + WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC); + + return old; +} + +struct map_state { + unsigned long iova_end; + unsigned int pgsize; + arm_lpae_iopte *pgtable; + arm_lpae_iopte *prev_pgtable; + arm_lpae_iopte *pte_start; + unsigned int num_pte; +}; +/* map state optimization works at level 3 */ +#define MAP_STATE_LVL 3 + +static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, + phys_addr_t paddr, size_t size, size_t pgcount, + arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep, + arm_lpae_iopte *prev_ptep, struct map_state *ms, + gfp_t gfp, unsigned long *flags, size_t *mapped) +{ + arm_lpae_iopte *cptep, pte; + size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); + size_t prev_block_size; + size_t tblsz = ARM_LPAE_GRANULE(data); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + void *cookie = data->iop.cookie; + arm_lpae_iopte *pgtable = ptep; + int ret = 0, num_entries, max_entries, map_idx_start; + + /* Find our entry at the current level */ + map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); + ptep += map_idx_start; + + /* If we can install a leaf entry at this level, then do so */ + if (size == block_size) { + if (!ms) { + max_entries = ARM_LPAE_PTES_PER_TABLE(data) - map_idx_start; + num_entries = min_t(int, pgcount, max_entries); + ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, + num_entries, ptep, prev_ptep, true); + if (!ret && mapped) + *mapped += size * num_entries; + + return ret; + } + + if (lvl == MAP_STATE_LVL) { + prev_block_size = ARM_LPAE_BLOCK_SIZE(lvl - 1, data); + + if (ms->pgtable && !cfg->coherent_walk) + dma_sync_single_for_device(cfg->iommu_dev, + __arm_lpae_dma_addr(ms->pte_start), + ms->num_pte * sizeof(*ptep), + DMA_TO_DEVICE); + + ms->iova_end = round_down(iova, prev_block_size) + prev_block_size; + ms->pgtable = pgtable; + ms->prev_pgtable = prev_ptep; + ms->pgsize = size; + ms->pte_start = ptep; + ms->num_pte = 1; + } else { + /* + * We have some map state from previous page mappings, + * but we're about to set up a block mapping. Flush + * out the previous page mappings. + */ + if (ms->pgtable && !cfg->coherent_walk) + dma_sync_single_for_device(cfg->iommu_dev, + __arm_lpae_dma_addr(ms->pte_start), + ms->num_pte * sizeof(*ptep), + DMA_TO_DEVICE); + memset(ms, 0, sizeof(*ms)); + ms = NULL; + } + + ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, 1, ptep, prev_ptep, + ms == NULL); + if (!ret && mapped) + *mapped += size; + + return ret; + } + + /* We can't allocate tables at the final level */ + if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) + return -EINVAL; + + /* Grab a pointer to the next level */ + pte = READ_ONCE(*ptep); + if (!pte) { + /* + * Drop the lock in order to support GFP_KERNEL. + * + * Only last level pagetables will be freed if iotlb_tblcnt reaches 0. + * Since we are installing a new pagetable, the current pagetable cannot + * be the last level. So we can assume ptep is still a valid memory location + * after reaquiring the lock. + */ + spin_unlock_irqrestore(&data->lock, *flags); + cptep = __arm_lpae_alloc_pages(data, tblsz, gfp, cfg, cookie); + spin_lock_irqsave(&data->lock, *flags); + if (!cptep) + return -ENOMEM; + + pte = arm_lpae_install_table(cptep, ptep, 0, cfg, 0, data); + if (pte) + __arm_lpae_free_pages(data, cptep, tblsz, cfg, cookie, false); + else + qcom_io_pgtable_log_new_table(data->pgtable_log_ops, + data->iop.cookie, cptep, + iova & ~(block_size - 1), + block_size); + } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) { + __arm_lpae_sync_pte(ptep, 1, cfg); + } + + if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) { + cptep = iopte_deref(pte, data); + } else if (pte) { + /* We require an unmap first */ + WARN_ON(!selftest_running); + return -EEXIST; + } + + /* Rinse, repeat */ + return __arm_lpae_map(data, iova, paddr, size, pgcount, prot, lvl + 1, + cptep, ptep, ms, gfp, flags, mapped); +} + +static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, + int prot) +{ + arm_lpae_iopte pte; + + if (data->iop.fmt == QCOM_ARM_64_LPAE_S1 || + data->iop.fmt == ARM_32_LPAE_S1) { + pte = ARM_LPAE_PTE_nG; + if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) + pte |= ARM_LPAE_PTE_AP_RDONLY; + if (!(prot & IOMMU_PRIV)) + pte |= ARM_LPAE_PTE_AP_UNPRIV; + } else { + pte = ARM_LPAE_PTE_HAP_FAULT; + if (prot & IOMMU_READ) + pte |= ARM_LPAE_PTE_HAP_READ; + if (prot & IOMMU_WRITE) + pte |= ARM_LPAE_PTE_HAP_WRITE; + } + + /* + * Note that this logic is structured to accommodate Mali LPAE + * having stage-1-like attributes but stage-2-like permissions. + */ + if (data->iop.fmt == ARM_64_LPAE_S2 || + data->iop.fmt == ARM_32_LPAE_S2) { + if (prot & IOMMU_MMIO) + pte |= ARM_LPAE_PTE_MEMATTR_DEV; + else if (prot & IOMMU_CACHE) + pte |= ARM_LPAE_PTE_MEMATTR_OIWB; + else + pte |= ARM_LPAE_PTE_MEMATTR_NC; + } else { + if (prot & IOMMU_MMIO) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV + << ARM_LPAE_PTE_ATTRINDX_SHIFT); + else if ((prot & IOMMU_CACHE) && (prot & IOMMU_SYS_CACHE_NWA)) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_ICACHE_OCACHE_NWA + << ARM_LPAE_PTE_ATTRINDX_SHIFT); + /* IOMMU_CACHE + IOMMU_SYS_CACHE equivalent to IOMMU_CACHE */ + else if (prot & IOMMU_CACHE) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE + << ARM_LPAE_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_SYS_CACHE) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE + << ARM_LPAE_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_SYS_CACHE_NWA) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE_NWA + << ARM_LPAE_PTE_ATTRINDX_SHIFT); + } + + if (prot & IOMMU_CACHE) + pte |= ARM_LPAE_PTE_SH_IS; + else + pte |= ARM_LPAE_PTE_SH_OS; + + if (prot & IOMMU_NOEXEC) + pte |= ARM_LPAE_PTE_XN; + + if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) + pte |= ARM_LPAE_PTE_NS; + + if (data->iop.fmt != ARM_MALI_LPAE) + pte |= ARM_LPAE_PTE_AF; + + return pte; +} + +static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int iommu_prot, gfp_t gfp, size_t *mapped) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + arm_lpae_iopte *ptep = data->pgd; + int ret, lvl = data->start_level; + arm_lpae_iopte prot; + long iaext = (s64)iova >> cfg->ias; + unsigned long flags; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) + return -EINVAL; + + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) + iaext = ~iaext; + if (WARN_ON(iaext || paddr >> cfg->oas)) + return -ERANGE; + + /* If no access, then nothing to do */ + if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) + return 0; + + prot = arm_lpae_prot_to_pte(data, iommu_prot); + + spin_lock_irqsave(&data->lock, flags); + ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl, + ptep, NULL, NULL, gfp, &flags, mapped); + spin_unlock_irqrestore(&data->lock, flags); + /* + * Synchronise all PTE updates for the new mapping before there's + * a chance for anything to kick off a table walk for the new iova. + */ + wmb(); + + return ret; +} + +static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int iommu_prot, gfp_t gfp) +{ + return arm_lpae_map_pages(ops, iova, paddr, size, 1, iommu_prot, gfp, + NULL); +} + +static size_t arm_lpae_pgsize(unsigned long pgsize_bitmap, unsigned long addr_merge, + size_t size) +{ + unsigned int pgsize_idx, align_pgsize_idx; + size_t pgsize; + + /* Max page size that still fits into 'size' */ + pgsize_idx = __fls(size); + + /* need to consider alignment requirements ? */ + if (likely(addr_merge)) { + /* Max page size allowed by address */ + align_pgsize_idx = __ffs(addr_merge); + pgsize_idx = min(pgsize_idx, align_pgsize_idx); + } + + /* build a mask of acceptable page sizes */ + pgsize = (1UL << (pgsize_idx + 1)) - 1; + + /* throw away page sizes not supported by the hardware */ + pgsize &= pgsize_bitmap; + + /* make sure we're still sane */ + BUG_ON(!pgsize); + + /* pick the biggest page */ + pgsize_idx = __fls(pgsize); + pgsize = 1UL << pgsize_idx; + + return pgsize; +} + +static int arm_lpae_map_by_pgsize(struct io_pgtable_ops *ops, + unsigned long iova, phys_addr_t paddr, + size_t size, int iommu_prot, gfp_t gfp, + size_t *mapped, struct map_state *ms, + unsigned long *flags) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + arm_lpae_iopte *ptep = data->pgd, *ms_ptep; + int ret, lvl = data->start_level; + arm_lpae_iopte prot = arm_lpae_prot_to_pte(data, iommu_prot); + unsigned int min_pagesz = 1 << __ffs(cfg->pgsize_bitmap); + long iaext = (s64)(iova + size - 1) >> cfg->ias; + size_t pgsize; + + if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { + pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", + iova, &paddr, size, min_pagesz); + return -EINVAL; + } + + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) + iaext = ~iaext; + if (WARN_ON(iaext || (paddr + size - 1) >> cfg->oas)) + return -ERANGE; + + /* If no access, then nothing to do */ + if (!(prot & (IOMMU_READ | IOMMU_WRITE))) { + /* Increment 'mapped' so that the IOVA can be incremented accordingly. */ + *mapped += size; + return 0; + } + + while (size) { + pgsize = arm_lpae_pgsize(cfg->pgsize_bitmap, iova | paddr, size); + + if (ms->pgtable && (iova < ms->iova_end)) { + ms_ptep = ms->pgtable + ARM_LPAE_LVL_IDX(iova, MAP_STATE_LVL, data); + arm_lpae_init_pte(data, iova, paddr, prot, MAP_STATE_LVL, + 1, ms_ptep, ms->prev_pgtable, false); + ms->num_pte++; + } else { + ret = __arm_lpae_map(data, iova, paddr, pgsize, 1, + prot, lvl, ptep, NULL, ms, gfp, + flags, NULL); + if (ret) + return ret; + } + + iova += pgsize; + paddr += pgsize; + *mapped += pgsize; + size -= pgsize; + } + + return 0; +} + +static int __maybe_unused arm_lpae_map_sg(struct io_pgtable_ops *ops, unsigned long iova, + struct scatterlist *sg, unsigned int nents, int prot, + gfp_t gfp, size_t *mapped) +{ + size_t len = 0; + unsigned int i = 0; + int ret; + phys_addr_t start; + struct map_state ms = {}; + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + unsigned long flags; + + *mapped = 0; + + spin_lock_irqsave(&data->lock, flags); + while (i <= nents) { + phys_addr_t s_phys = sg_phys(sg); + + if (len && s_phys != start + len) { + ret = arm_lpae_map_by_pgsize(ops, iova + *mapped, start, + len, prot, gfp, mapped, &ms, &flags); + + if (ret) { + spin_unlock_irqrestore(&data->lock, flags); + return ret; + } + + len = 0; + } + + if (len) { + len += sg->length; + } else { + len = sg->length; + start = s_phys; + } + + if (++i < nents) + sg = sg_next(sg); + } + spin_unlock_irqrestore(&data->lock, flags); + + if (ms.pgtable && !cfg->coherent_walk) + dma_sync_single_for_device(cfg->iommu_dev, + __arm_lpae_dma_addr(ms.pte_start), + ms.num_pte * sizeof(*ms.pte_start), + DMA_TO_DEVICE); + + /* + * Synchronise all PTE updates for the new mapping before there's + * a chance for anything to kick off a table walk for the new iova. + */ + wmb(); + + return 0; +} + +static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, + arm_lpae_iopte *ptep, bool deferred_free) +{ + arm_lpae_iopte *start, *end; + unsigned long table_size; + void *cookie = data->iop.cookie; + + if (lvl == data->start_level) + table_size = ARM_LPAE_PGD_SIZE(data); + else + table_size = ARM_LPAE_GRANULE(data); + + start = ptep; + + /* Only leaf entries at the last level */ + if (lvl == ARM_LPAE_MAX_LEVELS - 1) + end = ptep; + else + end = (void *)ptep + table_size; + + while (ptep != end) { + arm_lpae_iopte pte = *ptep++; + + if (!pte || iopte_leaf(pte, lvl, data->iop.fmt)) + continue; + + __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data), + deferred_free); + } + + __arm_lpae_free_pages(data, start, table_size, &data->iop.cfg, cookie, + deferred_free); + + qcom_io_pgtable_log_remove_table(data->pgtable_log_ops, + data->iop.cookie, start, + 0, /* iova unknown */ + ARM_LPAE_BLOCK_SIZE(lvl - 1, data)); +} + +static void arm_lpae_free_pgtable(struct io_pgtable *iop) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); + + __arm_lpae_free_pgtable(data, data->start_level, data->pgd, false); + qcom_io_pgtable_allocator_unregister(data->vmid); + kfree(data); +} + +static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, + struct iommu_iotlb_gather *gather, + unsigned long iova, size_t size, + arm_lpae_iopte blk_pte, int lvl, + arm_lpae_iopte *ptep, size_t pgcount) +{ + struct io_pgtable_cfg *cfg = &data->iop.cfg; + arm_lpae_iopte pte, *tablep; + phys_addr_t blk_paddr; + size_t tablesz = ARM_LPAE_GRANULE(data); + size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data); + int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data); + int i, unmap_idx_start = -1, num_entries = 0, max_entries; + void *cookie = data->iop.cookie; + int child_cnt = 0; + + if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) + return 0; + + tablep = __arm_lpae_alloc_pages(data, tablesz, GFP_ATOMIC, cfg, cookie); + if (!tablep) + return 0; /* Bytes unmapped */ + + if (size == split_sz) { + unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); + max_entries = ptes_per_table - unmap_idx_start; + num_entries = min_t(int, pgcount, max_entries); + } + + blk_paddr = iopte_to_paddr(blk_pte, data); + pte = iopte_prot(blk_pte); + + for (i = 0; i < ptes_per_table; i++, blk_paddr += split_sz) { + /* Unmap! */ + if (i >= unmap_idx_start && i < (unmap_idx_start + num_entries)) + continue; + + __arm_lpae_init_pte(data, blk_paddr, pte, lvl, 1, &tablep[i], true); + child_cnt++; + } + + pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg, child_cnt, data); + if (pte != blk_pte) { + __arm_lpae_free_pages(data, tablep, tablesz, cfg, cookie, false); + /* + * We may race against someone unmapping another part of this + * block, but anything else is invalid. We can't misinterpret + * a page entry here since we're never at the last level. + */ + if (iopte_type(pte) != ARM_LPAE_PTE_TYPE_TABLE) + return 0; + + tablep = iopte_deref(pte, data); + } else if (unmap_idx_start >= 0) { + /* + * note lvl + 1 due to split_sz above. + * add 0xDEA as flag since split_block_unmap shouldn't ever be called + */ + size_t prev_block_size = ARM_LPAE_BLOCK_SIZE(lvl + 1, data); + + qcom_io_pgtable_log_new_table(data->pgtable_log_ops, + data->iop.cookie, tablep, + iova & ~(prev_block_size - 1) + 0xDEA, + prev_block_size); + return num_entries * size; + } + + return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl, tablep); +} + +static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, + struct iommu_iotlb_gather *gather, + unsigned long iova, size_t size, size_t pgcount, + int lvl, arm_lpae_iopte *ptep) +{ + arm_lpae_iopte pte; + struct io_pgtable *iop = &data->iop; + int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data); + int i = 0, num_entries, max_entries, unmap_idx_start; + + /* Something went horribly wrong and we ran out of page table */ + if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) + return 0; + + unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); + ptep += unmap_idx_start; + pte = READ_ONCE(*ptep); + if (WARN_ON(!pte)) + return 0; + + /* If the size matches this level, we're in the right place */ + if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { + max_entries = ptes_per_table - unmap_idx_start; + num_entries = min_t(int, pgcount, max_entries); + + while (i < num_entries) { + pte = READ_ONCE(*ptep); + if (WARN_ON(!pte)) + break; + + __arm_lpae_set_pte(ptep, 0, 1, &iop->cfg); + + if (!iopte_leaf(pte, lvl, iop->fmt)) { + __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data), + true); + } else if (!iommu_iotlb_gather_queued(gather)) { + /* + * Order the PTE update against queueing the IOVA, to + * guarantee that a flush callback from a different CPU + * has observed it before the TLBIALL can be issued. + */ + smp_wmb(); + } + + ptep++; + i++; + } + + return i * size; + } else if ((lvl == ARM_LPAE_MAX_LEVELS - 2) && !iopte_leaf(pte, lvl, + iop->fmt)) { + arm_lpae_iopte *table; + arm_lpae_iopte *entry; + + table = iopte_deref(pte, data); + unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl + 1, data); + entry = table + unmap_idx_start; + + max_entries = ptes_per_table - unmap_idx_start; + num_entries = min_t(int, pgcount, max_entries); + __arm_lpae_set_pte(entry, 0, num_entries, &iop->cfg); + + iopte_tblcnt_sub(ptep, num_entries); + if (!iopte_tblcnt(*ptep)) { + size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); + /* + * no valid mappings left under this table. + * Defer table free until after iommu_iotlb_sync, or + * qcom_io_pgtable_tlb_sync, whichever occurs first. + */ + __arm_lpae_set_pte(ptep, 0, 1, &iop->cfg); + __arm_lpae_free_pgtable(data, lvl + 1, table, true); + + qcom_io_pgtable_log_remove_table(data->pgtable_log_ops, + data->iop.cookie, table, + iova & ~(block_size - 1), + block_size); + } + + return num_entries * size; + } else if (iopte_leaf(pte, lvl, iop->fmt)) { + /* + * Insert a table at the next level to map the old region, + * minus the part we want to unmap + */ + return arm_lpae_split_blk_unmap(data, gather, iova, size, pte, + lvl + 1, ptep, pgcount); + } + + /* Keep on walkin' */ + ptep = iopte_deref(pte, data); + return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep); +} + +static size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, + size_t pgsize, size_t pgcount, + struct iommu_iotlb_gather *gather) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + arm_lpae_iopte *ptep = data->pgd; + long iaext = (s64)iova >> cfg->ias; + size_t unmapped; + unsigned long flags; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) + return 0; + + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) + iaext = ~iaext; + if (WARN_ON(iaext)) + return 0; + + spin_lock_irqsave(&data->lock, flags); + unmapped = __arm_lpae_unmap(data, gather, iova, pgsize, pgcount, + data->start_level, ptep); + qcom_io_pgtable_tlb_add_inv(data->iommu_tlb_ops, data->iop.cookie); + spin_unlock_irqrestore(&data->lock, flags); + + return unmapped; +} + +static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, + size_t size, struct iommu_iotlb_gather *gather) +{ + return arm_lpae_unmap_pages(ops, iova, size, 1, gather); +} + +static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + arm_lpae_iopte pte, *ptep = data->pgd; + int lvl = data->start_level; + unsigned long flags; + + spin_lock_irqsave(&data->lock, flags); + do { + /* Valid IOPTE pointer? */ + if (!ptep) + goto err; + + /* Grab the IOPTE we're interested in */ + ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); + pte = READ_ONCE(*ptep); + + /* Valid entry? */ + if (!pte) + goto err; + + /* Leaf entry? */ + if (iopte_leaf(pte, lvl, data->iop.fmt)) + goto found_translation; + + /* Take it to the next level */ + ptep = iopte_deref(pte, data); + } while (++lvl < ARM_LPAE_MAX_LEVELS); + + /* Ran out of page tables to walk */ +err: + spin_unlock_irqrestore(&data->lock, flags); + return 0; + +found_translation: + spin_unlock_irqrestore(&data->lock, flags); + iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1); + return iopte_to_paddr(pte, data) | iova; +} + +static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) +{ + unsigned long granule, page_sizes; + unsigned int max_addr_bits = 48; + + /* + * We need to restrict the supported page sizes to match the + * translation regime for a particular granule. Aim to match + * the CPU page size if possible, otherwise prefer smaller sizes. + * While we're at it, restrict the block sizes to match the + * chosen granule. + */ + if (cfg->pgsize_bitmap & PAGE_SIZE) + granule = PAGE_SIZE; + else if (cfg->pgsize_bitmap & ~PAGE_MASK) + granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK); + else if (cfg->pgsize_bitmap & PAGE_MASK) + granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK); + else + granule = 0; + + switch (granule) { + case SZ_4K: + page_sizes = (SZ_4K | SZ_2M | SZ_1G); + break; + case SZ_16K: + page_sizes = (SZ_16K | SZ_32M); + break; + case SZ_64K: + max_addr_bits = 52; + page_sizes = (SZ_64K | SZ_512M); + if (cfg->oas > 48) + page_sizes |= 1ULL << 42; /* 4TB */ + break; + default: + page_sizes = 0; + } + + cfg->pgsize_bitmap &= page_sizes; + cfg->ias = min(cfg->ias, max_addr_bits); + cfg->oas = min(cfg->oas, max_addr_bits); +} + +static struct arm_lpae_io_pgtable * +arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) +{ + struct arm_lpae_io_pgtable *data; + struct qcom_io_pgtable_info *pgtbl_info = to_qcom_io_pgtable_info(cfg); + int levels, va_bits, pg_shift; + + arm_lpae_restrict_pgsizes(cfg); + + if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) + return NULL; + + if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS) + return NULL; + + if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS) + return NULL; + + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + + pg_shift = __ffs(cfg->pgsize_bitmap); + data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte)); + + va_bits = cfg->ias - pg_shift; + levels = DIV_ROUND_UP(va_bits, data->bits_per_level); + data->start_level = ARM_LPAE_MAX_LEVELS - levels; + + /* Calculate the actual size of our pgd (without concatenation) */ + data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1)); + + data->iop.ops = (struct io_pgtable_ops) { + .map = arm_lpae_map, + .map_pages = arm_lpae_map_pages, + .unmap = arm_lpae_unmap, + .unmap_pages = arm_lpae_unmap_pages, + .iova_to_phys = arm_lpae_iova_to_phys, + }; + + spin_lock_init(&data->lock); + data->iommu_tlb_ops = pgtbl_info->iommu_tlb_ops; + data->pgtable_log_ops = pgtbl_info->pgtable_log_ops; + data->vmid = pgtbl_info->vmid; + if (qcom_io_pgtable_allocator_register(data->vmid)) { + kfree(data); + return NULL; + } + + return data; +} + +static struct io_pgtable * +arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) +{ + u64 reg; + struct arm_lpae_io_pgtable *data; + typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr; + bool tg1; + + if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | + IO_PGTABLE_QUIRK_ARM_TTBR1 | + IO_PGTABLE_QUIRK_ARM_OUTER_WBWA | + IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA)) + return NULL; + + data = arm_lpae_alloc_pgtable(cfg); + if (!data) + return NULL; + + /* TCR */ + if (cfg->coherent_walk) { + tcr->sh = ARM_LPAE_TCR_SH_IS; + tcr->irgn = ARM_LPAE_TCR_RGN_WBWA; + tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; + if (WARN_ON(cfg->quirks & (IO_PGTABLE_QUIRK_ARM_OUTER_WBWA | + IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA))) + goto out_free_data; + } else { + tcr->sh = ARM_LPAE_TCR_SH_OS; + tcr->irgn = ARM_LPAE_TCR_RGN_NC; + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA) + tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; + else if (cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA) + tcr->orgn = ARM_LPAE_TCR_RGN_WB; + else + tcr->orgn = ARM_LPAE_TCR_RGN_NC; + } + + tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1; + switch (ARM_LPAE_GRANULE(data)) { + case SZ_4K: + tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K; + break; + case SZ_16K: + tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K; + break; + case SZ_64K: + tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K; + break; + } + + switch (cfg->oas) { + case 32: + tcr->ips = ARM_LPAE_TCR_PS_32_BIT; + break; + case 36: + tcr->ips = ARM_LPAE_TCR_PS_36_BIT; + break; + case 40: + tcr->ips = ARM_LPAE_TCR_PS_40_BIT; + break; + case 42: + tcr->ips = ARM_LPAE_TCR_PS_42_BIT; + break; + case 44: + tcr->ips = ARM_LPAE_TCR_PS_44_BIT; + break; + case 48: + tcr->ips = ARM_LPAE_TCR_PS_48_BIT; + break; + case 52: + tcr->ips = ARM_LPAE_TCR_PS_52_BIT; + break; + default: + goto out_free_data; + } + + tcr->tsz = 64ULL - cfg->ias; + + /* MAIRs */ + reg = (ARM_LPAE_MAIR_ATTR_NC + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) | + (ARM_LPAE_MAIR_ATTR_WBRWA + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | + (ARM_LPAE_MAIR_ATTR_DEVICE + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)) | + (ARM_LPAE_MAIR_ATTR_INC_OWBRWA + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE)) | + (ARM_LPAE_MAIR_ATTR_INC_OWBRANWA + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE_NWA)) | + (ARM_LPAE_MAIR_ATTR_IWBRWA_OWBRANWA + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_ICACHE_OCACHE_NWA)); + + cfg->arm_lpae_s1_cfg.mair = reg; + + /* Looking good; allocate a pgd */ + data->pgd = __arm_lpae_alloc_pages(data, ARM_LPAE_PGD_SIZE(data), + GFP_KERNEL, cfg, cookie); + if (!data->pgd) + goto out_free_data; + + /* Ensure the empty pgd is visible before any actual TTBR write */ + wmb(); + + /* TTBR */ + cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd); + return &data->iop; + +out_free_data: + qcom_io_pgtable_allocator_unregister(data->vmid); + kfree(data); + return NULL; +} + +static struct io_pgtable * +arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) +{ + u64 sl; + struct arm_lpae_io_pgtable *data; + typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr; + + /* The NS quirk doesn't apply at stage 2 */ + if (cfg->quirks) + return NULL; + + data = arm_lpae_alloc_pgtable(cfg); + if (!data) + return NULL; + + /* + * Concatenate PGDs at level 1 if possible in order to reduce + * the depth of the stage-2 walk. + */ + if (data->start_level == 0) { + unsigned long pgd_pages; + + pgd_pages = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte); + if (pgd_pages <= ARM_LPAE_S2_MAX_CONCAT_PAGES) { + data->pgd_bits += data->bits_per_level; + data->start_level++; + } + } + + /* VTCR */ + if (cfg->coherent_walk) { + vtcr->sh = ARM_LPAE_TCR_SH_IS; + vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA; + vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA; + } else { + vtcr->sh = ARM_LPAE_TCR_SH_OS; + vtcr->irgn = ARM_LPAE_TCR_RGN_NC; + vtcr->orgn = ARM_LPAE_TCR_RGN_NC; + } + + sl = data->start_level; + + switch (ARM_LPAE_GRANULE(data)) { + case SZ_4K: + vtcr->tg = ARM_LPAE_TCR_TG0_4K; + sl++; /* SL0 format is different for 4K granule size */ + break; + case SZ_16K: + vtcr->tg = ARM_LPAE_TCR_TG0_16K; + break; + case SZ_64K: + vtcr->tg = ARM_LPAE_TCR_TG0_64K; + break; + } + + switch (cfg->oas) { + case 32: + vtcr->ps = ARM_LPAE_TCR_PS_32_BIT; + break; + case 36: + vtcr->ps = ARM_LPAE_TCR_PS_36_BIT; + break; + case 40: + vtcr->ps = ARM_LPAE_TCR_PS_40_BIT; + break; + case 42: + vtcr->ps = ARM_LPAE_TCR_PS_42_BIT; + break; + case 44: + vtcr->ps = ARM_LPAE_TCR_PS_44_BIT; + break; + case 48: + vtcr->ps = ARM_LPAE_TCR_PS_48_BIT; + break; + case 52: + vtcr->ps = ARM_LPAE_TCR_PS_52_BIT; + break; + default: + goto out_free_data; + } + + vtcr->tsz = 64ULL - cfg->ias; + vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK; + + /* Allocate pgd pages */ + data->pgd = __arm_lpae_alloc_pages(data, ARM_LPAE_PGD_SIZE(data), + GFP_KERNEL, cfg, cookie); + if (!data->pgd) + goto out_free_data; + + /* Ensure the empty pgd is visible before any actual TTBR write */ + wmb(); + + /* VTTBR */ + cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); + return &data->iop; + +out_free_data: + qcom_io_pgtable_allocator_unregister(data->vmid); + kfree(data); + return NULL; +} + +static struct io_pgtable * +arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) +{ + if (cfg->ias > 32 || cfg->oas > 40) + return NULL; + + cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); + return arm_64_lpae_alloc_pgtable_s1(cfg, cookie); +} + +static struct io_pgtable * +arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) +{ + if (cfg->ias > 40 || cfg->oas > 40) + return NULL; + + cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); + return arm_64_lpae_alloc_pgtable_s2(cfg, cookie); +} + +static struct io_pgtable * +arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) +{ + struct arm_lpae_io_pgtable *data; + + /* No quirks for Mali (hopefully) */ + if (cfg->quirks) + return NULL; + + if (cfg->ias > 48 || cfg->oas > 40) + return NULL; + + cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); + + data = arm_lpae_alloc_pgtable(cfg); + if (!data) + return NULL; + + /* Mali seems to need a full 4-level table regardless of IAS */ + if (data->start_level > 0) { + data->start_level = 0; + data->pgd_bits = 0; + } + /* + * MEMATTR: Mali has no actual notion of a non-cacheable type, so the + * best we can do is mimic the out-of-tree driver and hope that the + * "implementation-defined caching policy" is good enough. Similarly, + * we'll use it for the sake of a valid attribute for our 'device' + * index, although callers should never request that in practice. + */ + cfg->arm_mali_lpae_cfg.memattr = + (ARM_MALI_LPAE_MEMATTR_IMP_DEF + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) | + (ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | + (ARM_MALI_LPAE_MEMATTR_IMP_DEF + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); + + data->pgd = __arm_lpae_alloc_pages(data, ARM_LPAE_PGD_SIZE(data), GFP_KERNEL, + cfg, cookie); + if (!data->pgd) + goto out_free_data; + + /* Ensure the empty pgd is visible before TRANSTAB can be written */ + wmb(); + + cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) | + ARM_MALI_LPAE_TTBR_READ_INNER | + ARM_MALI_LPAE_TTBR_ADRMODE_TABLE; + return &data->iop; + +out_free_data: + qcom_io_pgtable_allocator_unregister(data->vmid); + kfree(data); + return NULL; +} + +struct io_pgtable_init_fns qcom_io_pgtable_arm_64_lpae_s1_init_fns = { + .alloc = arm_64_lpae_alloc_pgtable_s1, + .free = arm_lpae_free_pgtable, +}; + +struct io_pgtable_init_fns qcom_io_pgtable_arm_64_lpae_s2_init_fns = { + .alloc = arm_64_lpae_alloc_pgtable_s2, + .free = arm_lpae_free_pgtable, +}; + +struct io_pgtable_init_fns qcom_io_pgtable_arm_32_lpae_s1_init_fns = { + .alloc = arm_32_lpae_alloc_pgtable_s1, + .free = arm_lpae_free_pgtable, +}; + +struct io_pgtable_init_fns qcom_io_pgtable_arm_32_lpae_s2_init_fns = { + .alloc = arm_32_lpae_alloc_pgtable_s2, + .free = arm_lpae_free_pgtable, +}; + +struct io_pgtable_init_fns qcom_io_pgtable_arm_mali_lpae_init_fns = { + .alloc = arm_mali_lpae_alloc_pgtable, + .free = arm_lpae_free_pgtable, +}; + +#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST + +static struct io_pgtable_cfg *cfg_cookie __initdata; + +static void __init dummy_tlb_flush_all(void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static void __init dummy_tlb_flush(unsigned long iova, size_t size, + size_t granule, void *cookie) +{ + WARN_ON(cookie != cfg_cookie); + WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); +} + +static void __init dummy_tlb_add_page(struct iommu_iotlb_gather *gather, + unsigned long iova, size_t granule, + void *cookie) +{ + dummy_tlb_flush(iova, granule, granule, cookie); +} + +static const struct iommu_flush_ops dummy_tlb_ops __initconst = { + .tlb_flush_all = dummy_tlb_flush_all, + .tlb_flush_walk = dummy_tlb_flush, + .tlb_add_page = dummy_tlb_add_page, +}; + +static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + + pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n", + cfg->pgsize_bitmap, cfg->ias); + pr_err("data: %d levels, 0x%zx pgd_size, %u pg_shift, %u bits_per_level, pgd @ %p\n", + ARM_LPAE_MAX_LEVELS - data->start_level, ARM_LPAE_PGD_SIZE(data), + ilog2(ARM_LPAE_GRANULE(data)), data->bits_per_level, data->pgd); +} + +#define __FAIL(ops, i) ({ \ + WARN(1, "selftest: test failed for fmt idx %d\n", (i)); \ + arm_lpae_dump_ops(ops); \ + selftest_running = false; \ + -EFAULT; \ +}) + +static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) +{ + static const enum io_pgtable_fmt fmts[] __initconst = { + QCOM_ARM_64_LPAE_S1, + ARM_64_LPAE_S2, + }; + + int i, j; + unsigned long iova; + size_t size; + struct io_pgtable_ops *ops; + + selftest_running = true; + + for (i = 0; i < ARRAY_SIZE(fmts); ++i) { + cfg_cookie = cfg; + ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg); + if (!ops) { + pr_err("selftest: failed to allocate io pgtable ops\n"); + return -ENOMEM; + } + + /* + * Initial sanity checks. + * Empty page tables shouldn't provide any translations. + */ + if (ops->iova_to_phys(ops, 42)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, SZ_1G + 42)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, SZ_2G + 42)) + return __FAIL(ops, i); + + /* + * Distinct mappings of different granule sizes. + */ + iova = 0; + for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { + size = 1UL << j; + + if (ops->map(ops, iova, iova, size, IOMMU_READ | + IOMMU_WRITE | + IOMMU_NOEXEC | + IOMMU_CACHE, GFP_KERNEL)) + return __FAIL(ops, i); + + /* Overlapping mappings */ + if (!ops->map(ops, iova, iova + size, size, + IOMMU_READ | IOMMU_NOEXEC, GFP_KERNEL)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + return __FAIL(ops, i); + + iova += SZ_1G; + } + + /* Partial unmap */ + size = 1UL << __ffs(cfg->pgsize_bitmap); + if (ops->unmap(ops, SZ_1G + size, size, NULL) != size) + return __FAIL(ops, i); + + /* Remap of partial unmap */ + if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ, GFP_KERNEL)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42)) + return __FAIL(ops, i); + + /* Full unmap */ + iova = 0; + for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { + size = 1UL << j; + + if (ops->unmap(ops, iova, size, NULL) != size) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, iova + 42)) + return __FAIL(ops, i); + + /* Remap full block */ + if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + return __FAIL(ops, i); + + iova += SZ_1G; + } + + free_io_pgtable_ops(ops); + } + + selftest_running = false; + return 0; +} + +int __init qcom_arm_lpae_do_selftests(void) +{ + static const unsigned long pgsize[] __initconst = { + SZ_4K | SZ_2M | SZ_1G, + SZ_16K | SZ_32M, + SZ_64K | SZ_512M, + }; + + static const unsigned int ias[] __initconst = { + 32, 36, 40, 42, 44, 48, + }; + + int i, j, pass = 0, fail = 0; + struct io_pgtable_cfg cfg = { + .tlb = &dummy_tlb_ops, + .oas = 48, + .coherent_walk = true, + }; + + for (i = 0; i < ARRAY_SIZE(pgsize); ++i) { + for (j = 0; j < ARRAY_SIZE(ias); ++j) { + cfg.pgsize_bitmap = pgsize[i]; + cfg.ias = ias[j]; + pr_info("selftest: pgsize_bitmap 0x%08lx, IAS %u\n", + pgsize[i], ias[j]); + if (arm_lpae_run_tests(&cfg)) + fail++; + else + pass++; + } + } + + pr_info("selftest: completed with %d PASS %d FAIL\n", pass, fail); + return fail ? -EFAULT : 0; +} +#else +int __init qcom_arm_lpae_do_selftests(void) +{ + return 0; +} +#endif diff --git a/drivers/iommu/qcom-io-pgtable.h b/drivers/iommu/qcom-io-pgtable.h new file mode 100644 index 000000000000..f142bcdf6039 --- /dev/null +++ b/drivers/iommu/qcom-io-pgtable.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ +#ifndef __QCOM_QCOM_IO_PGTABLE_H +#define __QCOM_QCOM_IO_PGTABLE_H + +#include + +struct qcom_iommu_pgtable_log_ops { + void (*log_new_table)(void *cookie, void *virt, unsigned long iova, size_t granule); + void (*log_remove_table)(void *cookie, void *virt, unsigned long iova, size_t granule); +}; + +struct qcom_iommu_flush_ops { + void (*tlb_add_walk_page)(void *cookie, void *virt); + void (*tlb_add_inv)(void *cookie); + void (*tlb_sync)(void *cookie); +}; + +struct qcom_io_pgtable_info { + struct io_pgtable_cfg cfg; + const struct qcom_iommu_flush_ops *iommu_tlb_ops; + const struct qcom_iommu_pgtable_log_ops *pgtable_log_ops; + /* When set to 0, all page table memory is treated as non-secure. */ + u32 vmid; + dma_addr_t iova_base; + dma_addr_t iova_end; +}; + +#define to_qcom_io_pgtable_info(x)\ +container_of((x), struct qcom_io_pgtable_info, cfg) + +#define IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA BIT(31) + +#define ARM_V8L_FAST ((unsigned int)-1) +#define QCOM_ARM_64_LPAE_S1 ((unsigned int)-2) + +struct io_pgtable_ops *qcom_alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, + struct qcom_io_pgtable_info *pgtbl_info, + void *cookie); +void qcom_free_io_pgtable_ops(struct io_pgtable_ops *ops); + +static inline void +qcom_io_pgtable_tlb_add_walk_page(const struct qcom_iommu_flush_ops *tlb_ops, void *cookie, + void *virt) +{ + tlb_ops->tlb_add_walk_page(cookie, virt); +} + +static inline void +qcom_io_pgtable_tlb_add_inv(const struct qcom_iommu_flush_ops *tlb_ops, void *cookie) +{ + tlb_ops->tlb_add_inv(cookie); +} + +static inline void +qcom_io_pgtable_tlb_sync(const struct qcom_iommu_flush_ops *tlb_ops, void *cookie) +{ + tlb_ops->tlb_sync(cookie); +} + +static inline void +qcom_io_pgtable_log_new_table(const struct qcom_iommu_pgtable_log_ops *ops, void *cookie, + void *virt, unsigned long iova, size_t granule) +{ + ops->log_new_table(cookie, virt, iova, granule); +} + +static inline void +qcom_io_pgtable_log_remove_table(const struct qcom_iommu_pgtable_log_ops *ops, void *cookie, + void *virt, unsigned long iova, size_t granule) +{ + ops->log_remove_table(cookie, virt, iova, granule); +} + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST +extern struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns; +#endif +#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE +extern struct io_pgtable_init_fns qcom_io_pgtable_arm_64_lpae_s1_init_fns; +#endif + +#endif /* __QCOM_QCOM_IO_PGTABLE_H */ diff --git a/drivers/iommu/qcom-iommu-debug-user.c b/drivers/iommu/qcom-iommu-debug-user.c new file mode 100644 index 000000000000..c4dff03d350b --- /dev/null +++ b/drivers/iommu/qcom-iommu-debug-user.c @@ -0,0 +1,1373 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2015-2021, The Linux Foundation. All rights reserved. + * + */ + +#include +#include +#include +#include +#include +#include "qcom-iommu-debug.h" + +#define kstrtoux kstrtou64 +#define kstrtox_from_user kstrtoull_from_user +#define kstrtosize_t kstrtoul + +static void *test_virt_addr; +static DEFINE_MUTEX(test_virt_addr_lock); + +static ssize_t iommu_debug_dma_atos_read(struct file *file, char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + struct iommu_fwspec *fwspec; + phys_addr_t phys; + char buf[100] = {0}; + struct qcom_iommu_atos_txn txn; + int len; + + if (*offset) + return 0; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err("%s: No domain. Have you selected a usecase?\n", __func__); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + fwspec = dev_iommu_fwspec_get(ddev->test_dev); + if (!fwspec) { + pr_err("%s: No fwspec.\n", __func__); + mutex_unlock(&ddev->state_lock); + return 0; + } + + txn.addr = ddev->iova; + txn.flags = IOMMU_TRANS_DEFAULT; + txn.id = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[0]); + phys = qcom_iommu_iova_to_phys_hard(ddev->domain, &txn); + if (!phys) + len = strscpy(buf, "FAIL\n", sizeof(buf)); + else + len = scnprintf(buf, sizeof(buf), "%pa\n", &phys); + mutex_unlock(&ddev->state_lock); + return simple_read_from_buffer(ubuf, count, offset, buf, len); +} + +static ssize_t iommu_debug_atos_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + dma_addr_t iova; + phys_addr_t phys; + unsigned long pfn; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err("%s: No domain. Have you selected a usecase?\n", __func__); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + if (kstrtox_from_user(ubuf, count, 0, &iova)) { + dev_err(ddev->test_dev, "Invalid format for iova\n"); + ddev->iova = 0; + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + phys = iommu_iova_to_phys(ddev->domain, iova); + pfn = __phys_to_pfn(phys); + if (!pfn_valid(pfn)) { + dev_err(ddev->test_dev, "Invalid ATOS operation page %pa\n", &phys); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + ddev->iova = iova; + + mutex_unlock(&ddev->state_lock); + pr_info("Saved iova=%pa for future ATOS commands\n", &iova); + return count; +} + +const struct file_operations iommu_debug_atos_fops = { + .open = simple_open, + .write = iommu_debug_atos_write, + .read = iommu_debug_dma_atos_read, +}; + +static ssize_t iommu_debug_map_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *offset) +{ + ssize_t retval = -EINVAL; + int ret; + char *comma1, *comma2, *comma3; + char buf[100] = {0}; + dma_addr_t iova; + phys_addr_t phys; + size_t size; + int prot; + struct iommu_debug_device *ddev = file->private_data; + + if (count >= 100) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + retval = -EFAULT; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + comma2 = strnchr(comma1 + 1, count, ','); + if (!comma2) + goto invalid_format; + + comma3 = strnchr(comma2 + 1, count, ','); + if (!comma3) + goto invalid_format; + + /* split up the words */ + *comma1 = *comma2 = *comma3 = '\0'; + + if (kstrtoux(buf, 0, &iova)) + goto invalid_format; + + if (kstrtoux(comma1 + 1, 0, &phys)) + goto invalid_format; + + if (kstrtosize_t(comma2 + 1, 0, &size)) + goto invalid_format; + + if (kstrtoint(comma3 + 1, 0, &prot)) + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("%s: No domain. Have you selected a usecase?\n", __func__); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + ret = iommu_map(ddev->domain, iova, phys, size, prot); + if (ret) { + pr_err_ratelimited("iommu_map failed with %d\n", ret); + retval = -EIO; + goto out; + } + + retval = count; + pr_info("Mapped %pa to %pa (len=0x%zx, prot=0x%x)\n", &iova, &phys, size, prot); +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: iova,phys,len,prot where `prot' is the bitwise OR of IOMMU_READ, IOMMU_WRITE, etc.\n"); + return -EINVAL; +} + +const struct file_operations iommu_debug_map_fops = { + .open = simple_open, + .write = iommu_debug_map_write, +}; + +static ssize_t iommu_debug_unmap_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *offset) +{ + ssize_t retval = 0; + char *comma1; + char buf[100] = {0}; + dma_addr_t iova; + size_t size; + size_t unmapped; + struct iommu_debug_device *ddev = file->private_data; + + if (count >= 100) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + if (!ddev->domain) { + pr_err_ratelimited("%s: No domain. Have you selected a usecase?\n", __func__); + return -EINVAL; + } + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + retval = -EFAULT; + goto out; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + /* split up the words */ + *comma1 = '\0'; + + if (kstrtoux(buf, 0, &iova)) + goto invalid_format; + + if (kstrtosize_t(comma1 + 1, 0, &size)) + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("No domain. Did you already attach?\n"); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + unmapped = iommu_unmap(ddev->domain, iova, size); + if (unmapped != size) { + pr_err_ratelimited("iommu_unmap failed. Expected to unmap: 0x%zx, unmapped: 0x%zx", + size, unmapped); + retval = -EIO; + goto out; + } + + retval = count; + pr_info("Unmapped %pa (len=0x%zx)\n", &iova, size); +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: iova,len\n"); + return -EINVAL; +} + +const struct file_operations iommu_debug_unmap_fops = { + .open = simple_open, + .write = iommu_debug_unmap_write, +}; + +/* + * Performs DMA mapping of a given virtual address and size to an iova address. + * User input format: (addr,len,dma attr) where dma attr is: + * 0: normal mapping + * 1: force coherent mapping + * 2: force non-cohernet mapping + * 3: use system cache + */ +static ssize_t iommu_debug_dma_map_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *offset) +{ + ssize_t retval = -EINVAL; + int ret; + char *comma1, *comma2; + char buf[100] = {0}; + unsigned long addr; + void *v_addr; + dma_addr_t iova; + size_t size; + unsigned int attr; + unsigned long dma_attrs; + struct iommu_debug_device *ddev = file->private_data; + struct device *dev = ddev->test_dev; + + if (count >= sizeof(buf)) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + return -EFAULT; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + comma2 = strnchr(comma1 + 1, count, ','); + if (!comma2) + goto invalid_format; + + *comma1 = *comma2 = '\0'; + + if (kstrtoul(buf, 0, &addr)) + goto invalid_format; + + v_addr = (void *)addr; + + if (kstrtosize_t(comma1 + 1, 0, &size)) + goto invalid_format; + + if (kstrtouint(comma2 + 1, 0, &attr)) + goto invalid_format; + + mutex_lock(&test_virt_addr_lock); + if (IS_ERR(test_virt_addr)) { + mutex_unlock(&test_virt_addr_lock); + goto allocation_failure; + } + + if (!test_virt_addr) { + mutex_unlock(&test_virt_addr_lock); + goto missing_allocation; + } + mutex_unlock(&test_virt_addr_lock); + + if (v_addr < test_virt_addr || v_addr + size > test_virt_addr + SZ_1M) + goto invalid_addr; + + if (attr == 0) + dma_attrs = 0; + else if (attr == 1) + dma_attrs = DMA_ATTR_FORCE_COHERENT; + else if (attr == 2) + dma_attrs = DMA_ATTR_FORCE_NON_COHERENT; + else if (attr == 3) + dma_attrs = DMA_ATTR_SYS_CACHE_ONLY; + else + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("%s: No domain. Have you selected a usecase?\n", __func__); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + + iova = dma_map_single_attrs(dev, v_addr, size, DMA_TO_DEVICE, dma_attrs); + + if (dma_mapping_error(dev, iova)) { + pr_err_ratelimited("Failed to perform dma_map_single\n"); + ret = -EINVAL; + goto out; + } + + retval = count; + pr_err_ratelimited("Mapped 0x%p to %pa (len=0x%zx)\n", v_addr, &iova, size); + ddev->iova = iova; + pr_err_ratelimited("Saved iova=%pa for future PTE commands\n", &iova); + +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: addr,len,dma attr where 'dma attr' is\n0: normal mapping\n1: force coherent\n2: force non-cohernet\n3: use system cache\n"); + return retval; + +invalid_addr: + pr_err_ratelimited("Invalid addr given (0x%p)! Address should be within 1MB size from start addr returned by doing 'cat test_virt_addr'.\n", + v_addr); + return retval; + +allocation_failure: + pr_err_ratelimited("Allocation of test_virt_addr failed.\n"); + return -ENOMEM; + +missing_allocation: + pr_err_ratelimited("Please attempt to do 'cat test_virt_addr'.\n"); + return retval; +} + +static ssize_t iommu_debug_dma_map_read(struct file *file, char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + char buf[100] = {}; + dma_addr_t iova; + int len; + + if (*offset) + return 0; + + iova = ddev->iova; + len = scnprintf(buf, sizeof(buf), "%pa\n", &iova); + return simple_read_from_buffer(ubuf, count, offset, buf, len); +} + +const struct file_operations iommu_debug_dma_map_fops = { + .open = simple_open, + .read = iommu_debug_dma_map_read, + .write = iommu_debug_dma_map_write, +}; + +static ssize_t iommu_debug_dma_unmap_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *offset) +{ + ssize_t retval = 0; + char *comma1, *comma2; + char buf[100] = {}; + size_t size; + unsigned int attr; + dma_addr_t iova; + unsigned long dma_attrs; + struct iommu_debug_device *ddev = file->private_data; + struct device *dev = ddev->test_dev; + + if (count >= sizeof(buf)) { + pr_err_ratelimited("Value too large\n"); + return -EINVAL; + } + + if (copy_from_user(buf, ubuf, count)) { + pr_err_ratelimited("Couldn't copy from user\n"); + retval = -EFAULT; + goto out; + } + + comma1 = strnchr(buf, count, ','); + if (!comma1) + goto invalid_format; + + comma2 = strnchr(comma1 + 1, count, ','); + if (!comma2) + goto invalid_format; + + *comma1 = *comma2 = '\0'; + + if (kstrtoux(buf, 0, &iova)) + goto invalid_format; + + if (kstrtosize_t(comma1 + 1, 0, &size)) + goto invalid_format; + + if (kstrtouint(comma2 + 1, 0, &attr)) + goto invalid_format; + + if (attr == 0) + dma_attrs = 0; + else if (attr == 1) + dma_attrs = DMA_ATTR_FORCE_COHERENT; + else if (attr == 2) + dma_attrs = DMA_ATTR_FORCE_NON_COHERENT; + else if (attr == 3) + dma_attrs = DMA_ATTR_SYS_CACHE_ONLY; + else + goto invalid_format; + + mutex_lock(&ddev->state_lock); + if (!ddev->domain) { + pr_err_ratelimited("%s: No domain. Have you selected a usecase?\n", __func__); + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + dma_unmap_single_attrs(dev, iova, size, DMA_TO_DEVICE, dma_attrs); + + retval = count; + pr_err_ratelimited("Unmapped %pa (len=0x%zx)\n", &iova, size); +out: + mutex_unlock(&ddev->state_lock); + return retval; + +invalid_format: + pr_err_ratelimited("Invalid format. Expected: iova,len, dma attr\n"); + return -EINVAL; +} + +const struct file_operations iommu_debug_dma_unmap_fops = { + .open = simple_open, + .write = iommu_debug_dma_unmap_write, +}; + +static int iommu_debug_build_phoney_sg_table(struct device *dev, + struct sg_table *table, + unsigned long total_size, + unsigned long chunk_size) +{ + unsigned long nents = total_size / chunk_size; + struct scatterlist *sg; + int i, j; + struct page *page; + + if (!IS_ALIGNED(total_size, PAGE_SIZE)) + return -EINVAL; + if (!IS_ALIGNED(total_size, chunk_size)) + return -EINVAL; + if (sg_alloc_table(table, nents, GFP_KERNEL)) + return -EINVAL; + + for_each_sg(table->sgl, sg, table->nents, i) { + page = alloc_pages(GFP_KERNEL, get_order(chunk_size)); + if (!page) + goto free_pages; + sg_set_page(sg, page, chunk_size, 0); + } + + return 0; +free_pages: + for_each_sg(table->sgl, sg, i--, j) + __free_pages(sg_page(sg), get_order(chunk_size)); + sg_free_table(table); + return -ENOMEM; +} + +static void iommu_debug_destroy_phoney_sg_table(struct device *dev, + struct sg_table *table, + unsigned long chunk_size) +{ + struct scatterlist *sg; + int i; + + for_each_sg(table->sgl, sg, table->nents, i) + __free_pages(sg_page(sg), get_order(chunk_size)); + sg_free_table(table); +} + +#define ps_printf(name, s, fmt, ...) ({ \ + pr_err("%s: " fmt, name, ##__VA_ARGS__); \ + seq_printf(s, fmt, ##__VA_ARGS__); \ + }) + +static int __functional_dma_api_alloc_test(struct device *dev, + struct seq_file *s, + struct iommu_domain *domain, + void *ignored) +{ + size_t size = SZ_1K * 742; + int ret = 0; + u8 *data; + dma_addr_t iova; + + /* Make sure we can allocate and use a buffer */ + ps_printf(dev_name(dev), s, "Allocating coherent buffer"); + data = dma_alloc_coherent(dev, size, &iova, GFP_KERNEL); + if (!data) { + ret = -ENOMEM; + } else { + int i; + + ps_printf(dev_name(dev), s, " -> SUCCEEDED\n"); + ps_printf(dev_name(dev), s, "Using coherent buffer"); + for (i = 0; i < 742; ++i) { + int ind = SZ_1K * i; + u8 *p = data + ind; + u8 val = i % 255; + + memset(data, 0xa5, size); + *p = val; + (*p)++; + if ((*p) != val + 1) { + ps_printf(dev_name(dev), s, + " -> FAILED on iter %d since %d != %d\n", + i, *p, val + 1); + ret = -EINVAL; + } + } + if (!ret) + ps_printf(dev_name(dev), s, " -> SUCCEEDED\n"); + dma_free_coherent(dev, size, data, iova); + } + + return ret; +} + +static int __functional_dma_api_basic_test(struct device *dev, + struct seq_file *s, + struct iommu_domain *domain, + void *ignored) +{ + size_t size = 1518; + int i, j, ret = 0; + u8 *data; + dma_addr_t iova; + + ps_printf(dev_name(dev), s, "Basic DMA API test"); + /* Make sure we can allocate and use a buffer */ + for (i = 0; i < 1000; ++i) { + data = kmalloc(size, GFP_KERNEL); + if (!data) { + ret = -ENOMEM; + goto out; + } + memset(data, 0xa5, size); + iova = dma_map_single(dev, data, size, DMA_TO_DEVICE); + ret = iommu_debug_check_mapping_fast(dev, iova, size, virt_to_phys(data)); + if (ret) + goto out; + + dma_unmap_single(dev, iova, size, DMA_TO_DEVICE); + for (j = 0; j < size; ++j) { + if (data[j] != 0xa5) { + dev_err_ratelimited(dev, "data[%d] != 0xa5\n", data[j]); + ret = -EINVAL; + goto out; + } + } + kfree(data); + } + +out: + if (ret) + ps_printf(dev_name(dev), s, " -> FAILED\n"); + else + ps_printf(dev_name(dev), s, " -> SUCCEEDED\n"); + + return ret; +} + +static int __functional_dma_api_map_sg_test(struct device *dev, struct seq_file *s, + struct iommu_domain *domain, size_t sizes[]) +{ + const size_t *sz; + int ret = 0, count = 0; + + ps_printf(dev_name(dev), s, "Map SG DMA API test"); + + for (sz = sizes; *sz; ++sz) { + size_t size = *sz; + struct sg_table table; + unsigned long chunk_size = SZ_4K; + + /* Build us a table */ + ret = iommu_debug_build_phoney_sg_table(dev, &table, size, chunk_size); + if (ret) { + seq_puts(s, "couldn't build phoney sg table! bailing...\n"); + goto out; + } + + count = dma_map_sg(dev, table.sgl, table.nents, DMA_BIDIRECTIONAL); + if (!count) { + ret = -EINVAL; + goto destroy_table; + } + + /* Check mappings... */ + ret = iommu_debug_check_mapping_sg_fast(dev, table.sgl, 0, table.nents, count); + + dma_unmap_sg(dev, table.sgl, table.nents, DMA_BIDIRECTIONAL); +destroy_table: + iommu_debug_destroy_phoney_sg_table(dev, &table, chunk_size); + } +out: + if (ret) + ps_printf(dev_name(dev), s, " -> FAILED\n"); + else + ps_printf(dev_name(dev), s, " -> SUCCEEDED\n"); + + return ret; +} + +static int iommu_debug_functional_arm_dma_api_show(struct seq_file *s, + void *ignored) +{ + struct iommu_debug_device *ddev = s->private; + struct device *dev; + size_t sizes[] = {SZ_4K, SZ_64K, SZ_2M, SZ_1M * 12, 0}; + int ret = -EINVAL; + + mutex_lock(&ddev->state_lock); + if (!iommu_debug_usecase_reset(ddev)) + goto out; + dev = ddev->test_dev; + + ret = __functional_dma_api_alloc_test(dev, s, ddev->domain, sizes); + ret |= __functional_dma_api_basic_test(dev, s, ddev->domain, sizes); + ret |= __functional_dma_api_map_sg_test(dev, s, ddev->domain, sizes); + +out: + mutex_unlock(&ddev->state_lock); + if (ret) + seq_printf(s, "FAIL %d\n", ret); + else + seq_puts(s, "SUCCESS\n"); + + return 0; +} + +static int iommu_debug_functional_arm_dma_api_open(struct inode *inode, + struct file *file) +{ + return single_open(file, iommu_debug_functional_arm_dma_api_show, + inode->i_private); +} + +const struct file_operations iommu_debug_functional_arm_dma_api_fops = { + .open = iommu_debug_functional_arm_dma_api_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* Creates a fresh fast mapping and applies @fn to it */ +static int __apply_to_new_mapping(struct seq_file *s, + int (*fn)(struct device *dev, + struct seq_file *s, + struct iommu_domain *domain, + void *priv), + void *priv) +{ + struct iommu_domain *domain; + struct iommu_debug_device *ddev = s->private; + struct device *dev; + int ret = -EINVAL; + + mutex_lock(&ddev->state_lock); + if (!iommu_debug_usecase_reset(ddev)) + goto out; + + domain = ddev->domain; + dev = ddev->test_dev; + + ret = fn(dev, s, domain, priv); + +out: + mutex_unlock(&ddev->state_lock); + seq_printf(s, "%s\n", ret ? "FAIL" : "SUCCESS"); + return ret; +} + +static const char * const _size_to_string(unsigned long size) +{ + switch (size) { + case SZ_4K: + return "4K"; + case SZ_8K: + return "8K"; + case SZ_16K: + return "16K"; + case SZ_64K: + return "64K"; + case SZ_1M: + return "1M"; + case SZ_2M: + return "2M"; + case SZ_1M * 12: + return "12M"; + case SZ_1M * 20: + return "20M"; + case SZ_1M * 24: + return "24M"; + case SZ_1M * 32: + return "32M"; + } + + pr_err("unknown size, please add to %s\n", __func__); + return "unknown size"; +} + +static int __check_mapping(struct device *dev, struct iommu_domain *domain, + dma_addr_t iova, phys_addr_t expected) +{ + struct iommu_fwspec *fwspec; + phys_addr_t res, res2; + struct qcom_iommu_atos_txn txn; + + fwspec = dev_iommu_fwspec_get(dev); + if (!fwspec) { + dev_err_ratelimited(dev, "No fwspec.\n"); + return -EINVAL; + } + + txn.addr = iova; + txn.flags = IOMMU_TRANS_DEFAULT; + txn.id = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[0]); + + res = qcom_iommu_iova_to_phys_hard(domain, &txn); + res2 = iommu_iova_to_phys(domain, iova); + + WARN(res != res2, "hard/soft iova_to_phys fns don't agree..."); + + if (res != expected) { + dev_err_ratelimited(dev, "Bad translation for %pa! Expected: %pa Got: %pa\n", + &iova, &expected, &res); + return -EINVAL; + } + + return 0; +} + +static int __full_va_sweep(struct device *dev, struct seq_file *s, + struct iommu_domain *domain, void *priv) +{ + u64 iova; + dma_addr_t dma_addr; + void *virt; + phys_addr_t phys; + const u64 max = SZ_1G * 4ULL - 1; + int ret = 0, i; + const size_t size = (size_t)priv; + unsigned long theiova; + + virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); + if (!virt) { + if (size > SZ_8K) { + dev_err_ratelimited(dev, "Failed to allocate %s of memory, which is a lot. Skipping test for this size\n", + _size_to_string(size)); + return 0; + } + return -ENOMEM; + } + phys = virt_to_phys(virt); + + for (iova = 0, i = 0; iova < max; iova += size, ++i) { + unsigned long expected = iova; + + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - size; + continue; + } + + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr != expected) { + dev_err_ratelimited(dev, "Unexpected iova on iter %d (expected: 0x%lx got: 0x%lx)\n", + i, expected, (unsigned long)dma_addr); + ret = -EINVAL; + goto out; + } + } + + if (domain) { + /* check every mapping from 0..6M */ + for (iova = 0, i = 0; iova < SZ_2M * 3; iova += size, ++i) { + phys_addr_t expected = phys; + + if (__check_mapping(dev, domain, iova, expected)) { + dev_err_ratelimited(dev, "iter: %d\n", i); + ret = -EINVAL; + goto out; + } + } + /* and from 4G..4G-6M */ + for (iova = 0, i = 0; iova < SZ_2M * 3; iova += size, ++i) { + phys_addr_t expected = phys; + + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - size; + continue; + } + theiova = ((SZ_1G * 4ULL) - size) - iova; + + if (__check_mapping(dev, domain, theiova, expected)) { + dev_err_ratelimited(dev, "iter: %d\n", i); + ret = -EINVAL; + goto out; + } + } + } + + /* at this point, our VA space should be full */ + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr != DMA_MAPPING_ERROR) { + dev_err_ratelimited(dev, "dma_map_single succeeded when it should have failed. Got iova: 0x%lx\n", + (unsigned long)dma_addr); + ret = -EINVAL; + } + +out: + for (iova = 0; iova < max; iova += size) { + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - size; + continue; + } + dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE); + } + + free_pages((unsigned long)virt, get_order(size)); + return ret; +} + +static int __tlb_stress_sweep(struct device *dev, struct seq_file *s, + struct iommu_domain *domain, void *unused) +{ + int i, ret = 0; + u64 iova; + const u64 max = SZ_1G * 4ULL - 1; + void *virt; + phys_addr_t phys; + dma_addr_t dma_addr; + + /* + * we'll be doing 4K and 8K mappings. Need to own an entire 8K + * chunk that we can work with. + */ + virt = (void *)__get_free_pages(GFP_KERNEL, get_order(SZ_8K)); + phys = virt_to_phys(virt); + + /* fill the whole 4GB space */ + for (iova = 0, i = 0; iova < max; iova += SZ_8K, ++i) { + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - SZ_8K; + continue; + } + + dma_addr = dma_map_single(dev, virt, SZ_8K, DMA_TO_DEVICE); + if (dma_addr == DMA_MAPPING_ERROR) { + dev_err_ratelimited(dev, "Failed map on iter %d\n", i); + ret = -EINVAL; + goto out; + } + } + + if (dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE) != DMA_MAPPING_ERROR) { + dev_err_ratelimited(dev, "dma_map_single unexpectedly (VA should have been exhausted)\n"); + ret = -EINVAL; + goto out; + } + + /* + * free up 4K at the very beginning, then leave one 4K mapping, + * then free up 8K. This will result in the next 8K map to skip + * over the 4K hole and take the 8K one. + */ + dma_unmap_single(dev, 0, SZ_4K, DMA_TO_DEVICE); + dma_unmap_single(dev, SZ_8K, SZ_4K, DMA_TO_DEVICE); + dma_unmap_single(dev, SZ_8K + SZ_4K, SZ_4K, DMA_TO_DEVICE); + + /* remap 8K */ + dma_addr = dma_map_single(dev, virt, SZ_8K, DMA_TO_DEVICE); + if (dma_addr != SZ_8K) { + dma_addr_t expected = SZ_8K; + + dev_err_ratelimited(dev, "Unexpected dma_addr. got: %pa expected: %pa\n", + &dma_addr, &expected); + ret = -EINVAL; + goto out; + } + + /* + * now remap 4K. We should get the first 4K chunk that was skipped + * over during the previous 8K map. If we missed a TLB invalidate + * at that point this should explode. + */ + dma_addr = dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE); + if (dma_addr != 0) { + dma_addr_t expected = 0; + + dev_err_ratelimited(dev, "Unexpected dma_addr. got: %pa expected: %pa\n", + &dma_addr, &expected); + ret = -EINVAL; + goto out; + } + + if (dma_map_single(dev, virt, SZ_4K, DMA_TO_DEVICE) != DMA_MAPPING_ERROR) { + dev_err_ratelimited(dev, "dma_map_single unexpectedly after remaps (VA should have been exhausted)\n"); + ret = -EINVAL; + goto out; + } + + /* we're all full again. unmap everything. */ + for (iova = 0; iova < max; iova += SZ_8K) { + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - SZ_8K; + continue; + } + dma_unmap_single(dev, (dma_addr_t)iova, SZ_8K, DMA_TO_DEVICE); + } + +out: + free_pages((unsigned long)virt, get_order(SZ_8K)); + return ret; +} + +struct fib_state { + unsigned long cur; + unsigned long prev; +}; + +static void fib_init(struct fib_state *f) +{ + f->cur = f->prev = 1; +} + +static unsigned long get_next_fib(struct fib_state *f) +{ + int next = f->cur + f->prev; + + f->prev = f->cur; + f->cur = next; + return next; +} + +/* + * Not actually random. Just testing the fibs (and max - the fibs). + */ +static int __rand_va_sweep(struct device *dev, struct seq_file *s, + struct iommu_domain *domain, void *priv) +{ + u64 iova; + const u64 max = SZ_1G * 4ULL - 1; + int i, remapped, unmapped, ret = 0; + void *virt; + dma_addr_t dma_addr, dma_addr2; + struct fib_state fib; + const size_t size = (size_t)priv; + + virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size)); + if (!virt) { + if (size > SZ_8K) { + dev_err_ratelimited(dev, "Failed to allocate %s of memory, which is a lot. Skipping test for this size\n", + _size_to_string(size)); + return 0; + } + return -ENOMEM; + } + + /* fill the whole 4GB space */ + for (iova = 0, i = 0; iova < max; iova += size, ++i) { + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - size; + continue; + } + + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr == DMA_MAPPING_ERROR) { + dev_err_ratelimited(dev, "Failed map on iter %d\n", i); + ret = -EINVAL; + goto out; + } + } + + /* now unmap "random" iovas */ + unmapped = 0; + fib_init(&fib); + for (iova = get_next_fib(&fib) * size; + iova < max - size; + iova = (u64)get_next_fib(&fib) * size) { + dma_addr = (dma_addr_t)(iova); + dma_addr2 = (dma_addr_t)((max + 1) - size - iova); + if (dma_addr == dma_addr2) { + WARN(1, "%s test needs update! The random number sequence is folding in on itself and should be changed.\n", + __func__); + return -EINVAL; + } + + if (!(MSI_IOVA_BASE <= dma_addr && MSI_IOVA_BASE + MSI_IOVA_LENGTH > dma_addr)) + dma_unmap_single(dev, dma_addr, size, DMA_TO_DEVICE); + + if (!(MSI_IOVA_BASE <= dma_addr2 && MSI_IOVA_BASE + MSI_IOVA_LENGTH > dma_addr2)) + dma_unmap_single(dev, dma_addr2, size, DMA_TO_DEVICE); + + unmapped += 2; + } + + /* and map until everything fills back up */ + for (remapped = 0; ; ++remapped) { + dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE); + if (dma_addr == DMA_MAPPING_ERROR) + break; + } + + if (unmapped != remapped) { + dev_err_ratelimited(dev, "Unexpected random remap count! Unmapped %d but remapped %d\n", + unmapped, remapped); + ret = -EINVAL; + } + + for (iova = 0; iova < max; iova += size) { + if (iova == MSI_IOVA_BASE) { + iova = MSI_IOVA_BASE + MSI_IOVA_LENGTH - size; + continue; + } + dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE); + } + +out: + free_pages((unsigned long)virt, get_order(size)); + return ret; +} + +static int __functional_dma_api_va_test(struct seq_file *s) +{ + int ret = 0; + size_t *sz; + size_t sizes[] = {SZ_4K, SZ_8K, SZ_16K, SZ_64K, 0}; + struct iommu_debug_device *ddev = s->private; + char *usecase_name; + + /* + * dev_name() cannot be used to get the usecase name as ddev->test_dev + * will be NULL in case __apply_to_new_mapping() fails. Since + * ddev->test_dev changes across calls to __apply_to_new_mapping(), we + * also can't hold a reference to its name by caching the result of + * dev_name() initially. + */ + mutex_lock(&ddev->state_lock); + if (!ddev->test_dev) { + mutex_unlock(&ddev->state_lock); + return -ENODEV; + } + + usecase_name = kstrdup(dev_name(ddev->test_dev), GFP_KERNEL); + mutex_unlock(&ddev->state_lock); + if (!usecase_name) + return -ENOMEM; + + for (sz = sizes; *sz; ++sz) { + ps_printf(usecase_name, s, "Full VA sweep @%s:", _size_to_string(*sz)); + if (__apply_to_new_mapping(s, __full_va_sweep, (void *)*sz)) { + ps_printf(usecase_name, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ps_printf(usecase_name, s, " -> SUCCEEDED\n"); + } + } + + ps_printf(usecase_name, s, "bonus map:"); + if (__apply_to_new_mapping(s, __full_va_sweep, (void *)SZ_4K)) { + ps_printf(usecase_name, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ps_printf(usecase_name, s, " -> SUCCEEDED\n"); + } + + for (sz = sizes; *sz; ++sz) { + ps_printf(usecase_name, s, "Rand VA sweep @%s:", _size_to_string(*sz)); + if (__apply_to_new_mapping(s, __rand_va_sweep, (void *)*sz)) { + ps_printf(usecase_name, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ps_printf(usecase_name, s, " -> SUCCEEDED\n"); + } + } + + ps_printf(usecase_name, s, "TLB stress sweep:"); + if (__apply_to_new_mapping(s, __tlb_stress_sweep, NULL)) { + ps_printf(usecase_name, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ps_printf(usecase_name, s, " -> SUCCEEDED\n"); + } + + ps_printf(usecase_name, s, "second bonus map:"); + if (__apply_to_new_mapping(s, __full_va_sweep, (void *)SZ_4K)) { + ps_printf(usecase_name, s, " -> FAILED\n"); + ret = -EINVAL; + } else { + ps_printf(usecase_name, s, " -> SUCCEEDED\n"); + } + + kfree(usecase_name); + return ret; +} + +static int iommu_debug_functional_fast_dma_api_show(struct seq_file *s, + void *ignored) +{ + int ret = 0; + + ret |= __apply_to_new_mapping(s, __functional_dma_api_alloc_test, NULL); + ret |= __apply_to_new_mapping(s, __functional_dma_api_basic_test, NULL); + ret |= __functional_dma_api_va_test(s); + return ret; +} + +static int iommu_debug_functional_fast_dma_api_open(struct inode *inode, + struct file *file) +{ + return single_open(file, iommu_debug_functional_fast_dma_api_show, + inode->i_private); +} + +const struct file_operations iommu_debug_functional_fast_dma_api_fops = { + .open = iommu_debug_functional_fast_dma_api_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static ssize_t iommu_debug_test_virt_addr_read(struct file *file, + char __user *ubuf, + size_t count, loff_t *offset) +{ + char buf[100]; + int len; + + if (*offset) + return 0; + + mutex_lock(&test_virt_addr_lock); + if (IS_ERR_OR_NULL(test_virt_addr)) + test_virt_addr = kzalloc(SZ_1M, GFP_KERNEL); + + if (!test_virt_addr) { + test_virt_addr = ERR_PTR(-ENOMEM); + len = strscpy(buf, "FAIL\n", sizeof(buf)); + } else { + len = scnprintf(buf, sizeof(buf), "0x%p\n", test_virt_addr); + } + mutex_unlock(&test_virt_addr_lock); + + return simple_read_from_buffer(ubuf, count, offset, buf, len); +} + +const struct file_operations iommu_debug_test_virt_addr_fops = { + .open = simple_open, + .read = iommu_debug_test_virt_addr_read, +}; + +#ifdef CONFIG_IOMMU_IOVA_ALIGNMENT +static unsigned long iommu_debug_get_align_mask(size_t size) +{ + unsigned long align_mask = ~0UL; + + align_mask <<= min_t(unsigned long, CONFIG_IOMMU_IOVA_ALIGNMENT + PAGE_SHIFT, + fls_long(size - 1)); + return ~align_mask; +} +#else +static unsigned long iommu_debug_get_align_mask(size_t size) +{ + unsigned long align_mask = ~0UL; + + align_mask <<= fls_long(size - 1); + return ~align_mask; +} +#endif + +static void iommu_debug_device_profiling(struct seq_file *s, struct iommu_debug_device *ddev, + const size_t sizes[]) +{ + const size_t *sz; + struct iommu_domain *domain; + struct device *dev; + unsigned long iova = 0x10000; + phys_addr_t paddr = 0x80000000; + + mutex_lock(&ddev->state_lock); + if (!iommu_debug_usecase_reset(ddev)) + goto out; + + domain = ddev->domain; + dev = ddev->test_dev; + + seq_printf(s, "(average over %d iterations)\n", ddev->nr_iters); + seq_printf(s, "%8s %19s %16s\n", "size", "iommu_map", "iommu_unmap"); + for (sz = sizes; *sz; ++sz) { + size_t size = *sz; + size_t unmapped; + u64 map_elapsed_ns = 0, unmap_elapsed_ns = 0; + u64 map_elapsed_us = 0, unmap_elapsed_us = 0; + u32 map_elapsed_rem = 0, unmap_elapsed_rem = 0; + ktime_t tbefore, tafter, diff; + int i; + unsigned long align_mask = iommu_debug_get_align_mask(size); + + for (i = 0; i < ddev->nr_iters; ++i) { + tbefore = ktime_get(); + if (iommu_map(domain, __ALIGN_MASK(iova, align_mask), + ALIGN(paddr, size), size, + IOMMU_READ | IOMMU_WRITE)) { + seq_puts(s, "Failed to map\n"); + continue; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + map_elapsed_ns += ktime_to_ns(diff); + + tbefore = ktime_get(); + unmapped = iommu_unmap(domain, + __ALIGN_MASK(iova, align_mask), + size); + if (unmapped != size) { + seq_printf(s, + "Only unmapped %zx instead of %zx\n", + unmapped, size); + continue; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + unmap_elapsed_ns += ktime_to_ns(diff); + } + + map_elapsed_ns = div_u64_rem(map_elapsed_ns, ddev->nr_iters, &map_elapsed_rem); + unmap_elapsed_ns = div_u64_rem(unmap_elapsed_ns, ddev->nr_iters, + &unmap_elapsed_rem); + + map_elapsed_us = div_u64_rem(map_elapsed_ns, 1000, &map_elapsed_rem); + unmap_elapsed_us = div_u64_rem(unmap_elapsed_ns, 1000, &unmap_elapsed_rem); + + seq_printf(s, "%8s %12lld.%03d us %9lld.%03d us\n", + _size_to_string(size), map_elapsed_us, map_elapsed_rem, + unmap_elapsed_us, unmap_elapsed_rem); + } + + seq_putc(s, '\n'); + seq_printf(s, "%8s %19s %16s\n", "size", "iommu_map_sg", "iommu_unmap"); + for (sz = sizes; *sz; ++sz) { + size_t size = *sz; + size_t unmapped; + u64 map_elapsed_ns = 0, unmap_elapsed_ns = 0; + u64 map_elapsed_us = 0, unmap_elapsed_us = 0; + u32 map_elapsed_rem = 0, unmap_elapsed_rem = 0; + ktime_t tbefore, tafter, diff; + struct sg_table table; + unsigned long chunk_size = SZ_4K; + int i; + unsigned long align_mask = iommu_debug_get_align_mask(size); + + if (iommu_debug_build_phoney_sg_table(dev, &table, size, + chunk_size)) { + seq_puts(s, "couldn't build phoney sg table! bailing...\n"); + goto out; + } + + for (i = 0; i < ddev->nr_iters; ++i) { + tbefore = ktime_get(); + if (iommu_map_sgtable(domain, __ALIGN_MASK(iova, align_mask), + &table, IOMMU_READ | IOMMU_WRITE) != size) { + seq_puts(s, "Failed to map_sg\n"); + goto next; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + map_elapsed_ns += ktime_to_ns(diff); + + tbefore = ktime_get(); + unmapped = iommu_unmap(domain, + __ALIGN_MASK(iova, align_mask), + size); + if (unmapped != size) { + seq_printf(s, "Only unmapped %zx instead of %zx\n", + unmapped, size); + goto next; + } + tafter = ktime_get(); + diff = ktime_sub(tafter, tbefore); + unmap_elapsed_ns += ktime_to_ns(diff); + } + + map_elapsed_ns = div_u64_rem(map_elapsed_ns, ddev->nr_iters, &map_elapsed_rem); + unmap_elapsed_ns = div_u64_rem(unmap_elapsed_ns, ddev->nr_iters, + &unmap_elapsed_rem); + + map_elapsed_us = div_u64_rem(map_elapsed_ns, 1000, &map_elapsed_rem); + unmap_elapsed_us = div_u64_rem(unmap_elapsed_ns, 1000, &unmap_elapsed_rem); + + seq_printf(s, "%8s %12lld.%03d us %9lld.%03d us\n", _size_to_string(size), + map_elapsed_us, map_elapsed_rem, unmap_elapsed_us, unmap_elapsed_rem); + +next: + iommu_debug_destroy_phoney_sg_table(dev, &table, chunk_size); + } + +out: + mutex_unlock(&ddev->state_lock); +} + +static int iommu_debug_profiling_show(struct seq_file *s, void *ignored) +{ + struct iommu_debug_device *ddev = s->private; + const size_t sizes[] = { SZ_4K, SZ_64K, SZ_1M, SZ_2M, SZ_1M * 12, + SZ_1M * 24, SZ_1M * 32, 0 }; + + iommu_debug_device_profiling(s, ddev, sizes); + return 0; +} + +static int iommu_debug_profiling_open(struct inode *inode, struct file *file) +{ + return single_open(file, iommu_debug_profiling_show, inode->i_private); +} + +const struct file_operations iommu_debug_profiling_fops = { + .open = iommu_debug_profiling_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; diff --git a/drivers/iommu/qcom-iommu-debug.c b/drivers/iommu/qcom-iommu-debug.c new file mode 100644 index 000000000000..134068ce8e9d --- /dev/null +++ b/drivers/iommu/qcom-iommu-debug.c @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2015-2021, The Linux Foundation. All rights reserved. + * + */ + +#define pr_fmt(fmt) "iommu-debug: %s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include "qcom-iommu-debug.h" + +#define USECASE_SWITCH_TIMEOUT_MSECS (500) + +static int iommu_debug_nr_iters_set(void *data, u64 val) +{ + struct iommu_debug_device *ddev = data; + + if (!val) + val = 1; + + if (val > 10000) + val = 10000; + + ddev->nr_iters = (u32)val; + + return 0; +} + +static int iommu_debug_nr_iters_get(void *data, u64 *val) +{ + struct iommu_debug_device *ddev = data; + + *val = ddev->nr_iters; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(iommu_debug_nr_iters_fops, + iommu_debug_nr_iters_get, + iommu_debug_nr_iters_set, + "%llu\n"); + +int iommu_debug_check_mapping_flags(struct device *dev, dma_addr_t iova, size_t size, + phys_addr_t expected_pa, u32 flags) +{ + struct qcom_iommu_atos_txn txn; + struct iommu_fwspec *fwspec; + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) { + dev_err(dev, "iommu_get_domain_for_dev() failed\n"); + return -EINVAL; + } + + fwspec = dev_iommu_fwspec_get(dev); + if (!fwspec) { + dev_err(dev, "dev_iommu_fwspec_get() failed\n"); + return -EINVAL; + } + + txn.addr = iova; + txn.id = FIELD_GET(ARM_SMMU_SMR_ID, (fwspec->ids[0])); + txn.flags = flags; + + size = PAGE_ALIGN(size); + while (size) { + phys_addr_t walk_pa, atos_pa; + + atos_pa = qcom_iommu_iova_to_phys_hard(domain, &txn); + walk_pa = iommu_iova_to_phys(domain, iova); + + if (expected_pa != atos_pa || expected_pa != walk_pa) { + dev_err_ratelimited(dev, + "Bad translation for %pad! Expected: %pa Got: %pa (ATOS) %pa (Table Walk) sid=%08x\n", + &iova, &expected_pa, &atos_pa, &walk_pa, txn.id); + return -EINVAL; + } + + size -= PAGE_SIZE; + iova += PAGE_SIZE; + expected_pa += PAGE_SIZE; + } + + return 0; +} + +int iommu_debug_check_mapping_sg_flags(struct device *dev, struct scatterlist *sgl, + unsigned int pgoffset, unsigned int dma_nents, + unsigned int nents, u32 flags) +{ + int ret; + struct sg_page_iter piter; + struct sg_dma_page_iter diter; + + for (__sg_page_iter_start(&piter, sgl, nents, pgoffset), + __sg_page_iter_start(&diter.base, sgl, dma_nents, pgoffset); + __sg_page_iter_next(&piter) && __sg_page_iter_dma_next(&diter);) { + + struct page *page = sg_page_iter_page(&piter); + dma_addr_t dma_addr = sg_page_iter_dma_address(&diter); + + ret = iommu_debug_check_mapping_flags(dev, dma_addr, PAGE_SIZE, + page_to_phys(page), flags); + if (ret) + return ret; + } + + return 0; +} + +static void iommu_debug_destroy_test_dev(struct iommu_debug_device *ddev) +{ + if (ddev->test_dev) { + of_platform_device_destroy(ddev->test_dev, NULL); + ddev->test_dev = NULL; + ddev->domain = NULL; + } +} + +/* + * Returns struct device corresponding to the new usecase. + * ddev->test_dev will change - caller must not use old value! + * Caller must hold ddev->state_lock + */ +struct device * +iommu_debug_switch_usecase(struct iommu_debug_device *ddev, u32 usecase_nr) +{ + struct platform_device *test_pdev; + struct device_node *child; + int child_nr = 0; + int ret; + + if (ddev->test_dev) + iommu_debug_destroy_test_dev(ddev); + + if (usecase_nr >= of_get_child_count(ddev->self->of_node)) { + dev_err(ddev->self, "Invalid usecase nr requested: %u\n", + usecase_nr); + return NULL; + } + + reinit_completion(&ddev->probe_wait); + for_each_child_of_node(ddev->self->of_node, child) { + if (child_nr == usecase_nr) + break; + child_nr++; + } + + test_pdev = of_platform_device_create(child, NULL, ddev->self); + if (!test_pdev) { + dev_err(ddev->self, "Creating platform device failed\n"); + return NULL; + } + + /* + * Wait for child device's probe function to be called. + * Its very unlikely to be asynchonrous... + */ + ret = wait_for_completion_interruptible_timeout(&ddev->probe_wait, + msecs_to_jiffies(USECASE_SWITCH_TIMEOUT_MSECS)); + if (ret <= 0) { + dev_err(ddev->self, "Timed out waiting for usecase to register\n"); + goto out; + } + + ddev->usecase_nr = usecase_nr; + ddev->test_dev = &test_pdev->dev; + ddev->domain = iommu_get_domain_for_dev(ddev->test_dev); + if (!ddev->domain) { + dev_err(ddev->self, "Oops, usecase not associated with iommu\n"); + goto out; + } + + return ddev->test_dev; +out: + iommu_debug_destroy_test_dev(ddev); + return NULL; +} + +/* + * Caller must hold ddev->state_lock + */ +struct device *iommu_debug_usecase_reset(struct iommu_debug_device *ddev) +{ + return iommu_debug_switch_usecase(ddev, ddev->usecase_nr); +} + +static int iommu_debug_usecase_register(struct device *dev) +{ + struct iommu_debug_device *ddev = dev_get_drvdata(dev->parent); + + complete(&ddev->probe_wait); + return 0; +} + +static ssize_t iommu_debug_usecase_read(struct file *file, char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + + return simple_read_from_buffer(ubuf, count, offset, ddev->buffer, + strnlen(ddev->buffer, PAGE_SIZE)); +} + +static ssize_t iommu_debug_usecase_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *offset) +{ + struct iommu_debug_device *ddev = file->private_data; + unsigned int usecase_nr; + int ret; + + ret = kstrtouint_from_user(ubuf, count, 0, &usecase_nr); + if (ret || usecase_nr >= ddev->nr_children) + return -EINVAL; + + mutex_lock(&ddev->state_lock); + if (!iommu_debug_switch_usecase(ddev, usecase_nr)) { + mutex_unlock(&ddev->state_lock); + return -EINVAL; + } + mutex_unlock(&ddev->state_lock); + + return count; +} + +static const struct file_operations iommu_debug_usecase_fops = { + .open = simple_open, + .read = iommu_debug_usecase_read, + .write = iommu_debug_usecase_write, + .llseek = no_llseek, +}; + +static int iommu_debug_debugfs_setup(struct iommu_debug_device *ddev) +{ + struct dentry *dir; + + dir = debugfs_create_dir("iommu-test", NULL); + if (IS_ERR(dir)) + return -EINVAL; + + ddev->root_dir = dir; + + debugfs_create_file("usecase", 0600, dir, ddev, &iommu_debug_usecase_fops); + debugfs_create_file("functional_arm_dma_api", 0400, dir, ddev, + &iommu_debug_functional_arm_dma_api_fops); + debugfs_create_file("functional_fast_dma_api", 0400, dir, ddev, + &iommu_debug_functional_fast_dma_api_fops); + debugfs_create_file("atos", 0600, dir, ddev, &iommu_debug_atos_fops); + debugfs_create_file("map", 0200, dir, ddev, &iommu_debug_map_fops); + debugfs_create_file("unmap", 0200, dir, ddev, &iommu_debug_unmap_fops); + debugfs_create_file("dma_map", 0200, dir, ddev, &iommu_debug_dma_map_fops); + debugfs_create_file("dma_unmap", 0200, dir, ddev, &iommu_debug_dma_unmap_fops); + debugfs_create_file("nr_iters", 0600, dir, ddev, &iommu_debug_nr_iters_fops); + debugfs_create_file("test_virt_addr", 0400, dir, ddev, &iommu_debug_test_virt_addr_fops); + debugfs_create_file("profiling", 0400, dir, ddev, &iommu_debug_profiling_fops); + + return 0; +} + +static int iommu_debug_probe(struct platform_device *pdev) +{ + struct iommu_debug_device *ddev; + struct device *dev = &pdev->dev; + struct device_node *child; + int ret; + int offset = 0; + + ddev = devm_kzalloc(dev, sizeof(*ddev), GFP_KERNEL); + if (!ddev) + return -ENOMEM; + + ddev->self = dev; + ddev->usecase_nr = U32_MAX; + ddev->nr_iters = 1; + mutex_init(&ddev->state_lock); + init_completion(&ddev->probe_wait); + + ddev->buffer = devm_kzalloc(dev, PAGE_SIZE, GFP_KERNEL); + if (!ddev->buffer) { + ret = -ENOMEM; + goto out; + } + + ddev->nr_children = 0; + for_each_child_of_node(dev->of_node, child) { + offset += scnprintf(ddev->buffer + offset, PAGE_SIZE - offset, + "%d: %s\n", ddev->nr_children, child->name); + if (offset + 1 == PAGE_SIZE) { + dev_err(dev, "Too many testcases?\n"); + break; + } + ddev->nr_children++; + } + dev_set_drvdata(dev, ddev); + + ret = iommu_debug_debugfs_setup(ddev); + if (ret) + goto out; + + return 0; + +out: + mutex_destroy(&ddev->state_lock); + return ret; +} + +static int iommu_debug_remove(struct platform_device *pdev) +{ + struct iommu_debug_device *ddev = platform_get_drvdata(pdev); + + debugfs_remove_recursive(ddev->root_dir); + if (ddev->test_dev) + of_platform_device_destroy(ddev->test_dev, NULL); + + mutex_destroy(&ddev->state_lock); + return 0; +} + +static const struct of_device_id iommu_debug_of_match[] = { + { .compatible = "qcom,iommu-debug-test" }, + { }, +}; + +static struct platform_driver iommu_debug_driver = { + .probe = iommu_debug_probe, + .remove = iommu_debug_remove, + .driver = { + .name = "qcom-iommu-debug", + .of_match_table = iommu_debug_of_match, + }, +}; + +/* + * This isn't really a "driver", we just need something in the device tree + * to hook up to the `iommus' property. + */ +static int iommu_debug_usecase_probe(struct platform_device *pdev) +{ + return iommu_debug_usecase_register(&pdev->dev); +} + +static const struct of_device_id iommu_debug_usecase_of_match[] = { + { .compatible = "qcom,iommu-debug-usecase" }, + { }, +}; + +static struct platform_driver iommu_debug_usecase_driver = { + .probe = iommu_debug_usecase_probe, + .driver = { + .name = "qcom-iommu-debug-usecase", + .of_match_table = iommu_debug_usecase_of_match, + }, +}; + +static int iommu_debug_init(void) +{ + int ret; + + ret = platform_driver_register(&iommu_debug_driver); + if (ret) + return ret; + + ret = platform_driver_register(&iommu_debug_usecase_driver); + if (ret) + platform_driver_unregister(&iommu_debug_driver); + return ret; +} + +static void iommu_debug_exit(void) +{ + platform_driver_unregister(&iommu_debug_usecase_driver); + platform_driver_unregister(&iommu_debug_driver); +} + +module_init(iommu_debug_init); +module_exit(iommu_debug_exit); + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/qcom-iommu-debug.h b/drivers/iommu/qcom-iommu-debug.h new file mode 100644 index 000000000000..41e06082e01b --- /dev/null +++ b/drivers/iommu/qcom-iommu-debug.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2015-2021, The Linux Foundation. All rights reserved. + * + */ + +#ifndef __DRIVERS_IOMMU_QCOM_IOMMU_DEBUG_H__ +#define __DRIVERS_IOMMU_QCOM_IOMMU_DEBUG_H__ + +#include +#include +#include +#include +#include + +#define MSI_IOVA_BASE 0x8000000 +#define MSI_IOVA_LENGTH 0x100000 +#define ARM_SMMU_SMR_ID GENMASK(15, 0) + +struct iommu_debug_device { + struct device *self; + u32 nr_children; + char *buffer; + struct dentry *root_dir; + /* for usecase under test */ + struct device *test_dev; + struct iommu_domain *domain; + u32 usecase_nr; + /* Protects test_dev */ + struct mutex state_lock; + /* For waiting for child probe to complete */ + struct completion probe_wait; + /* Used for atos */ + u64 iova; + /* number of iterations */ + u32 nr_iters; +}; + +struct device *iommu_debug_usecase_reset(struct iommu_debug_device *ddev); +struct device *iommu_debug_switch_usecase(struct iommu_debug_device *ddev, u32 usecase_nr); + +int iommu_debug_check_mapping_flags(struct device *dev, dma_addr_t iova, size_t size, + phys_addr_t expected_pa, u32 flags); +#define iommu_debug_check_mapping(d, i, s, p) \ + iommu_debug_check_mapping_flags(d, i, s, p, 0) +/* Only checks a single page */ +#define iommu_debug_check_mapping_fast(d, i, s, p) \ + iommu_debug_check_mapping_flags(d, i, PAGE_SIZE, p, 0) + +int iommu_debug_check_mapping_sg_flags(struct device *dev, struct scatterlist *sgl, + unsigned int pgoffset, unsigned int dma_nents, + unsigned int nents, u32 flags); +#define iommu_debug_check_mapping_sg(d, s, o, e1, e2) \ + iommu_debug_check_mapping_sg_flags(d, s, o, e1, e2, 0) + +/* Only checks the last page of first sgl */ +static inline int iommu_debug_check_mapping_sg_fast(struct device *dev, struct scatterlist *sgl, + unsigned int pgoffset, unsigned int dma_nents, + unsigned int nents) +{ + pgoffset = PAGE_ALIGN(sgl->offset + sgl->length) >> PAGE_SHIFT; + return iommu_debug_check_mapping_sg_flags(dev, sgl, pgoffset - 1, dma_nents, 1, 0); +} + +extern const struct file_operations iommu_debug_functional_arm_dma_api_fops; +extern const struct file_operations iommu_debug_functional_fast_dma_api_fops; +extern const struct file_operations iommu_debug_atos_fops; +extern const struct file_operations iommu_debug_map_fops; +extern const struct file_operations iommu_debug_unmap_fops; +extern const struct file_operations iommu_debug_dma_map_fops; +extern const struct file_operations iommu_debug_dma_unmap_fops; +extern const struct file_operations iommu_debug_test_virt_addr_fops; +extern const struct file_operations iommu_debug_profiling_fops; + +#endif diff --git a/drivers/iommu/qcom-iommu-util.c b/drivers/iommu/qcom-iommu-util.c new file mode 100644 index 000000000000..3fa9390bfba3 --- /dev/null +++ b/drivers/iommu/qcom-iommu-util.c @@ -0,0 +1,529 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include "qcom-dma-iommu-generic.h" +#include "qcom-io-pgtable.h" +#include "qcom-io-pgtable-alloc.h" + +struct qcom_iommu_range_prop_cb_data { + int (*range_prop_entry_cb_fn)(const __be32 *p, int naddr, int nsize, void *arg); + void *arg; +}; + +struct iova_range { + u64 base; + u64 end; +}; + +struct device_node *qcom_iommu_group_parse_phandle(struct device *dev) +{ + struct device_node *np; + + if (!dev->of_node) + return NULL; + + np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); + return np ? np : dev->of_node; +} + +static int of_property_walk_each_entry(struct device *dev, const char *propname, + struct qcom_iommu_range_prop_cb_data *cb_data) +{ + struct device_node *np; + const __be32 *p, *property_end; + int ret, len, naddr, nsize; + + np = qcom_iommu_group_parse_phandle(dev); + if (!np) + return -EINVAL; + + p = of_get_property(np, propname, &len); + if (!p) + return -ENODEV; + + len /= sizeof(u32); + naddr = of_n_addr_cells(np); + nsize = of_n_size_cells(np); + if (!naddr || !nsize || len % (naddr + nsize)) { + dev_err(dev, "%s Invalid length %d. Address cells %d. Size cells %d\n", + propname, len, naddr, nsize); + return -EINVAL; + } + property_end = p + len; + + while (p < property_end) { + ret = cb_data->range_prop_entry_cb_fn(p, naddr, nsize, cb_data->arg); + if (ret) + return ret; + + p += naddr + nsize; + } + + return 0; +} + +static bool check_overlap(struct iommu_resv_region *region, u64 start, u64 end) +{ + u64 region_end = region->start + region->length - 1; + + return end >= region->start && start <= region_end; +} + +static int insert_range(const __be32 *p, int naddr, int nsize, void *arg) +{ + struct list_head *head = arg; + struct iommu_resv_region *region, *new; + u64 start = of_read_number(p, naddr); + u64 end = start + of_read_number(p + naddr, nsize) - 1; + + list_for_each_entry(region, head, list) { + if (check_overlap(region, start, end)) + return -EINVAL; + + if (start < region->start) + break; + } + + new = iommu_alloc_resv_region(start, end - start + 1, + 0, IOMMU_RESV_RESERVED); + if (!new) + return -ENOMEM; + list_add_tail(&new->list, ®ion->list); + return 0; +} + +/* + * Returns a sorted list of all regions described by the + * "qcom,iommu-dma-addr-pool" property. + * + * Caller is responsible for freeing the entries on the list via + * generic_iommu_put_resv_regions + */ +int qcom_iommu_generate_dma_regions(struct device *dev, + struct list_head *head) +{ + struct qcom_iommu_range_prop_cb_data insert_range_cb_data = { + .range_prop_entry_cb_fn = insert_range, + .arg = head, + }; + + return of_property_walk_each_entry(dev, "qcom,iommu-dma-addr-pool", + &insert_range_cb_data); +} +EXPORT_SYMBOL(qcom_iommu_generate_dma_regions); + +static int invert_regions(struct list_head *head, struct list_head *inverted) +{ + struct iommu_resv_region *prev, *curr, *new; + phys_addr_t rsv_start; + size_t rsv_size; + int ret = 0; + + /* + * Since its not possible to express start 0, size 1<<64 return + * an error instead. Also an iova allocator without any iovas doesn't + * make sense. + */ + if (list_empty(head)) + return -EINVAL; + + /* + * Handle case where there is a non-zero sized area between + * iommu_resv_regions A & B. + */ + prev = NULL; + list_for_each_entry(curr, head, list) { + if (!prev) + goto next; + + rsv_start = prev->start + prev->length; + rsv_size = curr->start - rsv_start; + if (!rsv_size) + goto next; + + new = iommu_alloc_resv_region(rsv_start, rsv_size, + 0, IOMMU_RESV_RESERVED); + if (!new) { + ret = -ENOMEM; + goto out_err; + } + list_add_tail(&new->list, inverted); +next: + prev = curr; + } + + /* Now handle the beginning */ + curr = list_first_entry(head, struct iommu_resv_region, list); + rsv_start = 0; + rsv_size = curr->start; + if (rsv_size) { + new = iommu_alloc_resv_region(rsv_start, rsv_size, + 0, IOMMU_RESV_RESERVED); + if (!new) { + ret = -ENOMEM; + goto out_err; + } + list_add(&new->list, inverted); + } + + /* Handle the end - checking for overflow */ + rsv_start = prev->start + prev->length; + rsv_size = -rsv_start; + + if (rsv_size && (U64_MAX - prev->start > prev->length)) { + new = iommu_alloc_resv_region(rsv_start, rsv_size, + 0, IOMMU_RESV_RESERVED); + if (!new) { + ret = -ENOMEM; + goto out_err; + } + list_add_tail(&new->list, inverted); + } + + return 0; + +out_err: + list_for_each_entry_safe(curr, prev, inverted, list) + kfree(curr); + return ret; +} + +/* Used by iommu drivers to generate reserved regions for qcom,iommu-dma-addr-pool property */ +void qcom_iommu_generate_resv_regions(struct device *dev, + struct list_head *head) +{ + struct iommu_resv_region *region; + LIST_HEAD(dma_regions); + LIST_HEAD(resv_regions); + int ret; + + ret = qcom_iommu_generate_dma_regions(dev, &dma_regions); + if (ret) + return; + + ret = invert_regions(&dma_regions, &resv_regions); + generic_iommu_put_resv_regions(dev, &dma_regions); + if (ret) + return; + + list_for_each_entry(region, &resv_regions, list) { + dev_dbg(dev, "Reserved region %llx-%llx\n", + (u64)region->start, + (u64)(region->start + region->length - 1)); + } + + list_splice(&resv_regions, head); +} +EXPORT_SYMBOL(qcom_iommu_generate_resv_regions); + +void qcom_iommu_get_resv_regions(struct device *dev, struct list_head *list) +{ + const struct iommu_ops *ops = dev->bus->iommu_ops; + + if (ops && ops->get_resv_regions) + ops->get_resv_regions(dev, list); +} +EXPORT_SYMBOL(qcom_iommu_get_resv_regions); + +void qcom_iommu_put_resv_regions(struct device *dev, struct list_head *list) +{ + const struct iommu_ops *ops = dev->bus->iommu_ops; + + if (ops && ops->put_resv_regions) + ops->put_resv_regions(dev, list); +} +EXPORT_SYMBOL(qcom_iommu_put_resv_regions); + +static int get_addr_range(const __be32 *p, int naddr, int nsize, void *arg) +{ + + u64 start = of_read_number(p, naddr); + u64 end = start + of_read_number(p + naddr, nsize) - 1; + struct iova_range *range = arg; + + if (start >= SZ_4G || end >= SZ_4G) { + pr_err("fastmap does not support IOVAs >= 4 GB\n"); + return -EINVAL; + } + + range->base = min_not_zero(range->base, start); + range->end = max(range->end, end); + + return 0; +} + +int qcom_iommu_get_fast_iova_range(struct device *dev, dma_addr_t *ret_iova_base, + dma_addr_t *ret_iova_end) +{ + struct iova_range dma_range = {}; + struct iova_range geometry_range = {}; + struct qcom_iommu_range_prop_cb_data get_addr_range_cb_data = { + .range_prop_entry_cb_fn = get_addr_range, + }; + int ret; + + if (!dev || !ret_iova_base || !ret_iova_end) + return -EINVAL; + + get_addr_range_cb_data.arg = &dma_range; + ret = of_property_walk_each_entry(dev, "qcom,iommu-dma-addr-pool", + &get_addr_range_cb_data); + if (ret == -ENODEV) { + dma_range.base = 0; + dma_range.end = SZ_4G - 1; + } else if (ret) { + return ret; + } + + get_addr_range_cb_data.arg = &geometry_range; + ret = of_property_walk_each_entry(dev, "qcom,iommu-geometry", + &get_addr_range_cb_data); + if (ret == -ENODEV) { + geometry_range.base = 0; + geometry_range.end = SZ_4G - 1; + } else if (ret) { + return ret; + } + + *ret_iova_base = min(geometry_range.base, dma_range.base); + *ret_iova_end = max(geometry_range.end, dma_range.end); + return 0; +} +EXPORT_SYMBOL(qcom_iommu_get_fast_iova_range); + +phys_addr_t qcom_iommu_iova_to_phys_hard(struct iommu_domain *domain, + struct qcom_iommu_atos_txn *txn) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (unlikely(ops->iova_to_phys_hard == NULL)) + return 0; + + return ops->iova_to_phys_hard(domain, txn); +} +EXPORT_SYMBOL(qcom_iommu_iova_to_phys_hard); + +int qcom_iommu_sid_switch(struct device *dev, enum sid_switch_direction dir) +{ + struct qcom_iommu_ops *ops; + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return -EINVAL; + + ops = to_qcom_iommu_ops(domain->ops); + if (unlikely(ops->sid_switch == NULL)) + return -EINVAL; + + return ops->sid_switch(dev, dir); +} +EXPORT_SYMBOL(qcom_iommu_sid_switch); + +int qcom_iommu_get_fault_ids(struct iommu_domain *domain, + struct qcom_iommu_fault_ids *f_ids) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (unlikely(ops->get_fault_ids == NULL)) + return -EINVAL; + + return ops->get_fault_ids(domain, f_ids); +} +EXPORT_SYMBOL(qcom_iommu_get_fault_ids); + +int qcom_iommu_get_msi_size(struct device *dev, u32 *msi_size) +{ + struct device_node *np = qcom_iommu_group_parse_phandle(dev); + + if (!np) + return -EINVAL; + + return of_property_read_u32(np, "qcom,iommu-msi-size", msi_size); +} + +int qcom_iommu_get_context_bank_nr(struct iommu_domain *domain) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (unlikely(ops->get_context_bank_nr == NULL)) + return -EINVAL; + + return ops->get_context_bank_nr(domain); +} +EXPORT_SYMBOL(qcom_iommu_get_context_bank_nr); + +int qcom_iommu_set_secure_vmid(struct iommu_domain *domain, enum vmid vmid) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (unlikely(ops->set_secure_vmid == NULL)) + return -EINVAL; + + return ops->set_secure_vmid(domain, vmid); +} +EXPORT_SYMBOL(qcom_iommu_set_secure_vmid); + +int qcom_iommu_set_fault_model(struct iommu_domain *domain, int fault_model) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (unlikely(ops->set_fault_model == NULL)) + return -EINVAL; + else if (fault_model & ~(QCOM_IOMMU_FAULT_MODEL_NON_FATAL | + QCOM_IOMMU_FAULT_MODEL_NO_CFRE | + QCOM_IOMMU_FAULT_MODEL_NO_STALL | + QCOM_IOMMU_FAULT_MODEL_HUPCF)) + return -EINVAL; + + return ops->set_fault_model(domain, fault_model); +} +EXPORT_SYMBOL(qcom_iommu_set_fault_model); + +int qcom_iommu_enable_s1_translation(struct iommu_domain *domain) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (unlikely(ops->enable_s1_translation == NULL)) + return -EINVAL; + + return ops->enable_s1_translation(domain); +} +EXPORT_SYMBOL(qcom_iommu_enable_s1_translation); + +int qcom_iommu_get_mappings_configuration(struct iommu_domain *domain) +{ + struct qcom_iommu_ops *ops = to_qcom_iommu_ops(domain->ops); + + if (unlikely(ops->get_mappings_configuration == NULL)) + return -EINVAL; + + return ops->get_mappings_configuration(domain); +} +EXPORT_SYMBOL(qcom_iommu_get_mappings_configuration); + +struct io_pgtable_ops *qcom_alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, + struct qcom_io_pgtable_info *pgtbl_info, + void *cookie) +{ + struct io_pgtable *iop; + const struct io_pgtable_init_fns *fns; + struct io_pgtable_cfg *cfg = &pgtbl_info->cfg; + + if (fmt < IO_PGTABLE_NUM_FMTS) + return alloc_io_pgtable_ops(fmt, cfg, cookie); +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + else if (fmt == ARM_V8L_FAST) + fns = &io_pgtable_av8l_fast_init_fns; +#endif +#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE + else if (fmt == QCOM_ARM_64_LPAE_S1) + fns = &qcom_io_pgtable_arm_64_lpae_s1_init_fns; +#endif + else { + pr_err("Invalid io-pgtable fmt %u\n", fmt); + return NULL; + } + + iop = fns->alloc(cfg, cookie); + if (!iop) + return NULL; + + iop->fmt = fmt; + iop->cookie = cookie; + iop->cfg = *cfg; + + return &iop->ops; +} +EXPORT_SYMBOL(qcom_alloc_io_pgtable_ops); + +void qcom_free_io_pgtable_ops(struct io_pgtable_ops *ops) +{ + struct io_pgtable *iop; + enum io_pgtable_fmt fmt; + const struct io_pgtable_init_fns *fns; + + if (!ops) + return; + + iop = io_pgtable_ops_to_pgtable(ops); + fmt = iop->fmt; + if (fmt < IO_PGTABLE_NUM_FMTS) + return free_io_pgtable_ops(ops); +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + else if (fmt == ARM_V8L_FAST) + fns = &io_pgtable_av8l_fast_init_fns; +#endif +#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE + else if (fmt == QCOM_ARM_64_LPAE_S1) + fns = &qcom_io_pgtable_arm_64_lpae_s1_init_fns; +#endif + else { + pr_err("Invalid io-pgtable fmt %u\n", fmt); + return; + } + + io_pgtable_tlb_flush_all(iop); + fns->free(iop); +} +EXPORT_SYMBOL(qcom_free_io_pgtable_ops); + +/* + * These tables must have the same length. + * It is allowed to have a NULL exitcall corresponding to a non-NULL initcall. + */ +static initcall_t init_table[] __initdata = { + dma_mapping_fast_init, + qcom_dma_iommu_generic_driver_init, + qcom_arm_lpae_do_selftests, + qcom_io_pgtable_alloc_init, + NULL +}; + +static exitcall_t exit_table[] = { + NULL, /* dma_mapping_fast_exit */ + qcom_dma_iommu_generic_driver_exit, + NULL, /*qcom_arm_lpae_do_selftests */ + qcom_io_pgtable_alloc_exit, + NULL, +}; + +static int __init qcom_iommu_util_init(void) +{ + initcall_t *init_fn; + exitcall_t *exit_fn; + int ret; + + if (ARRAY_SIZE(init_table) != ARRAY_SIZE(exit_table)) { + pr_err("qcom-iommu-util: Invalid initcall/exitcall table\n"); + return -EINVAL; + } + + for (init_fn = init_table; *init_fn; init_fn++) { + ret = (**init_fn)(); + if (ret) { + pr_err("%ps returned %d\n", *init_fn, ret); + goto out_undo; + } + } + + return 0; + +out_undo: + exit_fn = exit_table + (init_fn - init_table); + for (exit_fn--; exit_fn >= exit_table; exit_fn--) { + if (!*exit_fn) + continue; + (**exit_fn)(); + } + return ret; +} +module_init(qcom_iommu_util_init); + +MODULE_LICENSE("GPL v2"); diff --git a/include/linux/dma-mapping-fast.h b/include/linux/dma-mapping-fast.h new file mode 100644 index 000000000000..6ee5406ef3e9 --- /dev/null +++ b/include/linux/dma-mapping-fast.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2016-2019, 2021, The Linux Foundation. All rights reserved. + */ + +#ifndef __LINUX_DMA_MAPPING_FAST_H +#define __LINUX_DMA_MAPPING_FAST_H + +#include +#include +#include + +struct dma_iommu_mapping; +struct io_pgtable_ops; +struct iova_domain; + +struct dma_fast_smmu_mapping { + struct device *dev; + struct iommu_domain *domain; + struct iova_domain *iovad; + + dma_addr_t base; + size_t size; + size_t num_4k_pages; + + unsigned int bitmap_size; + /* bitmap has 1s marked only valid mappings */ + unsigned long *bitmap; + /* clean_bitmap has 1s marked for both valid and stale tlb mappings */ + unsigned long *clean_bitmap; + unsigned long next_start; + bool have_stale_tlbs; + + dma_addr_t pgtbl_dma_handle; + struct io_pgtable_ops *pgtbl_ops; + + spinlock_t lock; + struct notifier_block notifier; + struct rb_node node; +}; + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST +int fast_smmu_init_mapping(struct device *dev, struct iommu_domain *domain, + struct io_pgtable_ops *pgtable_ops); +void fast_smmu_put_dma_cookie(struct iommu_domain *domain); +void fast_smmu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size); +int __init dma_mapping_fast_init(void); +void dma_mapping_fast_exit(void); +#else +static inline int fast_smmu_init_mapping(struct device *dev, + struct iommu_domain *domain, + struct io_pgtable_ops *pgtable_ops) +{ + return -ENODEV; +} + +static inline void fast_smmu_put_dma_cookie(struct iommu_domain *domain) {} +static inline void fast_smmu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size) {} + +static inline int __init dma_mapping_fast_init(void) +{ + return 0; +} + +static inline void dma_mapping_fast_exit(void) {} +#endif + +#endif /* __LINUX_DMA_MAPPING_FAST_H */ diff --git a/include/linux/io-pgtable-fast.h b/include/linux/io-pgtable-fast.h new file mode 100644 index 000000000000..5adff8ab891e --- /dev/null +++ b/include/linux/io-pgtable-fast.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef __LINUX_IO_PGTABLE_FAST_H +#define __LINUX_IO_PGTABLE_FAST_H + +#include +#include +/* + * This ought to be private to io-pgtable-fast, but dma-mapping-fast + * currently requires it for a debug usecase. + */ +typedef u64 av8l_fast_iopte; + +struct io_pgtable_ops; +struct scatterlist; + +struct av8l_fast_io_pgtable { + struct io_pgtable iop; + av8l_fast_iopte *pgd; + av8l_fast_iopte *puds[4]; + av8l_fast_iopte *pmds; + struct page **pages; /* page table memory */ + int nr_pages; + dma_addr_t base; + dma_addr_t end; +}; + +/* Struct accessors */ +#define iof_pgtable_to_data(x) \ + container_of((x), struct av8l_fast_io_pgtable, iop) + +#define iof_pgtable_ops_to_pgtable(x) \ + container_of((x), struct io_pgtable, ops) + +#define iof_pgtable_ops_to_data(x) \ + iof_pgtable_to_data(iof_pgtable_ops_to_pgtable(x)) + + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST + +int av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot); + +void av8l_fast_unmap_public(struct io_pgtable_ops *ops, unsigned long iova, + size_t size); + +int av8l_fast_map_sg_public(struct io_pgtable_ops *ops, + unsigned long iova, struct scatterlist *sgl, + unsigned int nents, int prot, size_t *size); + +bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops, + unsigned long iova); + +phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops, + unsigned long iova); +#else +static inline int +av8l_fast_map_public(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + return -EINVAL; +} +static inline void av8l_fast_unmap_public(struct io_pgtable_ops *ops, + unsigned long iova, size_t size) +{ +} + +static inline int av8l_fast_map_sg_public(struct io_pgtable_ops *ops, + unsigned long iova, struct scatterlist *sgl, + unsigned int nents, int prot, size_t *size) +{ + return 0; +} + +static inline bool av8l_fast_iova_coherent_public(struct io_pgtable_ops *ops, + unsigned long iova) +{ + return false; +} +static inline phys_addr_t +av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops, + unsigned long iova) +{ + return 0; +} +#endif /* CONFIG_IOMMU_IO_PGTABLE_FAST */ + + +/* events for notifiers passed to av8l_register_notify */ +#define MAPPED_OVER_STALE_TLB 1 + + +#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB +/* + * Doesn't matter what we use as long as bit 0 is unset. The reason why we + * need a different value at all is that there are certain hardware + * platforms with erratum that require that a PTE actually be zero'd out + * and not just have its valid bit unset. + */ +#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0xa + +void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, u64 base, u64 end, + bool skip_sync); +void av8l_register_notify(struct notifier_block *nb); + +#else /* !CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */ + +#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0 + +static inline void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, + u64 base, + u64 end, + bool skip_sync) +{ +} + +static inline void av8l_register_notify(struct notifier_block *nb) +{ +} + +#endif /* CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */ + +#endif /* __LINUX_IO_PGTABLE_FAST_H */ diff --git a/include/linux/qcom-iommu-util.h b/include/linux/qcom-iommu-util.h new file mode 100644 index 000000000000..291e91f52e21 --- /dev/null +++ b/include/linux/qcom-iommu-util.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef __QCOM_IOMMU_UTIL_H +#define __QCOM_IOMMU_UTIL_H + +#include +#include +#include + +#include + +/* IOMMU fault behaviors */ +#define QCOM_IOMMU_FAULT_MODEL_NON_FATAL BIT(0) +#define QCOM_IOMMU_FAULT_MODEL_NO_CFRE BIT(1) +#define QCOM_IOMMU_FAULT_MODEL_NO_STALL BIT(2) +#define QCOM_IOMMU_FAULT_MODEL_HUPCF BIT(3) + +/* IOMMU mapping configurations */ +#define QCOM_IOMMU_MAPPING_CONF_S1_BYPASS BIT(0) +#define QCOM_IOMMU_MAPPING_CONF_ATOMIC BIT(1) +#define QCOM_IOMMU_MAPPING_CONF_FAST BIT(2) + +/* iommu transaction flags */ +/* 1 Write, 0 Read */ +#define QCOM_IOMMU_ATOS_TRANS_WRITE BIT(0) +/* 1 Privileged, 0 Unprivileged */ +#define QCOM_IOMMU_ATOS_TRANS_PRIV BIT(1) +/* 1 Instruction fetch, 0 Data access */ +#define QCOM_IOMMU_ATOS_TRANS_INST BIT(2) +/* Non secure unprivileged Data read operation */ +#define QCOM_IOMMU_ATOS_TRANS_DEFAULT (0U) + +#ifndef IOMMU_SYS_CACHE +/* Attributes are not supported, so render them ineffective. */ +#define IOMMU_SYS_CACHE (0) +#define IOMMU_SYS_CACHE_NWA (0) +#endif + +/* Use upstream device's bus attribute */ +#define IOMMU_USE_UPSTREAM_HINT (IOMMU_SYS_CACHE) + +/* Use upstream device's bus attribute with no write-allocate cache policy */ +#define IOMMU_USE_LLC_NWA (IOMMU_SYS_CACHE_NWA) + +/* vendor iommu fault flags */ +#define IOMMU_FAULT_TRANSLATION (1 << 2) +#define IOMMU_FAULT_PERMISSION (1 << 3) +#define IOMMU_FAULT_EXTERNAL (1 << 4) +#define IOMMU_FAULT_TRANSACTION_STALLED (1 << 5) + +/* iommu transaction flags */ +#define IOMMU_TRANS_WRITE BIT(0) /* 1 Write, 0 Read */ +#define IOMMU_TRANS_PRIV BIT(1) /* 1 Privileged, 0 Unprivileged */ +#define IOMMU_TRANS_INST BIT(2) /* 1 Instruction fetch, 0 Data access */ +#define IOMMU_TRANS_SEC BIT(3) /* 1 Secure, 0 Non-secure access*/ + +/* Non secure unprivileged Data read operation */ +#define IOMMU_TRANS_DEFAULT (0U) + +struct iommu_pgtbl_info { + void *ops; +}; + +struct qcom_iommu_atos_txn { + u64 addr; + u32 flags; + u32 id; +}; + +enum sid_switch_direction { + SID_ACQUIRE, + SID_RELEASE, +}; + +struct qcom_iommu_fault_ids { + u32 bid; + u32 pid; + u32 mid; +}; + +/* + * @sid_switch: add/remove all SIDS in the iommu domain containing dev from + * iommu registers. + */ +struct qcom_iommu_ops { + phys_addr_t (*iova_to_phys_hard)(struct iommu_domain *domain, + struct qcom_iommu_atos_txn *txn); + int (*sid_switch)(struct device *dev, enum sid_switch_direction dir); + int (*get_fault_ids)(struct iommu_domain *domain, + struct qcom_iommu_fault_ids *ids); + int (*get_context_bank_nr)(struct iommu_domain *domain); + int (*set_secure_vmid)(struct iommu_domain *domain, enum vmid vmid); + int (*set_fault_model)(struct iommu_domain *domain, int fault_model); + int (*enable_s1_translation)(struct iommu_domain *domain); + int (*get_mappings_configuration)(struct iommu_domain *domain); + struct iommu_ops iommu_ops; +}; +#define to_qcom_iommu_ops(x) (container_of(x, struct qcom_iommu_ops, iommu_ops)) + +struct device_node *qcom_iommu_group_parse_phandle(struct device *dev); +int qcom_iommu_generate_dma_regions(struct device *dev, + struct list_head *head); +void qcom_iommu_generate_resv_regions(struct device *dev, + struct list_head *list); +int qcom_iommu_get_fast_iova_range(struct device *dev, + dma_addr_t *ret_iova_base, + dma_addr_t *ret_iova_end); + +/* Remove once these functions are exported by upstream kernel */ +void qcom_iommu_get_resv_regions(struct device *dev, struct list_head *list); +void qcom_iommu_put_resv_regions(struct device *dev, struct list_head *list); + +phys_addr_t qcom_iommu_iova_to_phys_hard(struct iommu_domain *domain, + struct qcom_iommu_atos_txn *txn); + +int qcom_iommu_sid_switch(struct device *dev, enum sid_switch_direction dir); + +extern int qcom_iommu_get_fault_ids(struct iommu_domain *domain, + struct qcom_iommu_fault_ids *f_ids); +extern int qcom_iommu_get_msi_size(struct device *dev, u32 *msi_size); + +int qcom_iommu_get_context_bank_nr(struct iommu_domain *domain); + +int qcom_iommu_set_secure_vmid(struct iommu_domain *domain, enum vmid vmid); + +int qcom_iommu_set_fault_model(struct iommu_domain *domain, int fault_model); + +int qcom_iommu_enable_s1_translation(struct iommu_domain *domain); + +int qcom_iommu_get_mappings_configuration(struct iommu_domain *domain); + +#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE +int __init qcom_arm_lpae_do_selftests(void); +#else +static inline int __init qcom_arm_lpae_do_selftests(void) +{ + return 0; +} +#endif +#endif /* __QCOM_IOMMU_UTIL_H */