diff --git a/drivers/iommu/arm/arm-smmu/Makefile b/drivers/iommu/arm/arm-smmu/Makefile index 1339ab877546..b0cc01aa20c9 100644 --- a/drivers/iommu/arm/arm-smmu/Makefile +++ b/drivers/iommu/arm/arm-smmu/Makefile @@ -2,4 +2,4 @@ obj-$(CONFIG_QCOM_IOMMU) += qcom_iommu.o obj-$(CONFIG_ARM_SMMU) += arm_smmu.o arm_smmu-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-nvidia.o -arm_smmu-$(CONFIG_ARM_SMMU_QCOM) += arm-smmu-qcom.o arm-smmu-qcom-pm.o +arm_smmu-$(CONFIG_ARM_SMMU_QCOM) += arm-smmu-qcom.o diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c index 9790cb3e160e..2c25cce38060 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c @@ -215,9 +215,6 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) of_device_is_compatible(np, "nvidia,tegra186-smmu")) return nvidia_smmu_impl_init(smmu); - if (of_device_is_compatible(smmu->dev->of_node, "qcom,qsmmu-v500")) - return qsmmuv500_impl_init(smmu); - if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM)) smmu = qcom_smmu_impl_init(smmu); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-pm.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-pm.c deleted file mode 100644 index b2218da12453..000000000000 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-pm.c +++ /dev/null @@ -1,336 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (c) 2021, The Linux Foundation. All rights reserved. - */ -#include -#include -#include -#include -#include -#include "arm-smmu.h" - -#define ARM_SMMU_ICC_AVG_BW 0 -#define ARM_SMMU_ICC_PEAK_BW_HIGH 1000 -#define ARM_SMMU_ICC_PEAK_BW_LOW 0 -#define ARM_SMMU_ICC_ACTIVE_ONLY_TAG 0x3 - -/* - * Theoretically, our interconnect does not guarantee the order between - * writes to different "register blocks" even with device memory type. - * It does guarantee that the completion of a read to a particular - * register block implies that previously issued writes to that - * register block have completed, with device memory type. - * - * In particular, we need to ensure that writes to iommu registers - * complete before we turn off the power. - */ -static void arm_smmu_arch_write_sync(struct arm_smmu_device *smmu) -{ - u32 id; - - if (!smmu) - return; - - /* Read to complete prior write transcations */ - id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0); - - /* Wait for read to complete before off */ - rmb(); -} - -static int arm_smmu_prepare_clocks(struct arm_smmu_power_resources *pwr) -{ - - int i, ret = 0; - - for (i = 0; i < pwr->num_clocks; ++i) { - ret = clk_prepare(pwr->clocks[i]); - if (ret) { - dev_err(pwr->dev, "Couldn't prepare clock #%d\n", i); - while (i--) - clk_unprepare(pwr->clocks[i]); - break; - } - } - return ret; -} - -static void arm_smmu_unprepare_clocks(struct arm_smmu_power_resources *pwr) -{ - int i; - - for (i = pwr->num_clocks; i; --i) - clk_unprepare(pwr->clocks[i - 1]); -} - -static int arm_smmu_enable_clocks(struct arm_smmu_power_resources *pwr) -{ - int i, ret = 0; - - for (i = 0; i < pwr->num_clocks; ++i) { - ret = clk_enable(pwr->clocks[i]); - if (ret) { - dev_err(pwr->dev, "Couldn't enable clock #%d\n", i); - while (i--) - clk_disable(pwr->clocks[i]); - break; - } - } - - return ret; -} - -static void arm_smmu_disable_clocks(struct arm_smmu_power_resources *pwr) -{ - int i; - - for (i = pwr->num_clocks; i; --i) - clk_disable(pwr->clocks[i - 1]); -} - -static int arm_smmu_raise_interconnect_bw(struct arm_smmu_power_resources *pwr) -{ - if (!pwr->icc_path) - return 0; - return icc_set_bw(pwr->icc_path, ARM_SMMU_ICC_AVG_BW, - ARM_SMMU_ICC_PEAK_BW_HIGH); -} - -static void arm_smmu_lower_interconnect_bw(struct arm_smmu_power_resources *pwr) -{ - if (!pwr->icc_path) - return; - WARN_ON(icc_set_bw(pwr->icc_path, ARM_SMMU_ICC_AVG_BW, - ARM_SMMU_ICC_PEAK_BW_LOW)); -} - -static int arm_smmu_enable_regulators(struct arm_smmu_power_resources *pwr) -{ - struct regulator_bulk_data *consumers; - int num_consumers, ret; - int i; - - num_consumers = pwr->num_gdscs; - consumers = pwr->gdscs; - for (i = 0; i < num_consumers; i++) { - ret = regulator_enable(consumers[i].consumer); - if (ret) - goto out; - } - return 0; - -out: - i -= 1; - for (; i >= 0; i--) - regulator_disable(consumers[i].consumer); - return ret; -} - -int arm_smmu_power_on(struct arm_smmu_power_resources *pwr) -{ - int ret; - - mutex_lock(&pwr->power_lock); - if (pwr->power_count > 0) { - pwr->power_count += 1; - mutex_unlock(&pwr->power_lock); - return 0; - } - - ret = arm_smmu_raise_interconnect_bw(pwr); - if (ret) - goto out_unlock; - - ret = arm_smmu_enable_regulators(pwr); - if (ret) - goto out_disable_bus; - - ret = arm_smmu_prepare_clocks(pwr); - if (ret) - goto out_disable_regulators; - - ret = arm_smmu_enable_clocks(pwr); - if (ret) - goto out_unprepare_clocks; - - if (pwr->resume) { - ret = pwr->resume(pwr); - if (ret) - goto out_disable_clocks; - } - - pwr->power_count = 1; - mutex_unlock(&pwr->power_lock); - return 0; -out_disable_clocks: - arm_smmu_disable_clocks(pwr); -out_unprepare_clocks: - arm_smmu_unprepare_clocks(pwr); -out_disable_regulators: - regulator_bulk_disable(pwr->num_gdscs, pwr->gdscs); -out_disable_bus: - arm_smmu_lower_interconnect_bw(pwr); -out_unlock: - mutex_unlock(&pwr->power_lock); - return ret; -} - -/* - * Needing to pass smmu to this api for arm_smmu_arch_write_sync is awkward. - */ -void arm_smmu_power_off(struct arm_smmu_device *smmu, - struct arm_smmu_power_resources *pwr) -{ - mutex_lock(&pwr->power_lock); - if (pwr->power_count == 0) { - WARN(1, "%s: Bad power count\n", dev_name(pwr->dev)); - mutex_unlock(&pwr->power_lock); - return; - - } else if (pwr->power_count > 1) { - pwr->power_count--; - mutex_unlock(&pwr->power_lock); - return; - } - - if (pwr->suspend) - pwr->suspend(pwr); - - arm_smmu_arch_write_sync(smmu); - arm_smmu_disable_clocks(pwr); - arm_smmu_unprepare_clocks(pwr); - regulator_bulk_disable(pwr->num_gdscs, pwr->gdscs); - arm_smmu_lower_interconnect_bw(pwr); - pwr->power_count = 0; - mutex_unlock(&pwr->power_lock); -} - -static int arm_smmu_init_clocks(struct arm_smmu_power_resources *pwr) -{ - const char *cname; - struct property *prop; - int i; - struct device *dev = pwr->dev; - - pwr->num_clocks = - of_property_count_strings(dev->of_node, "clock-names"); - - if (pwr->num_clocks < 1) { - pwr->num_clocks = 0; - return 0; - } - - pwr->clocks = devm_kzalloc( - dev, sizeof(*pwr->clocks) * pwr->num_clocks, - GFP_KERNEL); - - if (!pwr->clocks) - return -ENOMEM; - - i = 0; - of_property_for_each_string(dev->of_node, "clock-names", - prop, cname) { - struct clk *c = devm_clk_get(dev, cname); - - if (IS_ERR(c)) { - dev_err(dev, "Couldn't get clock: %s\n", - cname); - return PTR_ERR(c); - } - - if (clk_get_rate(c) == 0) { - long rate = clk_round_rate(c, 1000); - - clk_set_rate(c, rate); - } - - pwr->clocks[i] = c; - - ++i; - } - return 0; -} - -static int arm_smmu_init_regulators(struct arm_smmu_power_resources *pwr) -{ - const char *cname; - struct property *prop; - int i; - struct device *dev = pwr->dev; - - pwr->num_gdscs = - of_property_count_strings(dev->of_node, "qcom,regulator-names"); - - if (pwr->num_gdscs < 1) { - pwr->num_gdscs = 0; - return 0; - } - - pwr->gdscs = devm_kzalloc( - dev, sizeof(*pwr->gdscs) * pwr->num_gdscs, GFP_KERNEL); - - if (!pwr->gdscs) - return -ENOMEM; - - i = 0; - of_property_for_each_string(dev->of_node, "qcom,regulator-names", - prop, cname) - pwr->gdscs[i++].supply = cname; - - return devm_regulator_bulk_get(dev, pwr->num_gdscs, pwr->gdscs); -} - -static int arm_smmu_init_interconnect(struct arm_smmu_power_resources *pwr) -{ - struct device *dev = pwr->dev; - - /* We don't want the interconnect APIs to print an error message */ - if (!of_find_property(dev->of_node, "interconnects", NULL)) { - dev_dbg(dev, "No interconnect info\n"); - return 0; - } - - pwr->icc_path = devm_of_icc_get(dev, NULL); - if (IS_ERR_OR_NULL(pwr->icc_path)) { - if (PTR_ERR(pwr->icc_path) != -EPROBE_DEFER) - dev_err(dev, "Unable to read interconnect path from devicetree rc: %ld\n", - PTR_ERR(pwr->icc_path)); - return pwr->icc_path ? PTR_ERR(pwr->icc_path) : -EINVAL; - } - - if (of_property_read_bool(dev->of_node, "qcom,active-only")) - icc_set_tag(pwr->icc_path, ARM_SMMU_ICC_ACTIVE_ONLY_TAG); - - return 0; -} - -/* - * Cleanup done by devm. Any non-devm resources must clean up themselves. - */ -struct arm_smmu_power_resources *arm_smmu_init_power_resources( - struct device *dev) -{ - struct arm_smmu_power_resources *pwr; - int ret; - - pwr = devm_kzalloc(dev, sizeof(*pwr), GFP_KERNEL); - if (!pwr) - return ERR_PTR(-ENOMEM); - - pwr->dev = dev; - mutex_init(&pwr->power_lock); - - ret = arm_smmu_init_clocks(pwr); - if (ret) - return ERR_PTR(ret); - - ret = arm_smmu_init_regulators(pwr); - if (ret) - return ERR_PTR(ret); - - ret = arm_smmu_init_interconnect(pwr); - if (ret) - return ERR_PTR(ret); - - return pwr; -} diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 3086b552c638..ba6298c7140e 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -1,19 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2019, The Linux Foundation. All rights reserved. */ #include #include -#include -#include -#include -#include -#include #include #include -#include -#include #include "arm-smmu.h" @@ -88,7 +81,6 @@ static void qcom_adreno_smmu_resume_translation(const void *cookie, bool termina } #define QCOM_ADRENO_SMMU_GPU_SID 0 -#define QCOM_ADRENO_SMMU_GPU_LPAC_SID 1 static bool qcom_adreno_smmu_is_gpu_device(struct device *dev) { @@ -102,8 +94,7 @@ static bool qcom_adreno_smmu_is_gpu_device(struct device *dev) for (i = 0; i < fwspec->num_ids; i++) { u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); - if (sid == QCOM_ADRENO_SMMU_GPU_SID || - sid == QCOM_ADRENO_SMMU_GPU_LPAC_SID) + if (sid == QCOM_ADRENO_SMMU_GPU_SID) return true; } @@ -173,14 +164,14 @@ static int qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_doma int count; /* - * Assign context bank 0 and 1 to the GPU device so the GPU hardware can + * Assign context bank 0 to the GPU device so the GPU hardware can * switch pagetables */ if (qcom_adreno_smmu_is_gpu_device(dev)) { start = 0; - count = 2; + count = 1; } else { - start = 2; + start = 1; count = smmu->num_context_banks; } @@ -197,12 +188,10 @@ static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu) return true; } -static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[]; static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg *pgtbl_cfg, struct device *dev) { struct adreno_smmu_priv *priv; - const struct device_node *np = smmu_domain->smmu->dev->of_node; smmu_domain->cfg.flush_walk_prefer_tlbiasid = true; @@ -229,19 +218,8 @@ static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain, priv->get_ttbr1_cfg = qcom_adreno_smmu_get_ttbr1_cfg; priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg; priv->get_fault_info = qcom_adreno_smmu_get_fault_info; - - /* - * These functions are only compatible with the data structures used by the - * QCOM SMMU implementation hooks, and are thus not appropriate to set for other - * implementations (e.g. QSMMUV500). - * - * Providing these functions as part of the GPU interface also makes little sense - * as context banks are set to stall by default anyway. - */ - if (of_match_node(qcom_smmu_impl_of_match, np)) { - priv->set_stall = qcom_adreno_smmu_set_stall; - priv->resume_translation = qcom_adreno_smmu_resume_translation; - } + priv->set_stall = qcom_adreno_smmu_set_stall; + priv->resume_translation = qcom_adreno_smmu_resume_translation; return 0; } @@ -398,1351 +376,6 @@ static const struct arm_smmu_impl qcom_smmu_impl = { .write_s2cr = qcom_smmu_write_s2cr, }; -#define TBUID_SHIFT 10 - -#define DEBUG_SID_HALT_REG 0x0 -#define DEBUG_SID_HALT_REQ BIT(16) -#define DEBUG_SID_HALT_SID GENMASK(9, 0) - -#define DEBUG_VA_ADDR_REG 0x8 - -#define DEBUG_TXN_TRIGG_REG 0x18 -#define DEBUG_TXN_AXPROT GENMASK(8, 6) -#define DEBUG_TXN_AXCACHE GENMASK(5, 2) -#define DEBUG_TXN_WRITE BIT(1) -#define DEBUG_TXN_AXPROT_PRIV 0x1 -#define DEBUG_TXN_AXPROT_UNPRIV 0x0 -#define DEBUG_TXN_AXPROT_NSEC 0x2 -#define DEBUG_TXN_AXPROT_SEC 0x0 -#define DEBUG_TXN_AXPROT_INST 0x4 -#define DEBUG_TXN_AXPROT_DATA 0x0 -#define DEBUG_TXN_READ (0x0 << 1) -#define DEBUG_TXN_TRIGGER BIT(0) - -#define DEBUG_SR_HALT_ACK_REG 0x20 -#define DEBUG_SR_HALT_ACK_VAL (0x1 << 1) -#define DEBUG_SR_ECATS_RUNNING_VAL (0x1 << 0) - -#define DEBUG_PAR_REG 0x28 -#define DEBUG_PAR_PA GENMASK_ULL(47, 12) -#define DEBUG_PAR_FAULT_VAL BIT(0) - -#define DEBUG_AXUSER_REG 0x30 -#define DEBUG_AXUSER_CDMID GENMASK_ULL(43, 36) -#define DEBUG_AXUSER_CDMID_VAL 255 - -#define TBU_DBG_TIMEOUT_US 100 -#define TBU_MICRO_IDLE_DELAY_US 5 - -#define TNX_TCR_CNTL 0x130 -#define TNX_TCR_CNTL_TBU_OT_CAPTURE_EN BIT(18) -#define TNX_TCR_CNTL_ALWAYS_CAPTURE BIT(15) -#define TNX_TCR_CNTL_MATCH_MASK_UPD BIT(7) -#define TNX_TCR_CNTL_MATCH_MASK_VALID BIT(6) - -#define CAPTURE1_SNAPSHOT_1 0x138 - -#define TNX_TCR_CNTL_2 0x178 -#define TNX_TCR_CNTL_2_CAP1_VALID BIT(0) - -/* QTB constants */ -#define QTB_DBG_TIMEOUT_US 100 - -#define QTB_SWID_LOW 0x0 - -#define QTB_OVR_DBG_FENCEREQ 0x410 -#define QTB_OVR_DBG_FENCEREQ_HALT BIT(0) - -#define QTB_OVR_DBG_FENCEACK 0x418 -#define QTB_OVR_DBG_FENCEACK_ACK BIT(0) - -#define QTB_OVR_ECATS_INFLD0 0x430 -#define QTB_OVR_ECATS_INFLD0_PCIE_NO_SNOOP BIT(21) -#define QTB_OVR_ECATS_INFLD0_SEC_SID BIT(20) -#define QTB_OVR_ECATS_INFLD0_QAD GENMASK(19, 16) -#define QTB_OVR_ECATS_INFLD0_SID GENMASK(9, 0) - -#define QTB_OVR_ECATS_INFLD1 0x438 -#define QTB_OVR_ECATS_INFLD1_PNU BIT(13) -#define QTB_OVR_ECATS_INFLD1_IND BIT(12) -#define QTB_OVR_ECATS_INFLD1_DIRTY BIT(11) -#define QTB_OVR_ECATS_INFLD1_TR_TYPE GENMASK(10, 8) -#define QTB_OVR_ECATS_INFLD1_TR_TYPE_SHARED 4 -#define QTB_OVR_ECATS_INFLD1_ALLOC GENMASK(7, 4) -#define QTB_OVR_ECATS_INFLD1_NON_SEC BIT(3) -#define QTB_OVR_ECATS_INFLD1_OPC GENMASK(2, 0) -#define QTB_OVR_ECATS_INFLD1_OPC_WRI 1 - -#define QTB_OVR_ECATS_INFLD2 0x440 - -#define QTB_OVR_ECATS_TRIGGER 0x448 -#define QTB_OVR_ECATS_TRIGGER_START BIT(0) - -#define QTB_OVR_ECATS_STATUS 0x450 -#define QTB_OVR_ECATS_STATUS_DONE BIT(0) - -#define QTB_OVR_ECATS_OUTFLD0 0x458 -#define QTB_OVR_ECATS_OUTFLD0_PA GENMASK(63, 12) -#define QTB_OVR_ECATS_OUTFLD0_FAULT_TYPE GENMASK(5, 4) -#define QTB_OVR_ECATS_OUTFLD0_FAULT BIT(0) - -#define QTB_NS_DBG_PORT_N_OT_SNAPSHOT(port_num) (0xc10 + (0x10 * port_num)) - -struct actlr_setting { - struct arm_smmu_smr smr; - u32 actlr; -}; - -struct qsmmuv500_archdata { - struct arm_smmu_device smmu; - struct list_head tbus; - struct actlr_setting *actlrs; - u32 actlr_tbl_size; - struct work_struct outstanding_tnx_work; - spinlock_t atos_lock; -}; -#define to_qsmmuv500_archdata(smmu) \ - container_of(smmu, struct qsmmuv500_archdata, smmu) - -struct qsmmuv500_group_iommudata { - bool has_actlr; - u32 actlr; -}; -#define to_qsmmuv500_group_iommudata(group) \ - ((struct qsmmuv500_group_iommudata *) \ - (iommu_group_get_iommudata(group))) - -struct qsmmuv500_tbu_device { - struct list_head list; - struct device *dev; - struct arm_smmu_device *smmu; - void __iomem *base; - - const struct qsmmuv500_tbu_impl *impl; - struct arm_smmu_power_resources *pwr; - u32 sid_start; - u32 num_sids; - u32 iova_width; - - /* Protects halt count */ - spinlock_t halt_lock; - u32 halt_count; -}; - -struct qsmmuv500_tbu_impl { - int (*halt_req)(struct qsmmuv500_tbu_device *tbu); - int (*halt_poll)(struct qsmmuv500_tbu_device *tbu); - void (*resume)(struct qsmmuv500_tbu_device *tbu); - phys_addr_t (*trigger_atos)(struct qsmmuv500_tbu_device *tbu, dma_addr_t iova, u32 sid, - unsigned long trans_flags); - void (*write_sync)(struct qsmmuv500_tbu_device *tbu); - void (*log_outstanding_transactions)(struct qsmmuv500_tbu_device *tbu); -}; - -struct arm_tbu_device { - struct qsmmuv500_tbu_device tbu; - bool has_micro_idle; -}; - -#define to_arm_tbu(tbu) container_of(tbu, struct arm_tbu_device, tbu) - -struct qtb500_device { - struct qsmmuv500_tbu_device tbu; - bool no_halt; - u32 num_ports; -}; - -#define to_qtb500(tbu) container_of(tbu, struct qtb500_device, tbu) - -static int arm_tbu_halt_req(struct qsmmuv500_tbu_device *tbu) -{ - void __iomem *tbu_base = tbu->base; - u32 halt; - - halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); - halt |= DEBUG_SID_HALT_REQ; - writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); - - return 0; -} - -static int arm_tbu_halt_poll(struct qsmmuv500_tbu_device *tbu) -{ - void __iomem *tbu_base = tbu->base; - u32 halt, status; - - if (readl_poll_timeout_atomic(tbu_base + DEBUG_SR_HALT_ACK_REG, status, - (status & DEBUG_SR_HALT_ACK_VAL), - 0, TBU_DBG_TIMEOUT_US)) { - dev_err(tbu->dev, "Couldn't halt TBU!\n"); - - halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); - halt &= ~DEBUG_SID_HALT_REQ; - writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); - - return -ETIMEDOUT; - } - - return 0; -} - -static void arm_tbu_resume(struct qsmmuv500_tbu_device *tbu) -{ - void __iomem *base = tbu->base; - u32 val; - - val = readl_relaxed(base + DEBUG_SID_HALT_REG); - val &= ~DEBUG_SID_HALT_REQ; - writel_relaxed(val, base + DEBUG_SID_HALT_REG); -} - -static phys_addr_t arm_tbu_trigger_atos(struct qsmmuv500_tbu_device *tbu, dma_addr_t iova, u32 sid, - unsigned long trans_flags) -{ - void __iomem *tbu_base = tbu->base; - phys_addr_t phys = 0; - u64 val; - ktime_t timeout; - bool ecats_timedout = false; - - /* Set address and stream-id */ - val = readq_relaxed(tbu_base + DEBUG_SID_HALT_REG); - val &= ~DEBUG_SID_HALT_SID; - val |= FIELD_PREP(DEBUG_SID_HALT_SID, sid); - writeq_relaxed(val, tbu_base + DEBUG_SID_HALT_REG); - writeq_relaxed(iova, tbu_base + DEBUG_VA_ADDR_REG); - val = FIELD_PREP(DEBUG_AXUSER_CDMID, DEBUG_AXUSER_CDMID_VAL); - writeq_relaxed(val, tbu_base + DEBUG_AXUSER_REG); - - /* Write-back Read and Write-Allocate */ - val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xF); - - /* Non-secure Access */ - val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_NSEC); - - /* Write or Read Access */ - if (trans_flags & IOMMU_TRANS_WRITE) - val |= DEBUG_TXN_WRITE; - - /* Priviledged or Unpriviledged Access */ - if (trans_flags & IOMMU_TRANS_PRIV) - val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_PRIV); - - /* Data or Instruction Access */ - if (trans_flags & IOMMU_TRANS_INST) - val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_INST); - - val |= DEBUG_TXN_TRIGGER; - writeq_relaxed(val, tbu_base + DEBUG_TXN_TRIGG_REG); - - timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US); - for (;;) { - val = readl_relaxed(tbu_base + DEBUG_SR_HALT_ACK_REG); - if (!(val & DEBUG_SR_ECATS_RUNNING_VAL)) - break; - val = readl_relaxed(tbu_base + DEBUG_PAR_REG); - if (val & DEBUG_PAR_FAULT_VAL) - break; - if (ktime_compare(ktime_get(), timeout) > 0) { - ecats_timedout = true; - break; - } - } - - val = readq_relaxed(tbu_base + DEBUG_PAR_REG); - if (val & DEBUG_PAR_FAULT_VAL) - dev_err(tbu->dev, "ECATS generated a fault interrupt! PAR = %llx, SID=0x%x\n", - val, sid); - else if (ecats_timedout) - dev_err_ratelimited(tbu->dev, "ECATS translation timed out!\n"); - else - phys = FIELD_GET(DEBUG_PAR_PA, val); - - /* Reset hardware */ - writeq_relaxed(0, tbu_base + DEBUG_TXN_TRIGG_REG); - writeq_relaxed(0, tbu_base + DEBUG_VA_ADDR_REG); - val = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); - val &= ~DEBUG_SID_HALT_SID; - writel_relaxed(val, tbu_base + DEBUG_SID_HALT_REG); - - return phys; -} - -static void arm_tbu_write_sync(struct qsmmuv500_tbu_device *tbu) -{ - readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); -} - -static void arm_tbu_log_outstanding_transactions(struct qsmmuv500_tbu_device *tbu) -{ - void __iomem *base = tbu->base; - u64 outstanding_tnxs; - u64 tcr_cntl_val, res; - - tcr_cntl_val = readq_relaxed(base + TNX_TCR_CNTL); - - /* Write 1 into MATCH_MASK_UPD of TNX_TCR_CNTL */ - writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_MATCH_MASK_UPD, - base + TNX_TCR_CNTL); - - /* - * Simultaneously write 0 into MATCH_MASK_UPD, 0 into - * ALWAYS_CAPTURE, 0 into MATCH_MASK_VALID, and 1 into - * TBU_OT_CAPTURE_EN of TNX_TCR_CNTL - */ - tcr_cntl_val &= ~(TNX_TCR_CNTL_MATCH_MASK_UPD | - TNX_TCR_CNTL_ALWAYS_CAPTURE | - TNX_TCR_CNTL_MATCH_MASK_VALID); - writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, - base + TNX_TCR_CNTL); - - /* Poll for CAPTURE1_VALID to become 1 on TNX_TCR_CNTL_2 */ - if (readq_poll_timeout_atomic(base + TNX_TCR_CNTL_2, res, - res & TNX_TCR_CNTL_2_CAP1_VALID, - 0, TBU_DBG_TIMEOUT_US)) { - dev_err_ratelimited(tbu->dev, - "Timeout on TNX snapshot poll\n"); - goto poll_timeout; - } - - /* Read Register CAPTURE1_SNAPSHOT_1 */ - outstanding_tnxs = readq_relaxed(base + CAPTURE1_SNAPSHOT_1); - dev_err_ratelimited(tbu->dev, - "Outstanding Transaction Bitmap: 0x%llx\n", - outstanding_tnxs); - -poll_timeout: - /* Write TBU_OT_CAPTURE_EN to 0 of TNX_TCR_CNTL */ - writeq_relaxed(tcr_cntl_val & ~TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, - tbu->base + TNX_TCR_CNTL); -} - -static const struct qsmmuv500_tbu_impl arm_tbu_impl = { - .halt_req = arm_tbu_halt_req, - .halt_poll = arm_tbu_halt_poll, - .resume = arm_tbu_resume, - .trigger_atos = arm_tbu_trigger_atos, - .write_sync = arm_tbu_write_sync, - .log_outstanding_transactions = arm_tbu_log_outstanding_transactions, -}; - -/* - * Prior to accessing registers in the TBU local register space, - * TBU must be woken from micro idle. - */ -static int __arm_tbu_micro_idle_cfg(struct arm_smmu_device *smmu, - u32 val, u32 mask) -{ - void __iomem *reg; - u32 tmp, new; - unsigned long flags; - int ret; - - /* Protect APPS_SMMU_TBU_REG_ACCESS register. */ - spin_lock_irqsave(&smmu->global_sync_lock, flags); - new = arm_smmu_readl(smmu, ARM_SMMU_IMPL_DEF5, - APPS_SMMU_TBU_REG_ACCESS_REQ_NS); - new &= ~mask; - new |= val; - arm_smmu_writel(smmu, ARM_SMMU_IMPL_DEF5, - APPS_SMMU_TBU_REG_ACCESS_REQ_NS, - new); - - reg = arm_smmu_page(smmu, ARM_SMMU_IMPL_DEF5); - reg += APPS_SMMU_TBU_REG_ACCESS_ACK_NS; - ret = readl_poll_timeout_atomic(reg, tmp, ((tmp & mask) == val), 0, 200); - if (ret) - dev_WARN(smmu->dev, "Timed out configuring micro idle! %x instead of %x\n", tmp, - new); - /* - * While the micro-idle guard sequence registers may have been configured - * properly, it is possible that the intended effect has not been realized - * by the power management hardware due to delays in the system. - * - * Spin for a short amount of time to allow for the desired configuration to - * take effect before proceeding. - */ - udelay(TBU_MICRO_IDLE_DELAY_US); - spin_unlock_irqrestore(&smmu->global_sync_lock, flags); - return ret; -} - -int arm_tbu_micro_idle_wake(struct arm_smmu_power_resources *pwr) -{ - struct qsmmuv500_tbu_device *tbu = dev_get_drvdata(pwr->dev); - struct arm_tbu_device *arm_tbu = to_arm_tbu(tbu); - u32 val; - - if (!arm_tbu->has_micro_idle) - return 0; - - val = tbu->sid_start >> 10; - val = 1 << val; - return __arm_tbu_micro_idle_cfg(tbu->smmu, val, val); -} - -void arm_tbu_micro_idle_allow(struct arm_smmu_power_resources *pwr) -{ - struct qsmmuv500_tbu_device *tbu = dev_get_drvdata(pwr->dev); - struct arm_tbu_device *arm_tbu = to_arm_tbu(tbu); - u32 val; - - if (!arm_tbu->has_micro_idle) - return; - - val = tbu->sid_start >> 10; - val = 1 << val; - __arm_tbu_micro_idle_cfg(tbu->smmu, 0, val); -} - -static struct qsmmuv500_tbu_device *arm_tbu_impl_init(struct qsmmuv500_tbu_device *tbu) -{ - struct arm_tbu_device *arm_tbu; - struct device *dev = tbu->dev; - - arm_tbu = devm_krealloc(dev, tbu, sizeof(*arm_tbu), GFP_KERNEL); - if (!arm_tbu) - return ERR_PTR(-ENOMEM); - - arm_tbu->tbu.impl = &arm_tbu_impl; - arm_tbu->has_micro_idle = of_property_read_bool(dev->of_node, "qcom,micro-idle"); - - if (arm_tbu->has_micro_idle) { - arm_tbu->tbu.pwr->resume = arm_tbu_micro_idle_wake; - arm_tbu->tbu.pwr->suspend = arm_tbu_micro_idle_allow; - } - - return &arm_tbu->tbu; -} - -static int qtb500_tbu_halt_req(struct qsmmuv500_tbu_device *tbu) -{ - void __iomem *qtb_base = tbu->base; - struct qtb500_device *qtb = to_qtb500(tbu); - u64 val; - - if (qtb->no_halt) - return 0; - - val = readq_relaxed(qtb_base + QTB_OVR_DBG_FENCEREQ); - val |= QTB_OVR_DBG_FENCEREQ_HALT; - writeq_relaxed(val, qtb_base + QTB_OVR_DBG_FENCEREQ); - - return 0; -} - -static int qtb500_tbu_halt_poll(struct qsmmuv500_tbu_device *tbu) -{ - void __iomem *qtb_base = tbu->base; - struct qtb500_device *qtb = to_qtb500(tbu); - u64 val, status; - - if (qtb->no_halt) - return 0; - - if (readq_poll_timeout_atomic(qtb_base + QTB_OVR_DBG_FENCEACK, status, - (status & QTB_OVR_DBG_FENCEACK_ACK), 0, - QTB_DBG_TIMEOUT_US)) { - dev_err(tbu->dev, "Couldn't halt QTB\n"); - - val = readq_relaxed(qtb_base + QTB_OVR_DBG_FENCEREQ); - val &= ~QTB_OVR_DBG_FENCEREQ_HALT; - writeq_relaxed(val, qtb_base + QTB_OVR_DBG_FENCEREQ); - - return -ETIMEDOUT; - } - - return 0; -} - -static void qtb500_tbu_resume(struct qsmmuv500_tbu_device *tbu) -{ - void __iomem *qtb_base = tbu->base; - struct qtb500_device *qtb = to_qtb500(tbu); - u64 val; - - if (qtb->no_halt) - return; - - val = readq_relaxed(qtb_base + QTB_OVR_DBG_FENCEREQ); - val &= ~QTB_OVR_DBG_FENCEREQ_HALT; - writeq_relaxed(val, qtb_base + QTB_OVR_DBG_FENCEREQ); -} - -static phys_addr_t qtb500_trigger_atos(struct qsmmuv500_tbu_device *tbu, dma_addr_t iova, - u32 sid, unsigned long trans_flags) -{ - void __iomem *qtb_base = tbu->base; - u64 infld0, infld1, infld2, val; - phys_addr_t phys = 0; - ktime_t timeout; - bool ecats_timedout = false; - - /* - * Recommended to set: - * - * QTB_OVR_ECATS_INFLD0.QAD == 0 (AP Access Domain) - * QTB_OVR_EACTS_INFLD0.PCIE_NO_SNOOP == 0 (IO-Coherency enabled) - */ - infld0 = FIELD_PREP(QTB_OVR_ECATS_INFLD0_SID, sid); - if (trans_flags & IOMMU_TRANS_SEC) - infld0 |= QTB_OVR_ECATS_INFLD0_SEC_SID; - - infld1 = 0; - if (trans_flags & IOMMU_TRANS_PRIV) - infld1 |= QTB_OVR_ECATS_INFLD1_PNU; - if (trans_flags & IOMMU_TRANS_INST) - infld1 |= QTB_OVR_ECATS_INFLD1_IND; - /* - * Recommended to set: - * - * QTB_OVR_ECATS_INFLD1.DIRTY == 0, - * QTB_OVR_ECATS_INFLD1.TR_TYPE == 4 (Cacheable and Shareable memory) - * QTB_OVR_ECATS_INFLD1.ALLOC == 0 (No allocation in TLB/caches) - */ - infld1 |= FIELD_PREP(QTB_OVR_ECATS_INFLD1_TR_TYPE, QTB_OVR_ECATS_INFLD1_TR_TYPE_SHARED); - if (!(trans_flags & IOMMU_TRANS_SEC)) - infld1 |= QTB_OVR_ECATS_INFLD1_NON_SEC; - if (trans_flags & IOMMU_TRANS_WRITE) - infld1 |= FIELD_PREP(QTB_OVR_ECATS_INFLD1_OPC, QTB_OVR_ECATS_INFLD1_OPC_WRI); - - infld2 = iova; - - writeq_relaxed(infld0, qtb_base + QTB_OVR_ECATS_INFLD0); - writeq_relaxed(infld1, qtb_base + QTB_OVR_ECATS_INFLD1); - writeq_relaxed(infld2, qtb_base + QTB_OVR_ECATS_INFLD2); - writeq_relaxed(QTB_OVR_ECATS_TRIGGER_START, qtb_base + QTB_OVR_ECATS_TRIGGER); - - timeout = ktime_add_us(ktime_get(), QTB_DBG_TIMEOUT_US); - for (;;) { - val = readq_relaxed(qtb_base + QTB_OVR_ECATS_STATUS); - if (val & QTB_OVR_ECATS_STATUS_DONE) - break; - val = readq_relaxed(qtb_base + QTB_OVR_ECATS_OUTFLD0); - if (val & QTB_OVR_ECATS_OUTFLD0_FAULT) - break; - if (ktime_compare(ktime_get(), timeout) > 0) { - ecats_timedout = true; - break; - } - } - - val = readq_relaxed(qtb_base + QTB_OVR_ECATS_OUTFLD0); - if (val & QTB_OVR_ECATS_OUTFLD0_FAULT) - dev_err(tbu->dev, "ECATS generated a fault interrupt! OUTFLD0 = 0x%llx SID = 0x%x\n", - val, sid); - else if (ecats_timedout) - dev_err_ratelimited(tbu->dev, "ECATS translation timed out!\n"); - else - phys = FIELD_GET(QTB_OVR_ECATS_OUTFLD0_PA, val); - - /* Reset hardware for next transaction. */ - writeq_relaxed(0, qtb_base + QTB_OVR_ECATS_TRIGGER); - - return phys; -} - -static void qtb500_tbu_write_sync(struct qsmmuv500_tbu_device *tbu) -{ - readl_relaxed(tbu->base + QTB_SWID_LOW); -} - -static void qtb500_log_outstanding_transactions(struct qsmmuv500_tbu_device *tbu) -{ - void __iomem *qtb_base = tbu->base; - struct qtb500_device *qtb = to_qtb500(tbu); - u64 outstanding_tnx; - int i; - - for (i = 0; i < qtb->num_ports; i++) { - outstanding_tnx = readq_relaxed(qtb_base + QTB_NS_DBG_PORT_N_OT_SNAPSHOT(i)); - dev_err(tbu->dev, "port %d outstanding transactions bitmap: 0x%llx\n", i, - outstanding_tnx); - } -} - -static const struct qsmmuv500_tbu_impl qtb500_impl = { - .halt_req = qtb500_tbu_halt_req, - .halt_poll = qtb500_tbu_halt_poll, - .resume = qtb500_tbu_resume, - .trigger_atos = qtb500_trigger_atos, - .write_sync = qtb500_tbu_write_sync, - .log_outstanding_transactions = qtb500_log_outstanding_transactions, -}; - -static struct qsmmuv500_tbu_device *qtb500_impl_init(struct qsmmuv500_tbu_device *tbu) -{ - struct qtb500_device *qtb; - struct device *dev = tbu->dev; - int ret; - - qtb = devm_krealloc(dev, tbu, sizeof(*qtb), GFP_KERNEL); - if (!qtb) - return ERR_PTR(-ENOMEM); - - qtb->tbu.impl = &qtb500_impl; - - ret = of_property_read_u32(dev->of_node, "qcom,num-qtb-ports", &qtb->num_ports); - if (ret) - return ERR_PTR(ret); - - qtb->no_halt = of_property_read_bool(dev->of_node, "qcom,no-qtb-atos-halt"); - - return &qtb->tbu; -} - -static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu(struct arm_smmu_device *smmu, u32 sid) -{ - struct qsmmuv500_tbu_device *tbu = NULL; - struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); - - list_for_each_entry(tbu, &data->tbus, list) { - if (tbu->sid_start <= sid && - sid < tbu->sid_start + tbu->num_sids) - return tbu; - } - - return NULL; -} - -static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu, - struct arm_smmu_domain *smmu_domain) -{ - unsigned long flags; - struct arm_smmu_device *smmu = smmu_domain->smmu; - int ret = 0, idx = smmu_domain->cfg.cbndx; - u32 fsr; - - if (of_property_read_bool(tbu->dev->of_node, "qcom,opt-out-tbu-halting")) { - dev_notice(tbu->dev, "TBU opted-out for halting!\n"); - return -EBUSY; - } - - spin_lock_irqsave(&tbu->halt_lock, flags); - if (tbu->halt_count) { - tbu->halt_count++; - goto out; - } - - ret = tbu->impl->halt_req(tbu); - if (ret) - goto out; - - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if ((fsr & ARM_SMMU_FSR_FAULT) && (fsr & ARM_SMMU_FSR_SS)) { - u32 sctlr_orig, sctlr; - /* - * We are in a fault; Our request to halt the bus will not - * complete until transactions in front of us (such as the fault - * itself) have completed. Disable iommu faults and terminate - * any existing transactions. - */ - sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); - sctlr = sctlr_orig & ~(ARM_SMMU_SCTLR_CFCFG | ARM_SMMU_SCTLR_CFIE); - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - /* - * Barrier required to ensure that the FSR is cleared - * before resuming SMMU operation - */ - wmb(); - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, - ARM_SMMU_RESUME_TERMINATE); - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); - } - - ret = tbu->impl->halt_poll(tbu); - if (ret) - goto out; - - tbu->halt_count = 1; -out: - spin_unlock_irqrestore(&tbu->halt_lock, flags); - return ret; -} - -static void qsmmuv500_tbu_resume(struct qsmmuv500_tbu_device *tbu) -{ - unsigned long flags; - - spin_lock_irqsave(&tbu->halt_lock, flags); - if (WARN(!tbu->halt_count, "%s bad tbu->halt_count", dev_name(tbu->dev))) { - goto out; - - } else if (tbu->halt_count > 1) { - tbu->halt_count--; - goto out; - } - - tbu->impl->resume(tbu); - - tbu->halt_count = 0; -out: - spin_unlock_irqrestore(&tbu->halt_lock, flags); -} - -/* - * Provides mutually exclusive access to the registers used by the - * outstanding transaction snapshot feature and the transaction - * snapshot capture feature. - */ -static DEFINE_MUTEX(capture_reg_lock); - -static void qsmmuv500_log_outstanding_transactions(struct work_struct *work) -{ - struct qsmmuv500_tbu_device *tbu = NULL; - struct qsmmuv500_archdata *data = container_of(work, - struct qsmmuv500_archdata, - outstanding_tnx_work); - struct arm_smmu_device *smmu = &data->smmu; - - if (!mutex_trylock(&capture_reg_lock)) { - dev_warn_ratelimited(smmu->dev, - "Tnx snapshot regs in use, not dumping OT tnxs.\n"); - goto bug; - } - - if (arm_smmu_power_on(smmu->pwr)) { - dev_err_ratelimited(smmu->dev, - "%s: Failed to power on SMMU.\n", - __func__); - goto unlock; - } - - list_for_each_entry(tbu, &data->tbus, list) { - if (arm_smmu_power_on(tbu->pwr)) { - dev_err_ratelimited(tbu->dev, "%s: Failed to power on TBU.\n", __func__); - continue; - } - - tbu->impl->log_outstanding_transactions(tbu); - - arm_smmu_power_off(smmu, tbu->pwr); - } - - arm_smmu_power_off(smmu, smmu->pwr); -unlock: - mutex_unlock(&capture_reg_lock); -bug: - BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); -} - -static struct qsmmuv500_tbu_device *qsmmuv500_tbu_impl_init(struct qsmmuv500_tbu_device *tbu) -{ - if (of_device_is_compatible(tbu->dev->of_node, "qcom,qtb500")) - return qtb500_impl_init(tbu); - - return arm_tbu_impl_init(tbu); -} - -static int qsmmuv500_tbu_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct qsmmuv500_tbu_device *tbu; - struct resource *res; - const __be32 *cell; - int ret, len; - - tbu = devm_kzalloc(dev, sizeof(*tbu), GFP_KERNEL); - if (!tbu) - return -ENOMEM; - - tbu->dev = dev; - - /* - * ARM TBUs need to have power resources initialized before its - * implementation defined initialization occurs to setup the - * suspend and resure power callbacks. - */ - tbu->pwr = arm_smmu_init_power_resources(dev); - if (IS_ERR(tbu->pwr)) - return PTR_ERR(tbu->pwr); - - tbu = qsmmuv500_tbu_impl_init(tbu); - if (IS_ERR(tbu)) - return PTR_ERR(tbu); - - INIT_LIST_HEAD(&tbu->list); - - spin_lock_init(&tbu->halt_lock); - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) - return -EINVAL; - - tbu->base = devm_ioremap(dev, res->start, resource_size(res)); - if (!tbu->base) - return -ENOMEM; - - cell = of_get_property(dev->of_node, "qcom,stream-id-range", &len); - if (!cell || len < 8) - return -EINVAL; - - tbu->sid_start = of_read_number(cell, 1); - tbu->num_sids = of_read_number(cell + 1, 1); - - ret = of_property_read_u32(dev->of_node, "qcom,iova-width", &tbu->iova_width); - if (ret < 0) - return ret; - - dev_set_drvdata(dev, tbu); - return 0; -} - -static const struct of_device_id qsmmuv500_tbu_of_match[] = { - {.compatible = "qcom,qsmmuv500-tbu"}, - {} -}; - -struct platform_driver qsmmuv500_tbu_driver = { - .driver = { - .name = "qsmmuv500-tbu", - .of_match_table = of_match_ptr(qsmmuv500_tbu_of_match), - }, - .probe = qsmmuv500_tbu_probe, -}; - -static void qsmmuv500_tlb_sync_timeout(struct arm_smmu_device *smmu) -{ - u32 sync_inv_ack, tbu_pwr_status, sync_inv_progress; - u32 tbu_inv_pending = 0, tbu_sync_pending = 0; - u32 tbu_inv_acked = 0, tbu_sync_acked = 0; - u32 tcu_inv_pending = 0, tcu_sync_pending = 0; - unsigned long tbu_ids = 0; - struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); - int ret; - - static DEFINE_RATELIMIT_STATE(_rs, - DEFAULT_RATELIMIT_INTERVAL, - DEFAULT_RATELIMIT_BURST); - - dev_err_ratelimited(smmu->dev, - "TLB sync timed out -- SMMU may be deadlocked\n"); - - sync_inv_ack = arm_smmu_readl(smmu, - ARM_SMMU_IMPL_DEF5, - ARM_SMMU_STATS_SYNC_INV_TBU_ACK); - - if (sync_inv_ack) { - tbu_inv_pending = FIELD_GET(TBU_INV_REQ, sync_inv_ack); - tbu_inv_acked = FIELD_GET(TBU_INV_ACK, sync_inv_ack); - tbu_sync_pending = FIELD_GET(TBU_SYNC_REQ, sync_inv_ack); - tbu_sync_acked = FIELD_GET(TBU_SYNC_ACK, sync_inv_ack); - } - - ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + - ARM_SMMU_TBU_PWR_STATUS), &tbu_pwr_status); - if (ret) { - dev_err_ratelimited(smmu->dev, - "SCM read of TBU power status fails: %d\n", - ret); - goto out; - } - - ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + - ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR), - &sync_inv_progress); - if (ret) { - dev_err_ratelimited(smmu->dev, - "SCM read of TBU sync/inv prog fails: %d\n", - ret); - goto out; - } - - if (tbu_pwr_status) { - if (tbu_sync_pending) - tbu_ids = tbu_pwr_status & ~tbu_sync_acked; - else if (tbu_inv_pending) - tbu_ids = tbu_pwr_status & ~tbu_inv_acked; - } - - tcu_inv_pending = FIELD_GET(TCU_INV_IN_PRGSS, sync_inv_progress); - tcu_sync_pending = FIELD_GET(TCU_SYNC_IN_PRGSS, sync_inv_progress); - - if (__ratelimit(&_rs)) { - unsigned long tbu_id; - - dev_err(smmu->dev, - "TBU ACK 0x%x TBU PWR 0x%x TCU sync_inv 0x%x\n", - sync_inv_ack, tbu_pwr_status, sync_inv_progress); - dev_err(smmu->dev, - "TCU invalidation %s, TCU sync %s\n", - tcu_inv_pending?"pending":"completed", - tcu_sync_pending?"pending":"completed"); - - for_each_set_bit(tbu_id, &tbu_ids, sizeof(tbu_ids) * - BITS_PER_BYTE) { - - struct qsmmuv500_tbu_device *tbu; - - tbu = qsmmuv500_find_tbu(smmu, - (u16)(tbu_id << TBUID_SHIFT)); - if (tbu) { - dev_err(smmu->dev, - "TBU %s ack pending for TBU %s, %s\n", - tbu_sync_pending?"sync" : "inv", - dev_name(tbu->dev), - tbu_sync_pending ? - "check pending transactions on TBU" - : "check for TBU power status"); - } - } - } - - if (tcu_sync_pending) { - schedule_work(&data->outstanding_tnx_work); - return; - } -out: - if (ret) { - if (sync_inv_ack) { - /* TBU PWR status is not available so just dump raw - * fields - */ - dev_err(smmu->dev, - "TBU %s ack pending, got ack for TBUs %d, %s\n", - tbu_sync_pending ? "sync" : "inv", - tbu_sync_pending ? tbu_sync_acked:tbu_inv_acked, - tbu_sync_pending ? - "check pending transactions on TBU" - : "check for TBU power status"); - - } - - dev_err(smmu->dev, "TBU SYNC_INV_ACK reg 0x%x\n", sync_inv_ack); - } - - BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); -} - -static void qsmmuv500_device_remove(struct arm_smmu_device *smmu) -{ - struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); - - cancel_work_sync(&data->outstanding_tnx_work); -} - -/* - * Checks whether smr2 is a subset of smr - */ -static bool smr_is_subset(struct arm_smmu_smr *smr2, struct arm_smmu_smr *smr) -{ - return (smr->mask & smr2->mask) == smr2->mask && - !((smr->id ^ smr2->id) & ~smr->mask); -} - -/* - * Zero means failure. - */ -static phys_addr_t qsmmuv500_iova_to_phys(struct arm_smmu_domain *smmu_domain, dma_addr_t iova, - u32 sid, unsigned long trans_flags) -{ - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); - struct qsmmuv500_tbu_device *tbu; - phys_addr_t phys = 0; - int idx = cfg->cbndx; - int needs_redo = 0; - u32 sctlr_orig, sctlr, fsr; - unsigned long spinlock_flags; - - tbu = qsmmuv500_find_tbu(smmu, sid); - if (!tbu) - return 0; - - if (arm_smmu_power_on(tbu->pwr)) - return 0; - - if (iova >= (1ULL << tbu->iova_width)) { - dev_err_ratelimited(tbu->dev, "ECATS: address too large: %pad\n", &iova); - return 0; - } - - if (qsmmuv500_tbu_halt(tbu, smmu_domain)) - goto out_power_off; - - /* - * ECATS can trigger the fault interrupt, so disable it temporarily - * and check for an interrupt manually. - */ - sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); - sctlr = sctlr_orig & ~(ARM_SMMU_SCTLR_CFCFG | ARM_SMMU_SCTLR_CFIE); - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); - - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if (fsr & ARM_SMMU_FSR_FAULT) { - /* Clear pending interrupts */ - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - /* - * Barrier required to ensure that the FSR is cleared - * before resuming SMMU operation. - */ - wmb(); - - /* - * TBU halt takes care of resuming any stalled transcation. - * Kept it here for completeness sake. - */ - if (fsr & ARM_SMMU_FSR_SS) - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, - ARM_SMMU_RESUME_TERMINATE); - } - - /* Only one concurrent atos operation */ - spin_lock_irqsave(&data->atos_lock, spinlock_flags); - - /* - * After a failed translation, the next successful translation will - * incorrectly be reported as a failure. - */ - do { - phys = tbu->impl->trigger_atos(tbu, iova, sid, trans_flags); - - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if (fsr & ARM_SMMU_FSR_FAULT) { - /* Clear pending interrupts */ - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - /* - * Barrier required to ensure that the FSR is cleared - * before resuming SMMU operation. - */ - wmb(); - - if (fsr & ARM_SMMU_FSR_SS) - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, - ARM_SMMU_RESUME_TERMINATE); - } - } while (!phys && needs_redo++ < 2); - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); - spin_unlock_irqrestore(&data->atos_lock, spinlock_flags); - qsmmuv500_tbu_resume(tbu); - -out_power_off: - /* Read to complete prior write transcations */ - tbu->impl->write_sync(tbu); - - /* Wait for read to complete before off */ - rmb(); - - arm_smmu_power_off(tbu->smmu, tbu->pwr); - - return phys; -} - -static phys_addr_t qsmmuv500_iova_to_phys_hard( - struct arm_smmu_domain *smmu_domain, - struct qcom_iommu_atos_txn *txn) -{ - return qsmmuv500_iova_to_phys(smmu_domain, txn->addr, txn->id, - txn->flags); -} - -static void qsmmuv500_release_group_iommudata(void *data) -{ - kfree(data); -} - -/* If a device has a valid actlr, it must match */ -static int qsmmuv500_device_group(struct device *dev, - struct iommu_group *group) -{ - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); - struct arm_smmu_device *smmu; - struct qsmmuv500_archdata *data; - struct qsmmuv500_group_iommudata *iommudata; - u32 actlr, i, j, idx; - struct arm_smmu_smr *smr, *smr2; - - if (!fwspec || !cfg) - return -EINVAL; - - smmu = cfg->smmu; - data = to_qsmmuv500_archdata(smmu); - - iommudata = to_qsmmuv500_group_iommudata(group); - if (!iommudata) { - iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL); - if (!iommudata) - return -ENOMEM; - - iommu_group_set_iommudata(group, iommudata, - qsmmuv500_release_group_iommudata); - } - - for (i = 0; i < data->actlr_tbl_size; i++) { - smr = &data->actlrs[i].smr; - actlr = data->actlrs[i].actlr; - - for_each_cfg_sme(cfg, fwspec, j, idx) { - smr2 = &smmu->smrs[idx]; - if (!smr_is_subset(smr2, smr)) - continue; - - dev_dbg(dev, "Matched actlr sid=%x mask=%x actlr=%x\n", - smr->id, smr->mask, actlr); - - if (!iommudata->has_actlr) { - iommudata->actlr = actlr; - iommudata->has_actlr = true; - } else if (iommudata->actlr != actlr) { - dev_err(dev, "Invalid actlr setting\n"); - return -EINVAL; - } - } - } - - return 0; -} - -static void qsmmuv500_init_cb(struct arm_smmu_domain *smmu_domain, - struct device *dev) -{ - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct qsmmuv500_group_iommudata *iommudata = - to_qsmmuv500_group_iommudata(dev->iommu_group); - int idx = smmu_domain->cfg.cbndx; - - if (!iommudata->has_actlr) - return; - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ACTLR, iommudata->actlr); - - /* - * Flush the context bank after modifying ACTLR to ensure there - * are no cache entries with stale state - */ - iommu_flush_iotlb_all(&smmu_domain->domain); -} - -static int qsmmuv500_tbu_register(struct device *dev, void *cookie) -{ - struct qsmmuv500_tbu_device *tbu; - struct qsmmuv500_archdata *data = cookie; - - if (!dev->driver) { - dev_err(dev, "TBU failed probe, QSMMUV500 cannot continue!\n"); - return -EINVAL; - } - - tbu = dev_get_drvdata(dev); - - INIT_LIST_HEAD(&tbu->list); - tbu->smmu = &data->smmu; - list_add(&tbu->list, &data->tbus); - return 0; -} - -static int qsmmuv500_read_actlr_tbl(struct qsmmuv500_archdata *data) -{ - int len, i; - struct device *dev = data->smmu.dev; - struct actlr_setting *actlrs; - const __be32 *cell; - - cell = of_get_property(dev->of_node, "qcom,actlr", NULL); - if (!cell) - return 0; - - len = of_property_count_elems_of_size(dev->of_node, "qcom,actlr", - sizeof(u32) * 3); - if (len < 0) - return 0; - - actlrs = devm_kzalloc(dev, sizeof(*actlrs) * len, GFP_KERNEL); - if (!actlrs) - return -ENOMEM; - - for (i = 0; i < len; i++) { - actlrs[i].smr.id = of_read_number(cell++, 1); - actlrs[i].smr.mask = of_read_number(cell++, 1); - actlrs[i].actlr = of_read_number(cell++, 1); - } - - data->actlrs = actlrs; - data->actlr_tbl_size = len; - return 0; -} - -static int qsmmuv500_cfg_probe(struct arm_smmu_device *smmu) -{ - u32 val; - - val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); - val &= ~ARM_MMU500_ACR_CACHE_LOCK; - arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, val); - val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); - /* - * Modifiying the nonsecure copy of the sACR register is only - * allowed if permission is given in the secure sACR register. - * Attempt to detect if we were able to update the value. - */ - WARN_ON(val & ARM_MMU500_ACR_CACHE_LOCK); - - return 0; -} - -/* - * Case 1) - * Client wants to use the standard upstream dma ops from - * drivers/iommu/dma-iommu.c - * - * This requires domain->type == IOMMU_DOMAIN_DMA. - * - * Case 2) - * Client doesn't want to use the default dma domain, and wants to - * allocate their own via iommu_domain_alloc() - * - * There are insufficient context banks to "waste" one on a default - * dma domain that isn't going to be used. Therefore, give it - * IOMMU_DOMAIN_IDENTITY. Note that IOMMU_DOMAIN_IDENTITY is treated as - * IOMMU_DOMAIN_BLOCKED by our hypervisor, which doesn't allow setting - * S2CR.TYPE = 0x1. - * - * Case 3) - * Client wants to use our fastmap dma ops - * - * Per case 1) we cannot use IOMMU_DOMAIN_DMA, since those imply using - * the standard upstream dma ops. So use IOMMU_DOMAIN_UNMANAGED instead. - * - * Case 4) - * Client wants to use S1 bypass - * - * Same as Case 3, except use the platform dma ops. - */ -static int qsmmuv500_def_domain_type(struct device *dev) -{ - const char *str; - struct device_node *np; - - /* Default to iommu_def_domain_type */ - np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); - if (!np) - np = dev->of_node; - - if (of_property_read_string(np, "qcom,iommu-dma", &str)) - str = "default"; - - if (!strcmp(str, "fastmap") && - IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_FAST)) - return IOMMU_DOMAIN_UNMANAGED; - if (!strcmp(str, "bypass")) - return IOMMU_DOMAIN_UNMANAGED; - if (!strcmp(str, "atomic")) - return IOMMU_DOMAIN_DMA; - if (!strcmp(str, "disabled")) - return IOMMU_DOMAIN_IDENTITY; - - return IOMMU_DOMAIN_DMA; -} - -static const struct arm_smmu_impl qsmmuv500_impl = { - .cfg_probe = qsmmuv500_cfg_probe, - .init_context_bank = qsmmuv500_init_cb, - .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, - .tlb_sync_timeout = qsmmuv500_tlb_sync_timeout, - .device_remove = qsmmuv500_device_remove, - .device_group = qsmmuv500_device_group, - .def_domain_type = qsmmuv500_def_domain_type, -}; - -static const struct arm_smmu_impl qsmmuv500_adreno_impl = { - .init_context = qcom_adreno_smmu_init_context, - .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank, - .cfg_probe = qsmmuv500_cfg_probe, - .init_context_bank = qsmmuv500_init_cb, - .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, - .tlb_sync_timeout = qsmmuv500_tlb_sync_timeout, - .device_remove = qsmmuv500_device_remove, - .device_group = qsmmuv500_device_group, - .def_domain_type = qsmmuv500_def_domain_type, -}; - -/* We only have access to arm-architected registers */ -static const struct arm_smmu_impl qsmmuv500_virt_impl = { - .cfg_probe = qsmmuv500_cfg_probe, - .init_context_bank = qsmmuv500_init_cb, - .device_group = qsmmuv500_device_group, - .def_domain_type = qsmmuv500_def_domain_type, -}; - -struct arm_smmu_device *qsmmuv500_create(struct arm_smmu_device *smmu, - const struct arm_smmu_impl *impl) -{ - struct device *dev = smmu->dev; - struct qsmmuv500_archdata *data; - int ret; - - /* - * devm_krealloc() invokes devm_kmalloc(), so we pass __GFP_ZERO - * to ensure that fields after smmu are initialized, even if we don't - * initialize them (e.g. ACTLR related fields). - */ - data = devm_krealloc(dev, smmu, sizeof(*data), GFP_KERNEL | __GFP_ZERO); - if (!data) - return ERR_PTR(-ENOMEM); - - INIT_LIST_HEAD(&data->tbus); - spin_lock_init(&data->atos_lock); - INIT_WORK(&data->outstanding_tnx_work, - qsmmuv500_log_outstanding_transactions); - data->smmu.impl = impl; - - ret = qsmmuv500_read_actlr_tbl(data); - if (ret) - return ERR_PTR(ret); - - ret = of_platform_populate(dev->of_node, NULL, NULL, dev); - if (ret) - return ERR_PTR(ret); - - /* Attempt to register child devices */ - ret = device_for_each_child(dev, data, qsmmuv500_tbu_register); - if (ret) - return ERR_PTR(-EPROBE_DEFER); - - return &data->smmu; -} - -static struct arm_smmu_device *qsmmuv500_virt_create(struct arm_smmu_device *smmu, - const struct arm_smmu_impl *impl) -{ - struct device *dev = smmu->dev; - struct qsmmuv500_archdata *data; - int ret; - - data = devm_krealloc(dev, smmu, sizeof(*data), GFP_KERNEL | __GFP_ZERO); - if (!data) - return ERR_PTR(-ENOMEM); - - INIT_LIST_HEAD(&data->tbus); - spin_lock_init(&data->atos_lock); - INIT_WORK(&data->outstanding_tnx_work, - qsmmuv500_log_outstanding_transactions); - data->smmu.impl = impl; - - ret = qsmmuv500_read_actlr_tbl(data); - if (ret) - return ERR_PTR(ret); - - return &data->smmu; -} - -struct arm_smmu_device *qsmmuv500_impl_init(struct arm_smmu_device *smmu) -{ - if (of_device_is_compatible(smmu->dev->of_node, "qcom,adreno-smmu")) - return qsmmuv500_create(smmu, &qsmmuv500_adreno_impl); - - if (of_device_is_compatible(smmu->dev->of_node, "qcom,virt-smmu")) - return qsmmuv500_virt_create(smmu, &qsmmuv500_virt_impl); - - return qsmmuv500_create(smmu, &qsmmuv500_impl); -} - static const struct arm_smmu_impl qcom_adreno_smmu_impl = { .init_context = qcom_adreno_smmu_init_context, .def_domain_type = qcom_smmu_def_domain_type, diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-trace.h b/drivers/iommu/arm/arm-smmu/arm-smmu-trace.h deleted file mode 100644 index c6786808c6d7..000000000000 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-trace.h +++ /dev/null @@ -1,224 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2019, 2021 The Linux Foundation. All rights reserved. - */ - -#undef TRACE_SYSTEM -#define TRACE_SYSTEM arm_smmu - -#if !defined(_TRACE_ARM_SMMU_H) || defined(TRACE_HEADER_MULTI_READ) -#define _TRACE_ARM_SMMU_H - -#include -#include -#include -#include "arm-smmu.h" - -struct device; - -DECLARE_EVENT_CLASS(iommu_tlbi, - - TP_PROTO(struct arm_smmu_domain *domain), - - TP_ARGS(domain), - - TP_STRUCT__entry( - __string(group_name, dev_name(domain->dev)) - ), - - TP_fast_assign( - __assign_str(group_name, dev_name(domain->dev)); - ), - - TP_printk("group=%s", - __get_str(group_name) - ) -); - -DEFINE_EVENT(iommu_tlbi, tlbi_start, - - TP_PROTO(struct arm_smmu_domain *domain), - - TP_ARGS(domain) -); - -DEFINE_EVENT(iommu_tlbi, tlbi_end, - - TP_PROTO(struct arm_smmu_domain *domain), - - TP_ARGS(domain) -); - -DECLARE_EVENT_CLASS(iommu_pgtable, - - TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, - unsigned long long ipa, size_t granule), - - TP_ARGS(domain, iova, ipa, granule), - - TP_STRUCT__entry( - __string(group_name, dev_name(domain->dev)) - __field(unsigned long, iova) - __field(unsigned long long, ipa) - __field(size_t, granule) - ), - - TP_fast_assign( - __assign_str(group_name, dev_name(domain->dev)); - __entry->iova = iova; - __entry->ipa = ipa; - __entry->granule = granule; - ), - - TP_printk("group=%s table_base_iova=%lx table_ipa=%llx table_size=%zx", - __get_str(group_name), __entry->iova, - __entry->ipa, __entry->granule - ) -); - -DEFINE_EVENT(iommu_pgtable, iommu_pgtable_add, - - TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, - unsigned long long ipa, size_t granule), - - TP_ARGS(domain, iova, ipa, granule) -); - -DEFINE_EVENT(iommu_pgtable, iommu_pgtable_remove, - - TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, - unsigned long long ipa, size_t granule), - - TP_ARGS(domain, iova, ipa, granule) -); - -DECLARE_EVENT_CLASS(iommu_map_pages, - - TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, - size_t pgsize, size_t pgcount), - - TP_ARGS(domain, iova, pgsize, pgcount), - - TP_STRUCT__entry( - __string(group_name, dev_name(domain->dev)) - __field(unsigned long, iova) - __field(size_t, pgsize) - __field(size_t, pgcount) - ), - - TP_fast_assign( - __assign_str(group_name, dev_name(domain->dev)); - __entry->iova = iova; - __entry->pgsize = pgsize; - __entry->pgcount = pgcount; - ), - - TP_printk("group=%s iova=%lx size=%zx pgsize=%zx pgcount=%zx", - __get_str(group_name), __entry->iova, - __entry->pgsize * __entry->pgcount, - __entry->pgsize, __entry->pgcount - ) -); - -DEFINE_EVENT(iommu_map_pages, map_pages, - - TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, - size_t pgsize, size_t pgcount), - - TP_ARGS(domain, iova, pgsize, pgcount) -); - -DEFINE_EVENT(iommu_map_pages, unmap_pages, - - TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, - size_t pgsize, size_t pgcount), - - TP_ARGS(domain, iova, pgsize, pgcount) -); - -/* Refer to samples/ftrace_events */ -#ifndef __TRACE_EVENT_ARM_SMMU_HELPER_FUNCTIONS -#define __TRACE_EVENT_ARM_SMMU_HELPER_FUNCTIONS -static inline unsigned long sum_scatterlist_length(struct scatterlist *sgl, - unsigned int nents) -{ - int i = 0; - unsigned long sum = 0; - - for (i = 0; i < nents; i++, sgl = sg_next(sgl)) - sum += sgl->length; - - return sum; -} -#endif - -TRACE_EVENT(map_sg, - - TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, - struct scatterlist *sgl, unsigned int nents), - - TP_ARGS(domain, iova, sgl, nents), - - TP_STRUCT__entry( - __string(group_name, dev_name(domain->dev)) - __field(unsigned long, iova) - __field(unsigned long, size) - ), - - TP_fast_assign( - __assign_str(group_name, dev_name(domain->dev)); - __entry->iova = iova; - __entry->size = sum_scatterlist_length(sgl, nents); - ), - - TP_printk("group=%s iova=%lx size=%lx", - __get_str(group_name), __entry->iova, - __entry->size - ) -); - -TRACE_EVENT(tlbsync_timeout, - - TP_PROTO(struct device *dev), - - TP_ARGS(dev), - - TP_STRUCT__entry( - __string(device, dev_name(dev)) - ), - - TP_fast_assign( - __assign_str(device, dev_name(dev)); - ), - - TP_printk("smmu=%s", - __get_str(device) - ) -); - -TRACE_EVENT(smmu_init, - - TP_PROTO(u64 time), - - TP_ARGS(time), - - TP_STRUCT__entry( - __field(u64, time) - ), - - TP_fast_assign( - __entry->time = time; - ), - - TP_printk("ARM SMMU init latency: %lld us", __entry->time) -); -#endif /* _TRACE_ARM_SMMU_H */ - -#undef TRACE_INCLUDE_PATH -#define TRACE_INCLUDE_PATH ../../drivers/iommu/arm/arm-smmu - -#undef TRACE_INCLUDE_FILE -#define TRACE_INCLUDE_FILE arm-smmu-trace - -/* This part must be outside protection */ -#include diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 24a3887a5fd4..004be15302ce 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -13,8 +13,6 @@ * - Non-secure access to the SMMU * - Context fault reporting * - Extended Stream ID (16 bit) - * - * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #define pr_fmt(fmt) "arm-smmu: " fmt @@ -25,8 +23,6 @@ #include #include #include -#include -#include #include #include #include @@ -40,22 +36,11 @@ #include #include #include -#include -#include -#include #include #include #include "arm-smmu.h" -#include "../../iommu-logger.h" -#include "../../qcom-dma-iommu-generic.h" -#include "../../qcom-io-pgtable.h" -#include "../../qcom-io-pgtable-alloc.h" -#include - -#define CREATE_TRACE_POINTS -#include "arm-smmu-trace.h" /* * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU @@ -85,30 +70,6 @@ MODULE_PARM_DESC(disable_bypass, static bool using_legacy_binding, using_generic_binding; -struct arm_smmu_option_prop { - u32 opt; - const char *prop; -}; - -static struct arm_smmu_option_prop arm_smmu_options[] = { - { ARM_SMMU_OPT_FATAL_ASF, "qcom,fatal-asf" }, - { ARM_SMMU_OPT_3LVL_TABLES, "qcom,use-3-lvl-tables" }, - { ARM_SMMU_OPT_NO_ASID_RETENTION, "qcom,no-asid-retention" }, - { ARM_SMMU_OPT_DISABLE_ATOS, "qcom,disable-atos" }, - { ARM_SMMU_OPT_CONTEXT_FAULT_RETRY, "qcom,context-fault-retry" }, - { 0, NULL}, -}; - -static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, - dma_addr_t iova); -static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, - struct qcom_iommu_atos_txn *txn); -static void arm_smmu_destroy_domain_context(struct iommu_domain *domain); - -static int arm_smmu_setup_default_domain(struct device *dev, - struct iommu_domain *domain); -static void __arm_smmu_qcom_tlb_sync(struct iommu_domain *domain); - static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) { if (pm_runtime_enabled(smmu->dev)) @@ -119,10 +80,8 @@ static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu) { - if (pm_runtime_enabled(smmu->dev)) { - pm_runtime_mark_last_busy(smmu->dev); + if (pm_runtime_enabled(smmu->dev)) pm_runtime_put_autosuspend(smmu->dev); - } } static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) @@ -130,160 +89,8 @@ static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) return container_of(dom, struct arm_smmu_domain, domain); } -static void parse_driver_options(struct arm_smmu_device *smmu) -{ - int i = 0; - - do { - if (of_property_read_bool(smmu->dev->of_node, - arm_smmu_options[i].prop)) { - smmu->options |= arm_smmu_options[i].opt; - dev_dbg(smmu->dev, "option %s\n", - arm_smmu_options[i].prop); - } - } while (arm_smmu_options[++i].opt); -} - -static bool is_iommu_pt_coherent(struct arm_smmu_domain *smmu_domain) -{ - if (smmu_domain->force_coherent_walk) - return true; - else if (smmu_domain->smmu && smmu_domain->smmu->dev) - return dev_is_dma_coherent(smmu_domain->smmu->dev); - return false; -} - -static bool arm_smmu_has_secure_vmid(struct arm_smmu_domain *smmu_domain) -{ - return (smmu_domain->secure_vmid != VMID_INVAL); -} - -#ifdef CONFIG_ARM_SMMU_SELFTEST - -static int selftest; -module_param_named(selftest, selftest, int, 0644); -static int irq_count; - -struct arm_smmu_cf_selftest_data { - struct arm_smmu_device *smmu; - int cbndx; -}; - -static DECLARE_WAIT_QUEUE_HEAD(wait_int); -static irqreturn_t arm_smmu_cf_selftest(int irq, void *data) -{ - u32 fsr; - struct irq_data *irq_data = irq_get_irq_data(irq); - struct arm_smmu_cf_selftest_data *cb_data = data; - struct arm_smmu_device *smmu = cb_data->smmu; - int idx = cb_data->cbndx; - unsigned long hwirq = ULONG_MAX; - - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - - irq_count++; - if (irq_data) - hwirq = irq_data->hwirq; - pr_info("Interrupt (irq:%d hwirq:%ld) received, fsr:0x%x\n", - irq, hwirq, fsr); - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - - wake_up(&wait_int); - return IRQ_HANDLED; -} - -static void arm_smmu_interrupt_selftest(struct arm_smmu_device *smmu) -{ - int cb; - int cb_count = 0; - struct arm_smmu_cf_selftest_data *cb_data; - - if (!selftest) - return; - - cb = smmu->num_s2_context_banks; - - if (smmu->version < ARM_SMMU_V2) - return; - - cb_data = kmalloc(sizeof(*cb_data), GFP_KERNEL); - if (!cb_data) - return; - cb_data->smmu = smmu; - - for_each_clear_bit_from(cb, smmu->context_map, - smmu->num_context_banks) { - int irq; - int ret; - u32 reg; - u32 reg_orig; - int irq_cnt; - - irq = smmu->irqs[smmu->num_global_irqs + cb]; - cb_data->cbndx = cb; - - ret = request_threaded_irq(irq, NULL, arm_smmu_cf_selftest, - IRQF_ONESHOT | IRQF_SHARED, - "arm-smmu-context-fault", cb_data); - if (ret < 0) { - dev_err(smmu->dev, - "Failed to request cntx IRQ %d (%u)\n", - cb, irq); - continue; - } - - cb_count++; - irq_cnt = irq_count; - - reg_orig = arm_smmu_cb_read(smmu, cb, ARM_SMMU_CB_SCTLR); - reg = reg_orig | ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE; - - arm_smmu_cb_write(smmu, cb, ARM_SMMU_CB_SCTLR, reg); - dev_info(smmu->dev, "Testing cntx %d irq %d\n", cb, irq); - - /* Make sure ARM_SMMU_CB_SCTLR is configured */ - wmb(); - arm_smmu_cb_write(smmu, cb, ARM_SMMU_CB_FSRRESTORE, - ARM_SMMU_FSR_TF); - - if (!wait_event_timeout(wait_int, (irq_count > irq_cnt), - msecs_to_jiffies(1000))) { - u32 fsr; - - fsr = arm_smmu_cb_read(smmu, cb, ARM_SMMU_CB_FSR); - dev_info(smmu->dev, "timeout cb:%d, irq:%d, fsr:0x%x\n", - cb, irq_cnt, fsr); - - if (!fsr) - dev_err(smmu->dev, "SCTLR = 0x%08x\n", - arm_smmu_cb_read(smmu, cb, - ARM_SMMU_CB_SCTLR)); - } - - /* Make sure ARM_SMMU_CB_FSRRESTORE is written to */ - wmb(); - arm_smmu_cb_write(smmu, cb, ARM_SMMU_CB_SCTLR, reg_orig); - free_irq(irq, cb_data); - } - - kfree(cb_data); - dev_info(smmu->dev, - "Interrupt selftest completed...\n"); - dev_info(smmu->dev, - "Tested %d contexts, received %d interrupts\n", - cb_count, irq_count); - WARN_ON(cb_count != irq_count); - irq_count = 0; -} -#else -static void arm_smmu_interrupt_selftest(struct arm_smmu_device *smmu) -{ -} -#endif - static struct platform_driver arm_smmu_driver; -static struct qcom_iommu_ops arm_smmu_ops; +static struct iommu_ops arm_smmu_ops; #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS static int arm_smmu_bus_init(struct iommu_ops *ops); @@ -359,7 +166,7 @@ static int arm_smmu_register_legacy_master(struct device *dev, } err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode, - &arm_smmu_ops.iommu_ops); + &arm_smmu_ops); if (err) return err; @@ -383,7 +190,7 @@ static int arm_smmu_register_legacy_master(struct device *dev, static int arm_smmu_legacy_bus_init(void) { if (using_legacy_binding) - return arm_smmu_bus_init(&arm_smmu_ops.iommu_ops); + return arm_smmu_bus_init(&arm_smmu_ops); return 0; } device_initcall_sync(arm_smmu_legacy_bus_init); @@ -395,68 +202,33 @@ static int arm_smmu_register_legacy_master(struct device *dev, } #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */ -static int __arm_smmu_alloc_cb(unsigned long *map, int start, int end, - struct device *dev, - struct arm_smmu_domain *smmu_domain) -{ - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); - struct arm_smmu_device *smmu = cfg->smmu; - int idx; - int i; - - for_each_cfg_sme(cfg, fwspec, i, idx) { - if (smmu->s2crs[idx].pinned) - return smmu->s2crs[idx].cbndx; - } - - return __arm_smmu_alloc_bitmap(map, start, end); -} - static void __arm_smmu_free_bitmap(unsigned long *map, int idx) { clear_bit(idx, map); } /* Wait for any pending TLB invalidations to complete */ -static int __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, +static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, int sync, int status) { - unsigned int inc, delay; + unsigned int spin_cnt, delay; u32 reg; - /* - * Allowing an unbounded number of sync requests to be submitted when a - * TBU is not processing sync requests can cause a TBU's command queue - * to fill up. Once the queue is full, subsequent sync requests can - * stall the CPU indefinitely. Avoid this by gating subsequent sync - * requests after the first sync timeout on an SMMU. - */ - if (IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG) && - test_bit(0, &smmu->sync_timed_out)) - return -EINVAL; + if (smmu->impl && unlikely(smmu->impl->tlb_sync)) + return smmu->impl->tlb_sync(smmu, page, sync, status); arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL); - for (delay = 1, inc = 1; delay < TLB_LOOP_TIMEOUT; delay += inc) { - reg = arm_smmu_readl(smmu, page, status); - if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) - return 0; - - cpu_relax(); - udelay(inc); - if (inc < TLB_LOOP_INC_MAX) - inc *= 2; + for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { + for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { + reg = arm_smmu_readl(smmu, page, status); + if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) + return; + cpu_relax(); + } + udelay(delay); } - - if (IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG) && - test_and_set_bit_lock(0, &smmu->sync_timed_out)) - goto out; - - trace_tlbsync_timeout(smmu->dev); - if (smmu->impl && smmu->impl->tlb_sync_timeout) - smmu->impl->tlb_sync_timeout(smmu); -out: - return -EINVAL; + dev_err_ratelimited(smmu->dev, + "TLB sync timed out -- SMMU may be deadlocked\n"); } static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) @@ -464,10 +236,8 @@ static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) unsigned long flags; spin_lock_irqsave(&smmu->global_sync_lock, flags); - if (__arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC, - ARM_SMMU_GR0_sTLBGSTATUS)) - dev_err_ratelimited(smmu->dev, - "TLB global sync failed!\n"); + __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC, + ARM_SMMU_GR0_sTLBGSTATUS); spin_unlock_irqrestore(&smmu->global_sync_lock, flags); } @@ -476,14 +246,10 @@ static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain) struct arm_smmu_device *smmu = smmu_domain->smmu; unsigned long flags; - spin_lock_irqsave(&smmu_domain->sync_lock, flags); - if (__arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx), - ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS)) - dev_err_ratelimited(smmu->dev, - "TLB sync on cb%d failed for device %s\n", - smmu_domain->cfg.cbndx, - dev_name(smmu_domain->dev)); - spin_unlock_irqrestore(&smmu_domain->sync_lock, flags); + spin_lock_irqsave(&smmu_domain->cb_lock, flags); + __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx), + ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS); + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); } static void arm_smmu_tlb_inv_context_s1(void *cookie) @@ -494,11 +260,9 @@ static void arm_smmu_tlb_inv_context_s1(void *cookie) * current CPU are visible beforehand. */ wmb(); - trace_tlbi_start(smmu_domain); arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx, ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid); arm_smmu_tlb_sync_context(smmu_domain); - trace_tlbi_end(smmu_domain); } static void arm_smmu_tlb_inv_context_s2(void *cookie) @@ -642,343 +406,34 @@ static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = { .tlb_add_page = arm_smmu_tlb_add_page_s2_v1, }; -static void print_fault_regs(struct arm_smmu_domain *smmu_domain, - struct arm_smmu_device *smmu, int idx) -{ - u32 fsr, fsynr0, fsynr1, cbfrsynra; - unsigned long iova; - struct arm_smmu_cfg *cfg = smmu->cbs[idx].cfg; - bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; - - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - fsynr0 = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); - fsynr1 = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR1); - iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); - cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); - - dev_err(smmu->dev, "Unhandled arm-smmu context fault from %s!\n", - dev_name(smmu_domain->dev)); - dev_err(smmu->dev, "FAR = 0x%016lx\n", - iova); - dev_err(smmu->dev, "PAR = 0x%pK\n", - (void *) arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR)); - - dev_err(smmu->dev, - "FSR = 0x%08x [%s%s%s%s%s%s%s%s%s%s]\n", - fsr, - (fsr & ARM_SMMU_FSR_TF) ? (fsynr0 & ARM_SMMU_FSYNR0_WNR ? - "TF W " : "TF R ") : "", - (fsr & ARM_SMMU_FSR_AFF) ? "AFF " : "", - (fsr & ARM_SMMU_FSR_PF) ? (fsynr0 & ARM_SMMU_FSYNR0_WNR ? - "PF W " : "PF R ") : "", - (fsr & ARM_SMMU_FSR_EF) ? "EF " : "", - (fsr & ARM_SMMU_FSR_TLBMCF) ? "TLBMCF " : "", - (fsr & ARM_SMMU_FSR_TLBLKF) ? "TLBLKF " : "", - (fsr & ARM_SMMU_FSR_ASF) ? "ASF " : "", - (fsr & ARM_SMMU_FSR_UUT) ? "UUT " : "", - (fsr & ARM_SMMU_FSR_SS) ? "SS " : "", - (fsr & ARM_SMMU_FSR_MULTI) ? "MULTI " : ""); - - dev_err(smmu->dev, "FSYNR0 = 0x%x\n", fsynr0); - dev_err(smmu->dev, "FSYNR1 = 0x%x\n", fsynr1); - dev_err(smmu->dev, "context bank# = 0x%x\n", idx); - - if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { - dev_err(smmu->dev, "TTBR0 = 0x%pK\n", - (void *) (unsigned long) - arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_TTBR0)); - dev_err(smmu->dev, "TTBR1 = 0x%pK\n", - (void *) (unsigned long) - arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_TTBR1)); - } else { - dev_err(smmu->dev, "TTBR0 = 0x%pK\n", - (void *) arm_smmu_cb_readq(smmu, idx, - ARM_SMMU_CB_TTBR0)); - if (stage1) - dev_err(smmu->dev, "TTBR1 = 0x%pK\n", - (void *) arm_smmu_cb_readq(smmu, idx, - ARM_SMMU_CB_TTBR1)); - } - - dev_err(smmu->dev, "SCTLR = 0x%08x ACTLR = 0x%08x\n", - arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR), - arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_ACTLR)); - dev_err(smmu->dev, "CBAR = 0x%08x\n", - arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBAR(idx))); - dev_err(smmu->dev, "MAIR0 = 0x%08x MAIR1 = 0x%08x\n", - arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_S1_MAIR0), - arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_S1_MAIR1)); - - dev_err(smmu->dev, "SID = 0x%x\n", - cbfrsynra & CBFRSYNRA_SID_MASK); - dev_err(smmu->dev, "Client info: BID=0x%lx, PID=0x%lx, MID=0x%lx\n", - FIELD_GET(ARM_SMMU_FSYNR1_BID, fsynr1), - FIELD_GET(ARM_SMMU_FSYNR1_PID, fsynr1), - FIELD_GET(ARM_SMMU_FSYNR1_MID, fsynr1)); -} - -/* - * Iommu HW has generated a fault. If HW and SW states are in sync, - * then a SW page table walk should yield 0. - * - * WARNING!!! This check is racy!!!! - * For a faulting address x, the dma layer mapping may map address x - * into the iommu page table in parallel to the fault handler running. - * This is frequently seen due to dma-iommu's address reuse policy. - * Thus, arm_smmu_iova_to_phys() returning nonzero is not necessarily - * indicative of an issue. - */ -static void arm_smmu_verify_fault(struct arm_smmu_domain *smmu_domain, - struct arm_smmu_device *smmu, int idx) -{ - u32 fsynr, cbfrsynra; - unsigned long iova; - struct iommu_domain *domain = &smmu_domain->domain; - phys_addr_t phys_soft; - phys_addr_t phys_stimu, phys_hard_priv, phys_stimu_post_tlbiall; - unsigned long flags = 0; - struct qcom_iommu_atos_txn txn = {0}; - - fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); - iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); - cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); - - phys_soft = arm_smmu_iova_to_phys(domain, iova); - dev_err(smmu->dev, "soft iova-to-phys=%pa\n", &phys_soft); - - /* Get the transaction type */ - if (fsynr & ARM_SMMU_FSYNR0_WNR) - flags |= IOMMU_TRANS_WRITE; - if (fsynr & ARM_SMMU_FSYNR0_PNU) - flags |= IOMMU_TRANS_PRIV; - if (fsynr & ARM_SMMU_FSYNR0_IND) - flags |= IOMMU_TRANS_INST; - - txn.addr = iova; - txn.flags = flags; - txn.id = cbfrsynra & CBFRSYNRA_SID_MASK; - - /* Now replicate the faulty transaction */ - phys_stimu = arm_smmu_iova_to_phys_hard(domain, &txn); - - /* - * If the replicated transaction fails, it could be due to legitimate - * unmapped access (translation fault) or stale TLB with insufficient - * privileges (permission fault). Try ATOS operation with full access - * privileges to rule out stale entry with insufficient privileges case. - */ - if (!phys_stimu) { - txn.flags = QCOM_IOMMU_ATOS_TRANS_DEFAULT | - QCOM_IOMMU_ATOS_TRANS_PRIV; - phys_hard_priv = arm_smmu_iova_to_phys_hard(domain, &txn); - } - - /* Now replicate the faulty transaction post tlbiall */ - iommu_flush_iotlb_all(domain); - txn.flags = flags; - phys_stimu_post_tlbiall = arm_smmu_iova_to_phys_hard(domain, &txn); - - if (!phys_stimu && phys_hard_priv) { - dev_err(smmu->dev, - "ATOS results differed across access privileges...\n" - "Before: %pa After: %pa\n", - &phys_stimu, &phys_hard_priv); - } - - if (phys_stimu != phys_stimu_post_tlbiall) { - dev_err(smmu->dev, - "ATOS results differed across TLBIALL...\n" - "Before: %pa After: %pa\n", &phys_stimu, - &phys_stimu_post_tlbiall); - } - - dev_err(smmu->dev, "hard iova-to-phys (ATOS)=%pa\n", - phys_stimu ? &phys_stimu : &phys_stimu_post_tlbiall); -} - -static int report_iommu_fault_helper(struct arm_smmu_domain *smmu_domain, - struct arm_smmu_device *smmu, int idx) -{ - u32 fsr, fsynr; - unsigned long iova; - int flags; - - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); - iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); - - flags = fsynr & ARM_SMMU_FSYNR0_WNR ? - IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; - if (fsr & ARM_SMMU_FSR_TF) - flags |= IOMMU_FAULT_TRANSLATION; - if (fsr & ARM_SMMU_FSR_PF) - flags |= IOMMU_FAULT_PERMISSION; - if (fsr & ARM_SMMU_FSR_EF) - flags |= IOMMU_FAULT_EXTERNAL; - if (fsr & ARM_SMMU_FSR_SS) - flags |= IOMMU_FAULT_TRANSACTION_STALLED; - - - return report_iommu_fault(&smmu_domain->domain, - smmu->dev, iova, flags); -} - -static int arm_smmu_get_fault_ids(struct iommu_domain *domain, - struct qcom_iommu_fault_ids *f_ids) -{ - struct arm_smmu_domain *smmu_domain; - struct arm_smmu_device *smmu; - u32 fsr, fsynr1; - int idx, ret; - - if (!domain || !f_ids) - return -EINVAL; - - smmu_domain = to_smmu_domain(domain); - smmu = smmu_domain->smmu; - idx = smmu_domain->cfg.cbndx; - - ret = arm_smmu_rpm_get(smmu); - if (ret < 0) - return ret; - - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - - if (!(fsr & ARM_SMMU_FSR_FAULT)) { - arm_smmu_rpm_put(smmu); - return -EINVAL; - } - - fsynr1 = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR1); - arm_smmu_rpm_put(smmu); - - f_ids->bid = FIELD_GET(ARM_SMMU_FSYNR1_BID, fsynr1); - f_ids->pid = FIELD_GET(ARM_SMMU_FSYNR1_PID, fsynr1); - f_ids->mid = FIELD_GET(ARM_SMMU_FSYNR1_MID, fsynr1); - - return 0; -} - -#ifdef CONFIG_ARM_SMMU_CONTEXT_FAULT_RETRY -/* - * Retry faulting address after tlb invalidate. - * Applicable to: Waipio - */ -static irqreturn_t arm_smmu_context_fault_retry(struct arm_smmu_domain *smmu_domain) -{ - struct arm_smmu_device *smmu = smmu_domain->smmu; - int idx = smmu_domain->cfg.cbndx; - u64 iova; - u32 fsr; - - if (!(smmu->options & ARM_SMMU_OPT_CONTEXT_FAULT_RETRY) || - (smmu_domain->fault_model.no_stall)) - return IRQ_NONE; - - iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - - if (iova != smmu_domain->prev_fault_address || - !smmu_domain->fault_retry_counter) { - smmu_domain->prev_fault_address = iova; - smmu_domain->fault_retry_counter++; - arm_smmu_tlb_inv_context_s1(smmu_domain); - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - /* - * Barrier required to ensure that the FSR is cleared - * before resuming SMMU operation - */ - wmb(); - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, - ARM_SMMU_RESUME_RESUME); - return IRQ_HANDLED; - } - - return IRQ_NONE; -} -#else -static irqreturn_t arm_smmu_context_fault_retry(struct arm_smmu_domain *smmu_domain) -{ - return IRQ_NONE; -} -#endif - static irqreturn_t arm_smmu_context_fault(int irq, void *dev) { - u32 fsr; + u32 fsr, fsynr, cbfrsynra; + unsigned long iova; struct iommu_domain *domain = dev; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; int idx = smmu_domain->cfg.cbndx; - static DEFINE_RATELIMIT_STATE(_rs, - DEFAULT_RATELIMIT_INTERVAL, - DEFAULT_RATELIMIT_BURST); int ret; - ret = arm_smmu_rpm_get(smmu); - if (ret < 0) + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (!(fsr & ARM_SMMU_FSR_FAULT)) return IRQ_NONE; - fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if (!(fsr & ARM_SMMU_FSR_FAULT)) { - ret = IRQ_NONE; - goto out_power_off; - } + fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); + iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); + cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); - if ((smmu->options & ARM_SMMU_OPT_FATAL_ASF) && - (fsr & ARM_SMMU_FSR_ASF)) { - dev_err(smmu->dev, - "Took an address size fault. Refusing to recover.\n"); - BUG(); - } + ret = report_iommu_fault(domain, NULL, iova, + fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); - ret = arm_smmu_context_fault_retry(smmu_domain); - if (ret == IRQ_HANDLED) - goto out_power_off; + if (ret == -ENOSYS) + dev_err_ratelimited(smmu->dev, + "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", + fsr, iova, fsynr, cbfrsynra, idx); - /* - * If the fault helper returns -ENOSYS, then no client fault helper was - * registered. In that case, print the default report. - * - * If the client returns -EBUSY, do not clear FSR and do not RESUME - * if stalled. This is required to keep the IOMMU client stalled on - * the outstanding fault. This gives the client a chance to take any - * debug action and then terminate the stalled transaction. - * So, the sequence in case of stall on fault should be: - * 1) Do not clear FSR or write to RESUME here - * 2) Client takes any debug action - * 3) Client terminates the stalled transaction and resumes the IOMMU - * 4) Client clears FSR. The FSR should only be cleared after 3) and - * not before so that the fault remains outstanding. This ensures - * SCTLR.HUPCF has the desired effect if subsequent transactions also - * need to be terminated. - */ - ret = report_iommu_fault_helper(smmu_domain, smmu, idx); - if (ret == -ENOSYS) { - if (__ratelimit(&_rs)) { - print_fault_regs(smmu_domain, smmu, idx); - arm_smmu_verify_fault(smmu_domain, smmu, idx); - } - BUG_ON(!smmu_domain->fault_model.non_fatal); - } - if (ret != -EBUSY) { - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - - /* - * Barrier required to ensure that the FSR is cleared - * before resuming SMMU operation - */ - wmb(); - - if (fsr & ARM_SMMU_FSR_SS) - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, - ARM_SMMU_RESUME_TERMINATE); - } - - ret = IRQ_HANDLED; -out_power_off: - arm_smmu_rpm_put(smmu); - return ret; + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + return IRQ_HANDLED; } static irqreturn_t arm_smmu_global_fault(int irq, void *dev) @@ -987,21 +442,14 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) struct arm_smmu_device *smmu = dev; static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); - int ret; - - ret = arm_smmu_rpm_get(smmu); - if (ret < 0) - return IRQ_NONE; gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0); gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1); gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2); - if (!gfsr) { - arm_smmu_rpm_put(smmu); + if (!gfsr) return IRQ_NONE; - } if (__ratelimit(&rs)) { if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) && @@ -1018,7 +466,6 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) } arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr); - arm_smmu_rpm_put(smmu); return IRQ_HANDLED; } @@ -1077,33 +524,6 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32; } } - - memset(&cfg->sctlr, 0, sizeof(cfg->sctlr)); - /* - * Override cacheability, shareability, r/w allocation for - * clients who are io-coherent. Otherwise set NSH to force io-coherency - * to be disabled. - * These settings only take effect during bypass mode, when sctlr.M = 0 - */ - if (of_dma_is_coherent(smmu_domain->dev->of_node)) { - cfg->sctlr.wacfg = ARM_SMMU_SCTLR_WACFG_WA; - cfg->sctlr.racfg = ARM_SMMU_SCTLR_RACFG_RA; - cfg->sctlr.shcfg = ARM_SMMU_SCTLR_SHCFG_OSH; - cfg->sctlr.mtcfg = 1; - cfg->sctlr.memattr = ARM_SMMU_SCTLR_MEM_ATTR_OISH_WB_CACHE; - } else { - cfg->sctlr.shcfg = ARM_SMMU_SCTLR_SHCFG_NSH; - } - - cfg->sctlr.cfre = !smmu_domain->fault_model.no_cfre; - cfg->sctlr.cfcfg = !smmu_domain->fault_model.no_stall; - cfg->sctlr.hupcf = smmu_domain->fault_model.hupcf; - - if ((!smmu_domain->mapping_cfg.s1_bypass && !smmu_domain->delayed_s1_trans_enable) || - !stage1) - cfg->sctlr.m = 1; - - cb->sctlr = arm_smmu_lpae_sctlr(cfg); } void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) @@ -1182,87 +602,19 @@ void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) } /* SCTLR */ + reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE | + ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M; + if (stage1) + reg |= ARM_SMMU_SCTLR_S1_ASIDPNE; + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) + reg |= ARM_SMMU_SCTLR_E; + if (smmu->impl && smmu->impl->write_sctlr) - smmu->impl->write_sctlr(smmu, idx, cb->sctlr); + smmu->impl->write_sctlr(smmu, idx, reg); else - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, cb->sctlr); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg); } -/* This function assumes that the domain's init mutex is held */ -static int arm_smmu_get_dma_cookie(struct device *dev, - struct arm_smmu_domain *smmu_domain, - struct io_pgtable_ops *pgtbl_ops) -{ - bool fast = smmu_domain->mapping_cfg.fast; - - /* DMA cookie is allocated by the IOMMU core for DMA domains. */ - return fast ? fast_smmu_init_mapping(dev, &smmu_domain->domain, pgtbl_ops) : 0; -} - -static void arm_smmu_put_dma_cookie(struct iommu_domain *domain) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - - /* DMA cookie is freed by the IOMMU core for DMA domains. */ - if (smmu_domain->mapping_cfg.fast) - fast_smmu_put_dma_cookie(domain); -} - -static void arm_smmu_log_new_table(void *cookie, void *virt, unsigned long iova, size_t granule) -{ - struct arm_smmu_domain *smmu_domain = cookie; - - trace_iommu_pgtable_add(smmu_domain, iova, __pa(virt), granule); -} - -static void arm_smmu_log_remove_table(void *cookie, void *virt, unsigned long iova, size_t granule) -{ - struct arm_smmu_domain *smmu_domain = cookie; - - trace_iommu_pgtable_remove(smmu_domain, iova, __pa(virt), granule); -} - -static void arm_smmu_tlb_add_walk_page(void *cookie, void *virt) -{ - struct arm_smmu_domain *smmu_domain = cookie; - struct page *page = virt_to_page(virt); - unsigned long flags; - - spin_lock_irqsave(&smmu_domain->iotlb_gather_lock, flags); - smmu_domain->deferred_sync = true; - page->freelist = smmu_domain->freelist; - smmu_domain->freelist = page; - spin_unlock_irqrestore(&smmu_domain->iotlb_gather_lock, flags); -} - -static void arm_smmu_qcom_tlb_add_inv(void *cookie) -{ - struct arm_smmu_domain *smmu_domain = cookie; - unsigned long flags; - - spin_lock_irqsave(&smmu_domain->iotlb_gather_lock, flags); - smmu_domain->deferred_sync = true; - spin_unlock_irqrestore(&smmu_domain->iotlb_gather_lock, flags); -} - -static void arm_smmu_qcom_tlb_sync(void *cookie) -{ - struct arm_smmu_domain *smmu_domain = cookie; - - __arm_smmu_qcom_tlb_sync(&smmu_domain->domain); -} - -static const struct qcom_iommu_pgtable_log_ops arm_smmu_pgtable_log_ops = { - .log_new_table = arm_smmu_log_new_table, - .log_remove_table = arm_smmu_log_remove_table, -}; - -static const struct qcom_iommu_flush_ops arm_smmu_iotlb_ops = { - .tlb_add_walk_page = arm_smmu_tlb_add_walk_page, - .tlb_add_inv = arm_smmu_qcom_tlb_add_inv, - .tlb_sync = arm_smmu_qcom_tlb_sync, -}; - static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain, struct arm_smmu_device *smmu, struct device *dev, unsigned int start) @@ -1270,8 +622,7 @@ static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain, if (smmu->impl && smmu->impl->alloc_context_bank) return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start); - return __arm_smmu_alloc_cb(smmu->context_map, start, smmu->num_context_banks, dev, - smmu_domain); + return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); } static int arm_smmu_init_domain_context(struct iommu_domain *domain, @@ -1281,26 +632,16 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, int irq, start, ret = 0; unsigned long ias, oas; struct io_pgtable_ops *pgtbl_ops; - struct qcom_io_pgtable_info pgtbl_info = {}; - struct io_pgtable_cfg *pgtbl_cfg = &pgtbl_info.cfg; + struct io_pgtable_cfg pgtbl_cfg; enum io_pgtable_fmt fmt; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_cfg *cfg = &smmu_domain->cfg; irqreturn_t (*context_fault)(int irq, void *dev); - struct io_pgtable *iop; mutex_lock(&smmu_domain->init_mutex); if (smmu_domain->smmu) goto out_unlock; - smmu_domain->dev = dev; - ret = arm_smmu_setup_default_domain(dev, domain); - if (ret) { - dev_err(dev, "%s: default domain setup failed\n", - __func__); - goto out_unlock; - } - if (domain->type == IOMMU_DOMAIN_IDENTITY) { smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS; smmu_domain->smmu = smmu; @@ -1363,9 +704,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, ias = smmu->va_size; oas = smmu->ipa_size; if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { - fmt = QCOM_ARM_64_LPAE_S1; - if (smmu->options & ARM_SMMU_OPT_3LVL_TABLES) - ias = min(ias, 39UL); + fmt = ARM_64_LPAE_S1; } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { fmt = ARM_32_LPAE_S1; ias = min(ias, 32UL); @@ -1404,17 +743,6 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, goto out_unlock; } - if (smmu_domain->mapping_cfg.fast) { - fmt = ARM_V8L_FAST; - ret = qcom_iommu_get_fast_iova_range(dev, - &pgtbl_info.iova_base, - &pgtbl_info.iova_end); - if (ret < 0) - goto out_unlock; - } else if (arm_smmu_has_secure_vmid(smmu_domain)) { - pgtbl_info.vmid = smmu_domain->secure_vmid; - } - ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start); if (ret < 0) { goto out_unlock; @@ -1435,44 +763,34 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, else cfg->asid = cfg->cbndx; - pgtbl_info.iommu_tlb_ops = &arm_smmu_iotlb_ops; - pgtbl_info.pgtable_log_ops = &arm_smmu_pgtable_log_ops; - pgtbl_info.cfg = (struct io_pgtable_cfg) { + pgtbl_cfg = (struct io_pgtable_cfg) { .pgsize_bitmap = smmu->pgsize_bitmap, .ias = ias, .oas = oas, - .coherent_walk = is_iommu_pt_coherent(smmu_domain), + .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK, .tlb = smmu_domain->flush_ops, .iommu_dev = smmu->dev, }; if (smmu->impl && smmu->impl->init_context) { - ret = smmu->impl->init_context(smmu_domain, pgtbl_cfg, dev); + ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev); if (ret) goto out_clear_smmu; } if (smmu_domain->pgtbl_quirks) - pgtbl_cfg->quirks |= smmu_domain->pgtbl_quirks; + pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks; - pgtbl_ops = qcom_alloc_io_pgtable_ops(fmt, &pgtbl_info, smmu_domain); + pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); if (!pgtbl_ops) { ret = -ENOMEM; goto out_clear_smmu; } - iop = container_of(pgtbl_ops, struct io_pgtable, ops); - ret = iommu_logger_register(&smmu_domain->logger, domain, - smmu_domain->dev, iop); - if (ret) { - dev_err(dev, "Log registration failed\n"); - goto out_free_io_pgtable; - } - /* Update the domain's page sizes to reflect the page table format */ - domain->pgsize_bitmap = pgtbl_cfg->pgsize_bitmap; + domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; - if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) { + if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) { domain->geometry.aperture_start = ~0UL << ias; domain->geometry.aperture_end = ~0UL; } else { @@ -1481,23 +799,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, domain->geometry.force_aperture = true; - ret = arm_smmu_get_dma_cookie(dev, smmu_domain, pgtbl_ops); - if (ret) - goto out_logger; - - /* - * Matches with call to arm_smmu_rpm_put in - * arm_smmu_destroy_domain_context. - */ - if (smmu_domain->mapping_cfg.atomic) { - smmu_domain->rpm_always_on = true; - arm_smmu_rpm_get(smmu); - } - /* Initialise the context bank with our page table cfg */ - arm_smmu_init_context_bank(smmu_domain, pgtbl_cfg); - if (smmu->impl && smmu->impl->init_context_bank) - smmu->impl->init_context_bank(smmu_domain, dev); + arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); arm_smmu_write_context_bank(smmu, cfg->cbndx); /* @@ -1511,9 +814,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, else context_fault = arm_smmu_context_fault; - ret = devm_request_threaded_irq(smmu->dev, irq, NULL, - context_fault, IRQF_ONESHOT | IRQF_SHARED, - "arm-smmu-context-fault", domain); + ret = devm_request_irq(smmu->dev, irq, context_fault, + IRQF_SHARED, "arm-smmu-context-fault", domain); if (ret < 0) { dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", cfg->irptndx, irq); @@ -1526,11 +828,6 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, smmu_domain->pgtbl_ops = pgtbl_ops; return 0; -out_logger: - iommu_logger_unregister(smmu_domain->logger); - smmu_domain->logger = NULL; -out_free_io_pgtable: - qcom_free_io_pgtable_ops(smmu_domain->pgtbl_ops); out_clear_smmu: __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); smmu_domain->smmu = NULL; @@ -1544,8 +841,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - int ret, irq, i; - bool pinned = false; + int ret, irq; if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY) return; @@ -1554,13 +850,6 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) if (ret < 0) return; - /* - * Matches with call to arm_smmu_rpm_get in - * arm_smmu_init_domain_contxt. - */ - if (smmu_domain->rpm_always_on) - arm_smmu_rpm_put(smmu); - /* * Disable the context bank and free the page tables before freeing * it. @@ -1573,16 +862,8 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) devm_free_irq(smmu->dev, irq, domain); } - qcom_free_io_pgtable_ops(smmu_domain->pgtbl_ops); - - for (i = 0; i < smmu->num_mapping_groups; i++) - if ((cfg->cbndx == smmu->s2crs[i].cbndx) && - (smmu->s2crs[i].pinned)) { - pinned = true; - } - - if (!pinned) - __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); + free_io_pgtable_ops(smmu_domain->pgtbl_ops); + __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); arm_smmu_rpm_put(smmu); } @@ -1607,9 +888,6 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) mutex_init(&smmu_domain->init_mutex); spin_lock_init(&smmu_domain->cb_lock); - spin_lock_init(&smmu_domain->sync_lock); - spin_lock_init(&smmu_domain->iotlb_gather_lock); - smmu_domain->secure_vmid = VMID_INVAL; return &smmu_domain->domain; } @@ -1622,9 +900,7 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) * Free the domain resources. We assume that all devices have * already been detached. */ - arm_smmu_put_dma_cookie(domain); arm_smmu_destroy_domain_context(domain); - iommu_logger_unregister(smmu_domain->logger); kfree(smmu_domain); } @@ -1651,8 +927,7 @@ static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) | FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) | - FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg) | - FIELD_PREP(ARM_SMMU_S2CR_SHCFG, ARM_SMMU_S2CR_SHCFG_NSH); + FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg); if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && smmu->smrs[idx].valid) @@ -1751,48 +1026,22 @@ static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) { - bool pinned = smmu->s2crs[idx].pinned; - u8 cbndx = smmu->s2crs[idx].cbndx; - if (--smmu->s2crs[idx].count) return false; smmu->s2crs[idx] = s2cr_init_val; - if (pinned) { - smmu->s2crs[idx].pinned = true; - smmu->s2crs[idx].cbndx = cbndx; - } else if (smmu->smrs) { + if (smmu->smrs) smmu->smrs[idx].valid = false; - } return true; } -static struct device_node *arm_smmu_get_of_node(struct device *dev) -{ - struct device_node *np; - - if (!dev->of_node) - return NULL; - - np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); - return np ? np : dev->of_node; -} - -static bool dev_defer_smr_configuration(struct device *dev) -{ - struct device_node *np = arm_smmu_get_of_node(dev); - - return of_property_read_bool(np, "qcom,iommu-defer-smr-config"); -} - static int arm_smmu_master_alloc_smes(struct device *dev) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); struct arm_smmu_device *smmu = cfg->smmu; struct arm_smmu_smr *smrs = smmu->smrs; - bool config_smrs = !dev_defer_smr_configuration(dev); int i, idx, ret; mutex_lock(&smmu->stream_map_mutex); @@ -1814,20 +1063,17 @@ static int arm_smmu_master_alloc_smes(struct device *dev) if (smrs && smmu->s2crs[idx].count == 0) { smrs[idx].id = sid; smrs[idx].mask = mask; - smrs[idx].valid = config_smrs; - } else if (smrs && WARN_ON(smrs[idx].valid != config_smrs)) { - ret = -EINVAL; - goto out_err; + smrs[idx].valid = true; } smmu->s2crs[idx].count++; cfg->smendx[i] = (s16)idx; } - mutex_unlock(&smmu->stream_map_mutex); /* It worked! Now, poke the actual hardware */ for_each_cfg_sme(cfg, fwspec, i, idx) arm_smmu_write_sme(smmu, idx); + mutex_unlock(&smmu->stream_map_mutex); return 0; out_err: @@ -1869,7 +1115,6 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, else type = S2CR_TYPE_TRANS; - mutex_lock(&smmu->stream_map_mutex); for_each_cfg_sme(cfg, fwspec, i, idx) { if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) continue; @@ -1879,146 +1124,9 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, s2cr[idx].cbndx = cbndx; arm_smmu_write_s2cr(smmu, idx); } - mutex_unlock(&smmu->stream_map_mutex); - return 0; } -static int arm_smmu_setup_default_domain(struct device *dev, - struct iommu_domain *domain) -{ - struct device_node *np; - int ret; - const char *str; - u32 val; - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - - np = arm_smmu_get_of_node(dev); - if (!np) - return 0; - - ret = of_property_read_string(np, "qcom,iommu-dma", &str); - if (ret) - str = "default"; - - if (!strcmp(str, "bypass")) { - smmu_domain->mapping_cfg.s1_bypass = 1; - } else if (!strcmp(str, "fastmap")) { - /* - * Fallback to the upstream dma-allocator if fastmap is not enabled. - * "fastmap" implies "atomic" due to it not calling arm_smmu_rpm_get() - * in its map/unmap functions. Its clients may or may not actually - * use iommu apis from atomic context. - */ - smmu_domain->mapping_cfg.atomic = 1; - if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_FAST)) - smmu_domain->mapping_cfg.fast = 1; - } else if (!strcmp(str, "atomic")) { - smmu_domain->mapping_cfg.atomic = 1; - } else if (!strcmp(str, "disabled")) { - /* DT properties only intended for use by default-domains */ - return 0; - } - - /* - * default value: - * Stall-on-fault - * faults trigger kernel panic - * return abort - */ - if (of_property_match_string(np, "qcom,iommu-faults", - "stall-disable") >= 0) - smmu_domain->fault_model.no_stall = 1; - - if (of_property_match_string(np, "qcom,iommu-faults", "no-CFRE") >= 0) - smmu_domain->fault_model.no_cfre = 1; - - if (of_property_match_string(np, "qcom,iommu-faults", "HUPCF") >= 0) - smmu_domain->fault_model.hupcf = 1; - - if (of_property_match_string(np, "qcom,iommu-faults", "non-fatal") >= 0) - smmu_domain->fault_model.non_fatal = 1; - - /* Default value: disabled */ - ret = of_property_read_u32(np, "qcom,iommu-vmid", &val); - if (!ret) - smmu_domain->secure_vmid = val; - - /* Default value: disabled */ - ret = of_property_read_string(np, "qcom,iommu-pagetable", &str); - if (ret) - str = "disabled"; - if (!strcmp(str, "coherent")) - smmu_domain->force_coherent_walk = true; - else if (!strcmp(str, "LLC")) - smmu_domain->pgtbl_quirks = IO_PGTABLE_QUIRK_ARM_OUTER_WBWA; - else if (!strcmp(str, "LLC_NWA")) - smmu_domain->pgtbl_quirks = IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA; - - /* Default value: disabled */ - if (of_property_read_bool(np, "qcom,iommu-earlymap")) - smmu_domain->delayed_s1_trans_enable = true; - - return 0; -} - -struct lookup_iommu_group_data { - struct device_node *np; - struct iommu_group *group; -}; - -/* This isn't a "fast lookup" since its N^2, but probably good enough */ -static int __bus_lookup_iommu_group(struct device *dev, void *priv) -{ - struct lookup_iommu_group_data *data = priv; - struct device_node *np; - struct iommu_group *group; - - group = iommu_group_get(dev); - if (!group) - return 0; - - np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); - if (np != data->np) { - iommu_group_put(group); - return 0; - } - - data->group = group; - return 1; -} - -static struct iommu_group *of_get_device_group(struct device *dev) -{ - struct lookup_iommu_group_data data = { - .np = NULL, - .group = NULL, - }; - struct iommu_group *group; - int ret; - - data.np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); - if (!data.np) - return NULL; - - ret = bus_for_each_dev(&platform_bus_type, NULL, &data, - __bus_lookup_iommu_group); - if (ret > 0) - return data.group; - -#ifdef CONFIG_PCI - ret = bus_for_each_dev(&pci_bus_type, NULL, &data, - __bus_lookup_iommu_group); - if (ret > 0) - return data.group; -#endif - - group = generic_device_group(dev); - if (IS_ERR(group)) - return NULL; - return group; -} - static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); @@ -2027,7 +1135,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) struct arm_smmu_device *smmu; int ret; - if (!fwspec || fwspec->ops != &arm_smmu_ops.iommu_ops) { + if (!fwspec || fwspec->ops != &arm_smmu_ops) { dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); return -ENXIO; } @@ -2085,100 +1193,41 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) rpm_put: arm_smmu_rpm_put(smmu); - return ret; } -static gfp_t arm_smmu_domain_gfp_flags(struct arm_smmu_domain *smmu_domain) -{ - /* - * The dma layer always uses GFP_ATOMIC, which isn't indicative of - * the actual client needs. - */ - if (smmu_domain->mapping_cfg.atomic) - return GFP_ATOMIC; - - return GFP_KERNEL; -} - static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) { struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); int ret; if (!ops) return -ENODEV; - ret = arm_smmu_rpm_get(smmu); - if (ret < 0) - return ret; - - gfp = arm_smmu_domain_gfp_flags(smmu_domain); + arm_smmu_rpm_get(smmu); ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped); - arm_smmu_rpm_put(smmu); - if (!ret) - trace_map_pages(smmu_domain, iova, pgsize, pgcount); - - return ret; -} - -static int __maybe_unused arm_smmu_map_sg(struct iommu_domain *domain, unsigned long iova, - struct scatterlist *sg, unsigned int nents, int prot, - gfp_t gfp, size_t *mapped) -{ - struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; - struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - int ret, i; - struct scatterlist *tmp; - - if (!ops) - return -ENODEV; - - ret = arm_smmu_rpm_get(smmu); - if (ret < 0) - return ret; - - gfp = arm_smmu_domain_gfp_flags(smmu_domain); - /* - * Use ops->map_sg() when it is available. While this function is unreachable, this - * has to be implemented as such to avoid the usage of ops->map_sg(), as it - * does not exist. - */ - for_each_sg(sg, tmp, nents, i) { - ret = ops->map(ops, iova + *mapped, sg_phys(tmp), tmp->length, prot, gfp); - if (ret) - break; - - *mapped += tmp->length; - } - - arm_smmu_rpm_put(smmu); - if (!ret) - trace_map_sg(smmu_domain, iova, sg, nents); return ret; } static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, size_t pgsize, size_t pgcount, - struct iommu_iotlb_gather *gather) + struct iommu_iotlb_gather *iotlb_gather) { - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; + struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; size_t ret; if (!ops) return 0; - ret = ops->unmap_pages(ops, iova, pgsize, pgcount, gather); - if (ret) - trace_unmap_pages(smmu_domain, iova, pgsize, pgcount); + arm_smmu_rpm_get(smmu); + ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather); + arm_smmu_rpm_put(smmu); return ret; } @@ -2195,39 +1244,6 @@ static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) } } -/* Caller must call arm_smmu_rpm_get() beforehand. */ -static void __arm_smmu_qcom_tlb_sync(struct iommu_domain *domain) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct page *freelist, *page; - unsigned long flags; - - spin_lock_irqsave(&smmu_domain->iotlb_gather_lock, flags); - if (!smmu_domain->deferred_sync) { - spin_unlock_irqrestore(&smmu_domain->iotlb_gather_lock, flags); - return; - } - - if (smmu->version == ARM_SMMU_V2 || - smmu_domain->stage == ARM_SMMU_DOMAIN_S1) - arm_smmu_tlb_inv_context_s1(smmu_domain); - else - arm_smmu_tlb_sync_global(smmu); - - smmu_domain->deferred_sync = false; - - freelist = smmu_domain->freelist; - smmu_domain->freelist = NULL; - - while (freelist) { - page = freelist; - freelist = page->freelist; - qcom_io_pgtable_free_page(page); - } - spin_unlock_irqrestore(&smmu_domain->iotlb_gather_lock, flags); -} - static void arm_smmu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { @@ -2238,25 +1254,33 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain, return; arm_smmu_rpm_get(smmu); - __arm_smmu_qcom_tlb_sync(domain); + if (smmu->version == ARM_SMMU_V2 || + smmu_domain->stage == ARM_SMMU_DOMAIN_S1) + arm_smmu_tlb_sync_context(smmu_domain); + else + arm_smmu_tlb_sync_global(smmu); arm_smmu_rpm_put(smmu); } -static phys_addr_t __arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, +static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, dma_addr_t iova) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; struct device *dev = smmu->dev; void __iomem *reg; u32 tmp; u64 phys; unsigned long va, flags; - int idx = cfg->cbndx; + int ret, idx = cfg->cbndx; phys_addr_t addr = 0; + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + return 0; + spin_lock_irqsave(&smmu_domain->cb_lock, flags); va = iova & ~0xfffUL; if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) @@ -2271,6 +1295,7 @@ static phys_addr_t __arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, dev_err(dev, "iova to phys timed out on %pad. Falling back to software table walk.\n", &iova); + arm_smmu_rpm_put(smmu); return ops->iova_to_phys(ops, iova); } @@ -2284,6 +1309,7 @@ static phys_addr_t __arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); out: + arm_smmu_rpm_put(smmu); return addr; } @@ -2293,60 +1319,17 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; - struct arm_smmu_device *smmu = smmu_domain->smmu; - phys_addr_t phys; if (!ops) return 0; if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && - smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { - if (arm_smmu_rpm_get(smmu) < 0) - return 0; - - phys = __arm_smmu_iova_to_phys_hard(domain, iova); - - arm_smmu_rpm_put(smmu); - - return phys; - } + smmu_domain->stage == ARM_SMMU_DOMAIN_S1) + return arm_smmu_iova_to_phys_hard(domain, iova); return ops->iova_to_phys(ops, iova); } -/* - * This function can sleep, and cannot be called from atomic context. Will - * power on register block if required. This restriction does not apply to the - * original iova_to_phys() op. - */ -static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, - struct qcom_iommu_atos_txn *txn) -{ - phys_addr_t ret = 0; - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_device *smmu = smmu_domain->smmu; - - if (smmu->options & ARM_SMMU_OPT_DISABLE_ATOS) - return 0; - - if (arm_smmu_rpm_get(smmu) < 0) - return 0; - - if (smmu->impl && smmu->impl->iova_to_phys_hard) { - ret = smmu->impl->iova_to_phys_hard(smmu_domain, txn); - goto out; - } - - if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && - smmu_domain->stage == ARM_SMMU_DOMAIN_S1) - ret = __arm_smmu_iova_to_phys_hard(domain, txn->addr); - -out: - arm_smmu_rpm_put(smmu); - - return ret; -} - static bool arm_smmu_capable(enum iommu_cap cap) { switch (cap) { @@ -2390,10 +1373,8 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) fwspec = dev_iommu_fwspec_get(dev); if (ret) goto out_free; - } else if (fwspec && fwspec->ops == &arm_smmu_ops.iommu_ops) { + } else if (fwspec && fwspec->ops == &arm_smmu_ops) { smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); - if (!smmu) - return ERR_PTR(-ENODEV); } else { return ERR_PTR(-ENODEV); } @@ -2455,7 +1436,7 @@ static void arm_smmu_release_device(struct device *dev) struct arm_smmu_device *smmu; int ret; - if (!fwspec || fwspec->ops != &arm_smmu_ops.iommu_ops) + if (!fwspec || fwspec->ops != &arm_smmu_ops) return; cfg = dev_iommu_priv_get(dev); @@ -2495,15 +1476,9 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev) int i, idx; mutex_lock(&smmu->stream_map_mutex); - group = of_get_device_group(dev); - if (group) - goto finish; - for_each_cfg_sme(cfg, fwspec, i, idx) { if (group && smmu->s2crs[idx].group && group != smmu->s2crs[idx].group) { - dev_err(dev, "ID:%x IDX:%x is already in a group!\n", - fwspec->ids[i], idx); mutex_unlock(&smmu->stream_map_mutex); return ERR_PTR(-EINVAL); } @@ -2512,28 +1487,17 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev) } if (group) { - iommu_group_ref_get(group); - } else { - if (dev_is_pci(dev)) - group = pci_device_group(dev); - else if (dev_is_fsl_mc(dev)) - group = fsl_mc_device_group(dev); - else - group = generic_device_group(dev); - - if (IS_ERR(group)) { - mutex_unlock(&smmu->stream_map_mutex); - return NULL; - } - } -finish: - if (!IS_ERR(group) && smmu->impl && smmu->impl->device_group && - smmu->impl->device_group(dev, group)) { - iommu_group_put(group); mutex_unlock(&smmu->stream_map_mutex); - return ERR_PTR(-EINVAL); + return iommu_group_ref_get(group); } + if (dev_is_pci(dev)) + group = pci_device_group(dev); + else if (dev_is_fsl_mc(dev)) + group = fsl_mc_device_group(dev); + else + group = generic_device_group(dev); + /* Remember group for faster lookups */ if (!IS_ERR(group)) for_each_cfg_sme(cfg, fwspec, i, idx) @@ -2603,202 +1567,41 @@ static void arm_smmu_get_resv_regions(struct device *dev, list_add_tail(®ion->list, head); iommu_dma_get_resv_regions(dev, head); - - qcom_iommu_generate_resv_regions(dev, head); } static int arm_smmu_def_domain_type(struct device *dev) { struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); - const struct arm_smmu_impl *impl; + const struct arm_smmu_impl *impl = cfg->smmu->impl; - if (!cfg) - return 0; - - impl = cfg->smmu->impl; if (impl && impl->def_domain_type) return impl->def_domain_type(dev); return 0; } -static int __arm_smmu_sid_switch(struct device *dev, void *data) -{ - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); - struct arm_smmu_device *smmu; - enum sid_switch_direction dir = (typeof(dir))data; - int i, idx; - - if (!fwspec || !cfg) - return 0; - - smmu = cfg->smmu; - mutex_lock(&smmu->stream_map_mutex); - for_each_cfg_sme(cfg, fwspec, i, idx) { - smmu->smrs[idx].valid = dir == SID_ACQUIRE; - arm_smmu_write_sme(smmu, idx); - } - mutex_unlock(&smmu->stream_map_mutex); - return 0; -} - -/* - * Some devices support operation with different levels of security. In some - * modes, HLOS is no longer responsible for managing the S1 translations for - * a device. Unfortunately, the device may still use the same set of SIDS, so - * to prevent a potential stream-match conflict fault, HLOS needs to remove - * the SIDs fromits SMRs. Enforcement of this policy is implemented through - * virtualization of the SMR/S2CR regigisters. - */ -static int arm_smmu_sid_switch(struct device *dev, - enum sid_switch_direction dir) -{ - struct iommu_group *group; - int ret; - - group = iommu_group_get(dev); - ret = iommu_group_for_each_dev(group, (void *)dir, - __arm_smmu_sid_switch); - iommu_group_put(group); - - return ret; -} - -static int arm_smmu_get_context_bank_nr(struct iommu_domain *domain) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - int ret; - - mutex_lock(&smmu_domain->init_mutex); - if (!smmu_domain->smmu) - ret = -EINVAL; - else - ret = smmu_domain->cfg.cbndx; - mutex_unlock(&smmu_domain->init_mutex); - return ret; -} - -static int arm_smmu_set_secure_vmid(struct iommu_domain *domain, enum vmid vmid) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - int ret = 0; - - mutex_lock(&smmu_domain->init_mutex); - if (smmu_domain->smmu) - ret = -EPERM; - else if (WARN(smmu_domain->secure_vmid != VMID_INVAL, "secure vmid already set")) - ret = -EPERM; - else - smmu_domain->secure_vmid = vmid; - mutex_unlock(&smmu_domain->init_mutex); - return ret; -} - -static int arm_smmu_set_fault_model(struct iommu_domain *domain, int fault_model) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_fault_model *domain_model = &smmu_domain->fault_model; - int ret = 0; - - mutex_lock(&smmu_domain->init_mutex); - if (smmu_domain->smmu) { - ret = -EPERM; - } else { - domain_model->non_fatal = FIELD_GET(QCOM_IOMMU_FAULT_MODEL_NON_FATAL, fault_model); - domain_model->no_cfre = FIELD_GET(QCOM_IOMMU_FAULT_MODEL_NO_CFRE, fault_model); - domain_model->no_stall = FIELD_GET(QCOM_IOMMU_FAULT_MODEL_NO_STALL, fault_model); - domain_model->hupcf = FIELD_GET(QCOM_IOMMU_FAULT_MODEL_HUPCF, fault_model); - } - mutex_unlock(&smmu_domain->init_mutex); - return ret; -} - -static int arm_smmu_enable_s1_translation(struct iommu_domain *domain) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - struct arm_smmu_device *smmu = smmu_domain->smmu; - int idx; - struct arm_smmu_cb *cb; - int ret; - - mutex_lock(&smmu_domain->init_mutex); - if (!smmu_domain->smmu) { - ret = -EPERM; - goto out; - } else if (!smmu_domain->delayed_s1_trans_enable) { - ret = 0; - goto out; - } - - ret = arm_smmu_rpm_get(smmu); - if (ret < 0) - goto out; - - idx = cfg->cbndx; - cfg->sctlr.m = 1; - cb = &smmu->cbs[idx]; - cb->sctlr = arm_smmu_lpae_sctlr(cfg); - - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, cb->sctlr); - arm_smmu_rpm_put(smmu); - smmu_domain->delayed_s1_trans_enable = false; -out: - mutex_unlock(&smmu_domain->init_mutex); - return ret; -} - -static int arm_smmu_get_mappings_configuration(struct iommu_domain *domain) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - int ret = 0; - - mutex_lock(&smmu_domain->init_mutex); - if (!smmu_domain->smmu) { - ret = -EPERM; - } else { - ret |= smmu_domain->mapping_cfg.s1_bypass ? QCOM_IOMMU_MAPPING_CONF_S1_BYPASS : 0; - ret |= smmu_domain->mapping_cfg.atomic ? QCOM_IOMMU_MAPPING_CONF_ATOMIC : 0; - ret |= smmu_domain->mapping_cfg.fast ? QCOM_IOMMU_MAPPING_CONF_FAST : 0; - } - mutex_unlock(&smmu_domain->init_mutex); - return ret; -} - -static struct qcom_iommu_ops arm_smmu_ops = { - .iova_to_phys_hard = arm_smmu_iova_to_phys_hard, - .sid_switch = arm_smmu_sid_switch, - .get_fault_ids = arm_smmu_get_fault_ids, - .get_context_bank_nr = arm_smmu_get_context_bank_nr, - .set_secure_vmid = arm_smmu_set_secure_vmid, - .set_fault_model = arm_smmu_set_fault_model, - .enable_s1_translation = arm_smmu_enable_s1_translation, - .get_mappings_configuration = arm_smmu_get_mappings_configuration, - - .iommu_ops = { - .capable = arm_smmu_capable, - .domain_alloc = arm_smmu_domain_alloc, - .domain_free = arm_smmu_domain_free, - .attach_dev = arm_smmu_attach_dev, - .map_pages = arm_smmu_map_pages, - .unmap_pages = arm_smmu_unmap_pages, - .flush_iotlb_all = arm_smmu_flush_iotlb_all, - .iotlb_sync = arm_smmu_iotlb_sync, - .iova_to_phys = arm_smmu_iova_to_phys, - .probe_device = arm_smmu_probe_device, - .release_device = arm_smmu_release_device, - .probe_finalize = arm_smmu_probe_finalize, - .device_group = arm_smmu_device_group, - .enable_nesting = arm_smmu_enable_nesting, - .set_pgtable_quirks = arm_smmu_set_pgtable_quirks, - .of_xlate = arm_smmu_of_xlate, - .get_resv_regions = arm_smmu_get_resv_regions, - .put_resv_regions = generic_iommu_put_resv_regions, - .def_domain_type = arm_smmu_def_domain_type, - .pgsize_bitmap = -1UL, /* Restricted during device attach */ - .owner = THIS_MODULE, - } +static struct iommu_ops arm_smmu_ops = { + .capable = arm_smmu_capable, + .domain_alloc = arm_smmu_domain_alloc, + .domain_free = arm_smmu_domain_free, + .attach_dev = arm_smmu_attach_dev, + .map_pages = arm_smmu_map_pages, + .unmap_pages = arm_smmu_unmap_pages, + .flush_iotlb_all = arm_smmu_flush_iotlb_all, + .iotlb_sync = arm_smmu_iotlb_sync, + .iova_to_phys = arm_smmu_iova_to_phys, + .probe_device = arm_smmu_probe_device, + .release_device = arm_smmu_release_device, + .probe_finalize = arm_smmu_probe_finalize, + .device_group = arm_smmu_device_group, + .enable_nesting = arm_smmu_enable_nesting, + .set_pgtable_quirks = arm_smmu_set_pgtable_quirks, + .of_xlate = arm_smmu_of_xlate, + .get_resv_regions = arm_smmu_get_resv_regions, + .put_resv_regions = generic_iommu_put_resv_regions, + .def_domain_type = arm_smmu_def_domain_type, + .pgsize_bitmap = -1UL, /* Restricted during device attach */ + .owner = THIS_MODULE, }; static void arm_smmu_device_reset(struct arm_smmu_device *smmu) @@ -2855,10 +1658,6 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) if (smmu->features & ARM_SMMU_FEAT_EXIDS) reg |= ARM_SMMU_sCR0_EXIDENABLE; - /* Force bypass transaction to be Non-Shareable & not io-coherent */ - reg &= ~ARM_SMMU_sCR0_SHCFG; - reg |= FIELD_PREP(ARM_SMMU_sCR0_SHCFG, ARM_SMMU_sCR0_SHCFG_NSH); - if (smmu->impl && smmu->impl->reset) smmu->impl->reset(smmu); @@ -2886,148 +1685,6 @@ static int arm_smmu_id_size_to_bits(int size) } } -static int arm_smmu_handoff_cbs(struct arm_smmu_device *smmu) -{ - u32 i, smr, s2cr; - u32 index; - struct arm_smmu_smr smrs; - struct arm_smmu_smr *handoff_smrs; - int num_handoff_smrs; - const __be32 *cell; - - cell = of_get_property(smmu->dev->of_node, "qcom,handoff-smrs", NULL); - if (!cell) - return 0; - - num_handoff_smrs = of_property_count_elems_of_size(smmu->dev->of_node, - "qcom,handoff-smrs", sizeof(u32) * 2); - if (num_handoff_smrs < 0) - return 0; - - handoff_smrs = kcalloc(num_handoff_smrs, sizeof(*handoff_smrs), - GFP_KERNEL); - if (!handoff_smrs) - return -ENOMEM; - - for (i = 0; i < num_handoff_smrs; i++) { - handoff_smrs[i].id = of_read_number(cell++, 1); - handoff_smrs[i].mask = of_read_number(cell++, 1); - handoff_smrs[i].valid = true; - } - - - for (i = 0; i < smmu->num_mapping_groups; ++i) { - - smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); - - if (smmu->features & ARM_SMMU_FEAT_EXIDS) { - s2cr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_S2CR(i)); - smrs.valid = FIELD_GET(ARM_SMMU_S2CR_EXIDVALID, s2cr); - if (!smrs.valid) - continue; - - smrs.id = FIELD_GET(ARM_SMMU_SMR_ID, smr); - smrs.mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); - - } else { - smrs.valid = FIELD_GET(ARM_SMMU_SMR_VALID, smr); - if (!smrs.valid) - continue; - - smrs.id = FIELD_GET(ARM_SMMU_SMR_ID, smr); - /* - * The SMR mask covers bits 30:16 when extended stream - * matching is not enabled. - */ - smrs.mask = FIELD_GET(ARM_SMMU_SMR_MASK, - smr & ~ARM_SMMU_SMR_VALID); - } - - for (index = 0; index < num_handoff_smrs; index++) { - - if (!handoff_smrs[index].valid) - continue; - - if ((handoff_smrs[index].mask & smrs.mask) == handoff_smrs[index].mask && - !((handoff_smrs[index].id ^ smrs.id) & ~smrs.mask)) { - - dev_dbg(smmu->dev, - "handoff-smrs match idx %d, id, 0x%x, mask 0x%x\n", - i, smrs.id, smrs.mask); - - s2cr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_S2CR(i)); - - smmu->smrs[i] = smrs; - - smmu->s2crs[i].group = NULL; - smmu->s2crs[i].count = 0; - smmu->s2crs[i].type = FIELD_GET(ARM_SMMU_S2CR_TYPE, s2cr); - smmu->s2crs[i].privcfg = FIELD_GET(ARM_SMMU_S2CR_PRIVCFG, s2cr); - smmu->s2crs[i].cbndx = FIELD_GET(ARM_SMMU_S2CR_CBNDX, s2cr); - - smmu->s2crs[i].pinned = true; - bitmap_set(smmu->context_map, smmu->s2crs[i].cbndx, 1); - handoff_smrs[index].valid = false; - - break; - - } else { - dev_dbg(smmu->dev, - "handoff-smrs no match idx %d, id, 0x%x, mask 0x%x\n", - i, smrs.id, smrs.mask); - } - } - } - - kfree(handoff_smrs); - - return 0; -} - -static int arm_smmu_parse_impl_def_registers(struct arm_smmu_device *smmu) -{ - struct device *dev = smmu->dev; - int i, ntuples, ret; - u32 *tuples; - struct arm_smmu_impl_def_reg *regs, *regit; - - if (!of_find_property(dev->of_node, "attach-impl-defs", &ntuples)) - return 0; - - ntuples /= sizeof(u32); - if (ntuples % 2) { - dev_err(dev, - "Invalid number of attach-impl-defs registers: %d\n", - ntuples); - return -EINVAL; - } - - regs = devm_kmalloc_array(dev, ntuples, sizeof(*regs), GFP_KERNEL); - if (!regs) - return -ENOMEM; - - tuples = kmalloc_array(ntuples * 2, sizeof(*tuples), GFP_KERNEL); - if (!tuples) - return -ENOMEM; - - ret = of_property_read_u32_array(dev->of_node, "attach-impl-defs", - tuples, ntuples); - if (ret) - goto out; - - for (i = 0, regit = regs; i < ntuples; i += 2, ++regit) { - regit->offset = tuples[i]; - regit->value = tuples[i + 1]; - } - - smmu->impl_def_attach_registers = regs; - smmu->num_impl_def_attach_registers = ntuples / 2; - -out: - kfree(tuples); - return ret; -} - static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) { unsigned int size; @@ -3213,10 +1870,10 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) smmu->pgsize_bitmap |= SZ_64K | SZ_512M; - if (arm_smmu_ops.iommu_ops.pgsize_bitmap == -1UL) - arm_smmu_ops.iommu_ops.pgsize_bitmap = smmu->pgsize_bitmap; + if (arm_smmu_ops.pgsize_bitmap == -1UL) + arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; else - arm_smmu_ops.iommu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; + arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", smmu->pgsize_bitmap); @@ -3420,10 +2077,6 @@ static int arm_smmu_device_probe(struct platform_device *pdev) int num_irqs, i, err; irqreturn_t (*global_fault)(int irq, void *dev); - /* We depend on this device for fastmap */ - if (!qcom_dma_iommu_is_ready()) - return -EPROBE_DEFER; - smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) { dev_err(dev, "failed to allocate arm_smmu_device\n"); @@ -3477,40 +2130,32 @@ static int arm_smmu_device_probe(struct platform_device *pdev) for (i = 0; i < num_irqs; ++i) { int irq = platform_get_irq(pdev, i); - if (irq < 0) { - dev_err(dev, "failed to get irq index %d\n", i); + if (irq < 0) return -ENODEV; - } smmu->irqs[i] = irq; } - smmu->pwr = arm_smmu_init_power_resources(dev); - if (IS_ERR(smmu->pwr)) - return PTR_ERR(smmu->pwr); + err = devm_clk_bulk_get_all(dev, &smmu->clks); + if (err < 0) { + dev_err(dev, "failed to get clocks %d\n", err); + return err; + } + smmu->num_clks = err; - /* - * We can't use arm_smmu_rpm_get() because pm-runtime isn't - * enabled yet. - */ - err = arm_smmu_power_on(smmu->pwr); + err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks); if (err) return err; err = arm_smmu_device_cfg_probe(smmu); if (err) - goto out_power_off; - - err = arm_smmu_parse_impl_def_registers(smmu); - if (err) - goto out_power_off; + return err; if (smmu->version == ARM_SMMU_V2) { if (smmu->num_context_banks > smmu->num_context_irqs) { dev_err(dev, "found only %d context irq(s) but %d required\n", smmu->num_context_irqs, smmu->num_context_banks); - err = -ENODEV; - goto out_power_off; + return -ENODEV; } /* Ignore superfluous interrupts */ @@ -3523,35 +2168,26 @@ static int arm_smmu_device_probe(struct platform_device *pdev) global_fault = arm_smmu_global_fault; for (i = 0; i < smmu->num_global_irqs; ++i) { - err = devm_request_threaded_irq(smmu->dev, smmu->irqs[i], - NULL, + err = devm_request_irq(smmu->dev, smmu->irqs[i], global_fault, - IRQF_ONESHOT | IRQF_SHARED, + IRQF_SHARED, "arm-smmu global fault", smmu); if (err) { dev_err(dev, "failed to request global IRQ %d (%u)\n", i, smmu->irqs[i]); - goto out_power_off; + return err; } } - /* QCOM Additions */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - smmu->phys_addr = res->start; - parse_driver_options(smmu); - err = arm_smmu_handoff_cbs(smmu); - if (err) - goto out_power_off; - err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL, "smmu.%pa", &ioaddr); if (err) { dev_err(dev, "Failed to register iommu in sysfs\n"); - goto out_power_off; + return err; } - err = iommu_device_register(&smmu->iommu, &arm_smmu_ops.iommu_ops, dev); + err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); if (err) { dev_err(dev, "Failed to register iommu\n"); goto err_sysfs_remove; @@ -3560,7 +2196,6 @@ static int arm_smmu_device_probe(struct platform_device *pdev) platform_set_drvdata(pdev, smmu); arm_smmu_device_reset(smmu); arm_smmu_test_smr_masks(smmu); - arm_smmu_interrupt_selftest(smmu); /* * We want to avoid touching dev->power.lock in fastpaths unless @@ -3568,11 +2203,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) * can serve as an ideal proxy for that decision. So, conditionally * enable pm_runtime. */ - /* - * QCOM's nonupstream gdsc driver doesn't support pm_domains. - * So check for presence of gdsc instead. - */ - if (smmu->pwr->num_gdscs) { + if (dev->pm_domain) { pm_runtime_set_active(dev); pm_runtime_enable(dev); } @@ -3583,7 +2214,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) * ready to handle default domain setup as soon as any SMMU exists. */ if (!using_legacy_binding) { - err = arm_smmu_bus_init(&arm_smmu_ops.iommu_ops); + err = arm_smmu_bus_init(&arm_smmu_ops); if (err) goto err_unregister_device; } @@ -3594,8 +2225,6 @@ static int arm_smmu_device_probe(struct platform_device *pdev) iommu_device_unregister(&smmu->iommu); err_sysfs_remove: iommu_device_sysfs_remove(&smmu->iommu); -out_power_off: - arm_smmu_power_off(smmu, smmu->pwr); return err; } @@ -3613,9 +2242,6 @@ static int arm_smmu_device_remove(struct platform_device *pdev) iommu_device_unregister(&smmu->iommu); iommu_device_sysfs_remove(&smmu->iommu); - if (smmu->impl && smmu->impl->device_remove) - smmu->impl->device_remove(smmu); - arm_smmu_rpm_get(smmu); /* Turn the thing off */ arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD); @@ -3624,8 +2250,9 @@ static int arm_smmu_device_remove(struct platform_device *pdev) if (pm_runtime_enabled(smmu->dev)) pm_runtime_force_suspend(smmu->dev); else - arm_smmu_power_off(smmu, smmu->pwr); + clk_bulk_disable(smmu->num_clks, smmu->clks); + clk_bulk_unprepare(smmu->num_clks, smmu->clks); return 0; } @@ -3639,10 +2266,12 @@ static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) struct arm_smmu_device *smmu = dev_get_drvdata(dev); int ret; - ret = arm_smmu_power_on(smmu->pwr); + ret = clk_bulk_enable(smmu->num_clks, smmu->clks); if (ret) return ret; + arm_smmu_device_reset(smmu); + return 0; } @@ -3650,7 +2279,7 @@ static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev) { struct arm_smmu_device *smmu = dev_get_drvdata(dev); - arm_smmu_power_off(smmu, smmu->pwr); + clk_bulk_disable(smmu->num_clks, smmu->clks); return 0; } @@ -3668,17 +2297,9 @@ static int __maybe_unused arm_smmu_pm_resume(struct device *dev) return 0; ret = arm_smmu_runtime_resume(dev); - if (ret) { + if (ret) clk_bulk_unprepare(smmu->num_clks, smmu->clks); - return ret; - } - /* - * QCOM HW supports register retention. So we really only need to - * re-program the registers for hibernation. Don't do this during - * runtime_resume to avoid latency. - */ - arm_smmu_device_reset(smmu); return ret; } @@ -3716,34 +2337,7 @@ static struct platform_driver arm_smmu_driver = { .remove = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; - -static int __init arm_smmu_init(void) -{ - int ret; - ktime_t cur; - - cur = ktime_get(); - ret = platform_driver_register(&qsmmuv500_tbu_driver); - if (ret) - return ret; - - ret = platform_driver_register(&arm_smmu_driver); - if (ret) { - platform_driver_unregister(&qsmmuv500_tbu_driver); - return ret; - } - - trace_smmu_init(ktime_us_delta(ktime_get(), cur)); - return ret; -} -subsys_initcall(arm_smmu_init); - -static void __exit arm_smmu_exit(void) -{ - platform_driver_unregister(&arm_smmu_driver); - platform_driver_unregister(&qsmmuv500_tbu_driver); -} -module_exit(arm_smmu_exit); +module_platform_driver(arm_smmu_driver); MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon "); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h index f75f380d5ec8..0d210edb28b5 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h @@ -5,8 +5,6 @@ * Copyright (C) 2013 ARM Limited * * Author: Will Deacon - * - * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _ARM_SMMU_H @@ -24,15 +22,10 @@ #include #include #include -#include - -#include "../../qcom-io-pgtable.h" /* Configuration registers */ #define ARM_SMMU_GR0_sCR0 0x0 #define ARM_SMMU_sCR0_VMID16EN BIT(31) -#define ARM_SMMU_sCR0_SHCFG GENMASK(23, 22) -#define ARM_SMMU_sCR0_SHCFG_NSH 0x3 #define ARM_SMMU_sCR0_BSU GENMASK(15, 14) #define ARM_SMMU_sCR0_FB BIT(13) #define ARM_SMMU_sCR0_PTM BIT(12) @@ -124,8 +117,6 @@ enum arm_smmu_s2cr_type { S2CR_TYPE_FAULT, }; #define ARM_SMMU_S2CR_EXIDVALID BIT(10) -#define ARM_SMMU_S2CR_SHCFG GENMASK(9, 8) -#define ARM_SMMU_S2CR_SHCFG_NSH 0x3 #define ARM_SMMU_S2CR_CBNDX GENMASK(7, 0) /* Context bank attribute registers */ @@ -145,23 +136,12 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CBAR_VMID GENMASK(7, 0) #define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2)) -#define CBFRSYNRA_SID_MASK (0xffff) #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) #define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16) #define ARM_SMMU_CBA2R_VA64 BIT(0) #define ARM_SMMU_CB_SCTLR 0x0 -#define ARM_SMMU_SCTLR_WACFG GENMASK(27, 26) -#define ARM_SMMU_SCTLR_WACFG_WA 0x2 -#define ARM_SMMU_SCTLR_RACFG GENMASK(25, 24) -#define ARM_SMMU_SCTLR_RACFG_RA 0x2 -#define ARM_SMMU_SCTLR_SHCFG GENMASK(23, 22) -#define ARM_SMMU_SCTLR_SHCFG_OSH 0x1 -#define ARM_SMMU_SCTLR_SHCFG_NSH 0x3 -#define ARM_SMMU_SCTLR_MTCFG BIT(20) -#define ARM_SMMU_SCTLR_MEM_ATTR GENMASK(19, 16) -#define ARM_SMMU_SCTLR_MEM_ATTR_OISH_WB_CACHE 0xf #define ARM_SMMU_SCTLR_S1_ASIDPNE BIT(12) #define ARM_SMMU_SCTLR_CFCFG BIT(7) #define ARM_SMMU_SCTLR_HUPCF BIT(8) @@ -176,7 +156,6 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CB_RESUME 0x8 #define ARM_SMMU_RESUME_TERMINATE BIT(0) -#define ARM_SMMU_RESUME_RESUME 0 #define ARM_SMMU_CB_TCR2 0x10 #define ARM_SMMU_TCR2_SEP GENMASK(17, 15) @@ -240,19 +219,10 @@ enum arm_smmu_cbar_type { ARM_SMMU_FSR_TF | \ ARM_SMMU_FSR_IGN) -#define ARM_SMMU_CB_FSRRESTORE 0x5c #define ARM_SMMU_CB_FAR 0x60 #define ARM_SMMU_CB_FSYNR0 0x68 #define ARM_SMMU_FSYNR0_WNR BIT(4) -#define ARM_SMMU_FSYNR0_PNU BIT(5) -#define ARM_SMMU_FSYNR0_IND BIT(6) -#define ARM_SMMU_FSYNR0_NSATTR BIT(8) - -#define ARM_SMMU_CB_FSYNR1 0x6c -#define ARM_SMMU_FSYNR1_BID GENMASK(15, 13) -#define ARM_SMMU_FSYNR1_PID GENMASK(12, 8) -#define ARM_SMMU_FSYNR1_MID GENMASK(7, 0) #define ARM_SMMU_CB_FSYNR1 0x6c @@ -265,24 +235,6 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CB_TLBSTATUS 0x7f4 #define ARM_SMMU_CB_ATS1PR 0x800 -/* Implementation Defined Register Space 5 registers*/ -/* Relative to IMPL_DEF5 page */ -#define ARM_SMMU_STATS_SYNC_INV_TBU_ACK 0x5dc -#define TBU_SYNC_ACK GENMASK(31, 17) -#define TBU_SYNC_REQ BIT(16) -#define TBU_INV_ACK GENMASK(15, 1) -#define TBU_INV_REQ BIT(0) -#define APPS_SMMU_TBU_REG_ACCESS_REQ_NS 0x5f8 -#define APPS_SMMU_TBU_REG_ACCESS_ACK_NS 0x5fc - -/* Relative to SMMU_BASE */ -#define ARM_SMMU_TBU_PWR_STATUS 0x2204 - -/* Relative SMMU_BASE */ -#define ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR 0x2670 -#define TCU_SYNC_IN_PRGSS BIT(20) -#define TCU_INV_IN_PRGSS BIT(16) - #define ARM_SMMU_CB_ATSR 0x8f0 #define ARM_SMMU_ATSR_ACTIVE BIT(0) @@ -290,9 +242,8 @@ enum arm_smmu_cbar_type { /* Maximum number of context banks per SMMU */ #define ARM_SMMU_MAX_CBS 128 -#define TLB_LOOP_TIMEOUT 500000 /* 500ms */ +#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */ #define TLB_SPIN_COUNT 10 -#define TLB_LOOP_INC_MAX 1000 /*1ms*/ /* Shared driver definitions */ enum arm_smmu_arch_version { @@ -309,42 +260,12 @@ enum arm_smmu_implementation { QCOM_SMMUV500, }; -struct arm_smmu_impl_def_reg { - u32 offset; - u32 value; -}; - -/* - * Describes resources required for on/off power operation. - * Separate reference count is provided for atomic/nonatomic - * operations. - */ -struct arm_smmu_power_resources { - struct device *dev; - - struct clk **clocks; - int num_clocks; - - struct regulator_bulk_data *gdscs; - int num_gdscs; - - struct icc_path *icc_path; - - /* Protects power_count */ - struct mutex power_lock; - int power_count; - - int (*resume)(struct arm_smmu_power_resources *pwr); - void (*suspend)(struct arm_smmu_power_resources *pwr); -}; - struct arm_smmu_s2cr { struct iommu_group *group; int count; enum arm_smmu_s2cr_type type; enum arm_smmu_s2cr_privcfg privcfg; u8 cbndx; - bool pinned; }; struct arm_smmu_smr { @@ -376,12 +297,6 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_EXIDS (1 << 12) u32 features; -#define ARM_SMMU_OPT_FATAL_ASF (1 << 0) -#define ARM_SMMU_OPT_3LVL_TABLES (1 << 2) -#define ARM_SMMU_OPT_NO_ASID_RETENTION (1 << 3) -#define ARM_SMMU_OPT_DISABLE_ATOS (1 << 4) -#define ARM_SMMU_OPT_CONTEXT_FAULT_RETRY (1 << 5) - u32 options; enum arm_smmu_arch_version version; enum arm_smmu_implementation model; const struct arm_smmu_impl *impl; @@ -414,17 +329,6 @@ struct arm_smmu_device { /* IOMMU core code handle */ struct iommu_device iommu; - - /* Specific to QCOM */ - struct arm_smmu_impl_def_reg *impl_def_attach_registers; - unsigned int num_impl_def_attach_registers; - - struct arm_smmu_power_resources *pwr; - - /* used for qsmmuv500 scm_io_readl */ - phys_addr_t phys_addr; - - unsigned long sync_timed_out; }; enum arm_smmu_context_fmt { @@ -441,19 +345,6 @@ struct arm_smmu_cfg { u16 asid; u16 vmid; }; - u32 procid; - struct { - u32 wacfg:2; - u32 racfg:2; - u32 shcfg:2; - u32 mtcfg:1; - u32 memattr:4; - u32 hupcf:1; - u32 cfcfg:1; - u32 cfre:1; - u32 m:1; - } sctlr; - enum arm_smmu_cbar_type cbar; enum arm_smmu_context_fmt fmt; bool flush_walk_prefer_tlbiasid; @@ -464,7 +355,6 @@ struct arm_smmu_cb { u64 ttbr[2]; u32 tcr[2]; u32 mair[2]; - u32 sctlr; struct arm_smmu_cfg *cfg; }; @@ -475,53 +365,16 @@ enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_BYPASS, }; -struct arm_smmu_fault_model { - char non_fatal : 1; - char no_cfre : 1; - char no_stall : 1; - char hupcf : 1; -}; - -struct arm_smmu_mapping_cfg { - char s1_bypass : 1; - char atomic : 1; - char fast : 1; -}; - struct arm_smmu_domain { struct arm_smmu_device *smmu; - struct device *dev; struct io_pgtable_ops *pgtbl_ops; unsigned long pgtbl_quirks; - bool force_coherent_walk; const struct iommu_flush_ops *flush_ops; struct arm_smmu_cfg cfg; enum arm_smmu_domain_stage stage; struct mutex init_mutex; /* Protects smmu pointer */ - spinlock_t cb_lock; /* Serialises ATS1* ops */ - spinlock_t sync_lock; /* Serialises TLB syncs */ - struct arm_smmu_fault_model fault_model; - struct arm_smmu_mapping_cfg mapping_cfg; - bool delayed_s1_trans_enable; - u32 secure_vmid; - - /* - * Track PMDs which require tlb invalidate prior to being - * freed, or before their iovas can be reused by iommu_map(). - */ - spinlock_t iotlb_gather_lock; - struct page *freelist; - bool deferred_sync; - - struct iommu_debug_attachment *logger; + spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */ struct iommu_domain domain; - /* mapping_cfg.atomic indicates that runtime power management should be disabled. */ - bool rpm_always_on; - -#ifdef CONFIG_ARM_SMMU_CONTEXT_FAULT_RETRY - u64 prev_fault_address; - u32 fault_retry_counter; -#endif }; struct arm_smmu_master_cfg { @@ -568,28 +421,7 @@ static inline u32 arm_smmu_lpae_vtcr(const struct io_pgtable_cfg *cfg) FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz); } -static inline u32 arm_smmu_lpae_sctlr(struct arm_smmu_cfg *cfg) -{ - bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; - - return FIELD_PREP(ARM_SMMU_SCTLR_WACFG, cfg->sctlr.wacfg) | - FIELD_PREP(ARM_SMMU_SCTLR_RACFG, cfg->sctlr.racfg) | - FIELD_PREP(ARM_SMMU_SCTLR_SHCFG, cfg->sctlr.shcfg) | - FIELD_PREP(ARM_SMMU_SCTLR_MTCFG, cfg->sctlr.mtcfg) | - FIELD_PREP(ARM_SMMU_SCTLR_MEM_ATTR, cfg->sctlr.memattr) | - FIELD_PREP(ARM_SMMU_SCTLR_S1_ASIDPNE, stage1) | - FIELD_PREP(ARM_SMMU_SCTLR_HUPCF, cfg->sctlr.hupcf) | - FIELD_PREP(ARM_SMMU_SCTLR_CFCFG, cfg->sctlr.cfcfg) | - ARM_SMMU_SCTLR_CFIE | - FIELD_PREP(ARM_SMMU_SCTLR_CFRE, cfg->sctlr.cfre) | - FIELD_PREP(ARM_SMMU_SCTLR_E, IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) | - ARM_SMMU_SCTLR_AFE | - ARM_SMMU_SCTLR_TRE | - FIELD_PREP(ARM_SMMU_SCTLR_M, cfg->sctlr.m); -} - /* Implementation details, yay! */ - struct arm_smmu_impl { u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset); void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset, @@ -601,13 +433,6 @@ struct arm_smmu_impl { int (*reset)(struct arm_smmu_device *smmu); int (*init_context)(struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg *cfg, struct device *dev); - void (*init_context_bank)(struct arm_smmu_domain *smmu_domain, - struct device *dev); - phys_addr_t (*iova_to_phys_hard)(struct arm_smmu_domain *smmu_domain, - struct qcom_iommu_atos_txn *txn); - void (*tlb_sync_timeout)(struct arm_smmu_device *smmu); - void (*device_remove)(struct arm_smmu_device *smmu); - int (*device_group)(struct device *dev, struct iommu_group *group); void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync, int status); int (*def_domain_type)(struct device *dev); @@ -679,13 +504,6 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, #define ARM_SMMU_GR0 0 #define ARM_SMMU_GR1 1 - -/* - * Implementation defined space starts after SMMU GR space, so IMPL_DEF page n - * is page n + 2 in the SMMU register space. - */ -#define ARM_SMMU_IMPL_DEF5 7 - #define ARM_SMMU_CB(s, n) ((s)->numpage + (n)) #define arm_smmu_gr0_read(s, o) \ @@ -710,21 +528,8 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); -struct arm_smmu_device *qsmmuv500_impl_init(struct arm_smmu_device *smmu); -struct arm_smmu_device *qcom_adreno_smmu_impl_init(struct arm_smmu_device *smmu); void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx); int arm_mmu500_reset(struct arm_smmu_device *smmu); -int arm_smmu_power_on(struct arm_smmu_power_resources *pwr); -void arm_smmu_power_off(struct arm_smmu_device *smmu, - struct arm_smmu_power_resources *pwr); -struct arm_smmu_power_resources *arm_smmu_init_power_resources( - struct device *dev); - -extern struct platform_driver qsmmuv500_tbu_driver; - -/* Misc. constants */ -#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) - #endif /* _ARM_SMMU_H */