From 283a6d9eee231a6f44fb1a9a2288b44a76cc1031 Mon Sep 17 00:00:00 2001 From: Chris Goldsworthy Date: Fri, 29 Apr 2022 15:39:54 -0700 Subject: [PATCH 1/3] arm64: defconfig: Compile in IOMMU util module Compile in our IOMMU helper module. Change-Id: Id6a38e8eb9cd7d7ba673352323f38199f15b62d2 Signed-off-by: Chris Goldsworthy --- arch/arm64/configs/vendor/pineapple_GKI.config | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/configs/vendor/pineapple_GKI.config b/arch/arm64/configs/vendor/pineapple_GKI.config index 9ce7f099e602..7a74b96cb3b1 100644 --- a/arch/arm64/configs/vendor/pineapple_GKI.config +++ b/arch/arm64/configs/vendor/pineapple_GKI.config @@ -11,6 +11,7 @@ CONFIG_PHY_QCOM_UFS_QRBTC_SDM845=m CONFIG_PINCTRL_MSM=m CONFIG_PINCTRL_PINEAPPLE=m CONFIG_QCOM_COMMAND_DB=m +CONFIG_QCOM_IOMMU_UTIL=m CONFIG_QCOM_PDC=m CONFIG_QCOM_RPMH=m CONFIG_QCOM_SCM=m From 95096b4df0130b3c6a27f7dad2b0f7c7f1ca10bf Mon Sep 17 00:00:00 2001 From: Chris Goldsworthy Date: Thu, 5 May 2022 16:47:07 -0700 Subject: [PATCH 2/3] modules.list.msm.pineapple: Add qcom_iommu_util.ko Add in qcom_iommu_util.ko as arm_smmu.ko depends on it. Change-Id: I05c29816f0baff03fda8e5cb82f3e4876b9a1f1e Signed-off-by: Chris Goldsworthy --- modules.list.msm.pineapple | 1 + 1 file changed, 1 insertion(+) diff --git a/modules.list.msm.pineapple b/modules.list.msm.pineapple index 0531bcad07d8..6c57172d64a4 100644 --- a/modules.list.msm.pineapple +++ b/modules.list.msm.pineapple @@ -1,3 +1,4 @@ +qcom_iommu_util.ko arm_smmu.ko qcom-scm.ko pinctrl-msm.ko From 360e0c34e76a7a9eea1d8ef42431f41de78db6a2 Mon Sep 17 00:00:00 2001 From: Chris Goldsworthy Date: Thu, 28 Apr 2022 11:31:10 -0700 Subject: [PATCH 3/3] arm-smmu: Port our downstream SMMU changes from 5.15 Take a snapshot of our SMMU driver as of commit f843a80d2c45 ("Merge "defconfig: gen3auto: Separated if_changed"") on msm-5.15. Change the devm_request_threaded_irq() and devm_free_irq() calls in arm_smmu_interrupt_selftest() to request_threaded_irq() and free_irq() respectively, since we release the resources within the same function that is also only being called during probe. Change-Id: Ib3fea5b0cfb4e78eebacecfd7cb24e17000fda5e Signed-off-by: Chris Goldsworthy --- drivers/iommu/arm/arm-smmu/Makefile | 2 +- drivers/iommu/arm/arm-smmu/arm-smmu-impl.c | 3 + drivers/iommu/arm/arm-smmu/arm-smmu-qcom-pm.c | 336 ++++ drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 1381 ++++++++++++- drivers/iommu/arm/arm-smmu/arm-smmu-trace.h | 224 +++ drivers/iommu/arm/arm-smmu/arm-smmu.c | 1722 +++++++++++++++-- drivers/iommu/arm/arm-smmu/arm-smmu.h | 199 +- 7 files changed, 3699 insertions(+), 168 deletions(-) create mode 100644 drivers/iommu/arm/arm-smmu/arm-smmu-qcom-pm.c create mode 100644 drivers/iommu/arm/arm-smmu/arm-smmu-trace.h diff --git a/drivers/iommu/arm/arm-smmu/Makefile b/drivers/iommu/arm/arm-smmu/Makefile index b0cc01aa20c9..1339ab877546 100644 --- a/drivers/iommu/arm/arm-smmu/Makefile +++ b/drivers/iommu/arm/arm-smmu/Makefile @@ -2,4 +2,4 @@ obj-$(CONFIG_QCOM_IOMMU) += qcom_iommu.o obj-$(CONFIG_ARM_SMMU) += arm_smmu.o arm_smmu-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-nvidia.o -arm_smmu-$(CONFIG_ARM_SMMU_QCOM) += arm-smmu-qcom.o +arm_smmu-$(CONFIG_ARM_SMMU_QCOM) += arm-smmu-qcom.o arm-smmu-qcom-pm.o diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c index 2c25cce38060..9790cb3e160e 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c @@ -215,6 +215,9 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) of_device_is_compatible(np, "nvidia,tegra186-smmu")) return nvidia_smmu_impl_init(smmu); + if (of_device_is_compatible(smmu->dev->of_node, "qcom,qsmmu-v500")) + return qsmmuv500_impl_init(smmu); + if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM)) smmu = qcom_smmu_impl_init(smmu); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-pm.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-pm.c new file mode 100644 index 000000000000..b2218da12453 --- /dev/null +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom-pm.c @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ +#include +#include +#include +#include +#include +#include "arm-smmu.h" + +#define ARM_SMMU_ICC_AVG_BW 0 +#define ARM_SMMU_ICC_PEAK_BW_HIGH 1000 +#define ARM_SMMU_ICC_PEAK_BW_LOW 0 +#define ARM_SMMU_ICC_ACTIVE_ONLY_TAG 0x3 + +/* + * Theoretically, our interconnect does not guarantee the order between + * writes to different "register blocks" even with device memory type. + * It does guarantee that the completion of a read to a particular + * register block implies that previously issued writes to that + * register block have completed, with device memory type. + * + * In particular, we need to ensure that writes to iommu registers + * complete before we turn off the power. + */ +static void arm_smmu_arch_write_sync(struct arm_smmu_device *smmu) +{ + u32 id; + + if (!smmu) + return; + + /* Read to complete prior write transcations */ + id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0); + + /* Wait for read to complete before off */ + rmb(); +} + +static int arm_smmu_prepare_clocks(struct arm_smmu_power_resources *pwr) +{ + + int i, ret = 0; + + for (i = 0; i < pwr->num_clocks; ++i) { + ret = clk_prepare(pwr->clocks[i]); + if (ret) { + dev_err(pwr->dev, "Couldn't prepare clock #%d\n", i); + while (i--) + clk_unprepare(pwr->clocks[i]); + break; + } + } + return ret; +} + +static void arm_smmu_unprepare_clocks(struct arm_smmu_power_resources *pwr) +{ + int i; + + for (i = pwr->num_clocks; i; --i) + clk_unprepare(pwr->clocks[i - 1]); +} + +static int arm_smmu_enable_clocks(struct arm_smmu_power_resources *pwr) +{ + int i, ret = 0; + + for (i = 0; i < pwr->num_clocks; ++i) { + ret = clk_enable(pwr->clocks[i]); + if (ret) { + dev_err(pwr->dev, "Couldn't enable clock #%d\n", i); + while (i--) + clk_disable(pwr->clocks[i]); + break; + } + } + + return ret; +} + +static void arm_smmu_disable_clocks(struct arm_smmu_power_resources *pwr) +{ + int i; + + for (i = pwr->num_clocks; i; --i) + clk_disable(pwr->clocks[i - 1]); +} + +static int arm_smmu_raise_interconnect_bw(struct arm_smmu_power_resources *pwr) +{ + if (!pwr->icc_path) + return 0; + return icc_set_bw(pwr->icc_path, ARM_SMMU_ICC_AVG_BW, + ARM_SMMU_ICC_PEAK_BW_HIGH); +} + +static void arm_smmu_lower_interconnect_bw(struct arm_smmu_power_resources *pwr) +{ + if (!pwr->icc_path) + return; + WARN_ON(icc_set_bw(pwr->icc_path, ARM_SMMU_ICC_AVG_BW, + ARM_SMMU_ICC_PEAK_BW_LOW)); +} + +static int arm_smmu_enable_regulators(struct arm_smmu_power_resources *pwr) +{ + struct regulator_bulk_data *consumers; + int num_consumers, ret; + int i; + + num_consumers = pwr->num_gdscs; + consumers = pwr->gdscs; + for (i = 0; i < num_consumers; i++) { + ret = regulator_enable(consumers[i].consumer); + if (ret) + goto out; + } + return 0; + +out: + i -= 1; + for (; i >= 0; i--) + regulator_disable(consumers[i].consumer); + return ret; +} + +int arm_smmu_power_on(struct arm_smmu_power_resources *pwr) +{ + int ret; + + mutex_lock(&pwr->power_lock); + if (pwr->power_count > 0) { + pwr->power_count += 1; + mutex_unlock(&pwr->power_lock); + return 0; + } + + ret = arm_smmu_raise_interconnect_bw(pwr); + if (ret) + goto out_unlock; + + ret = arm_smmu_enable_regulators(pwr); + if (ret) + goto out_disable_bus; + + ret = arm_smmu_prepare_clocks(pwr); + if (ret) + goto out_disable_regulators; + + ret = arm_smmu_enable_clocks(pwr); + if (ret) + goto out_unprepare_clocks; + + if (pwr->resume) { + ret = pwr->resume(pwr); + if (ret) + goto out_disable_clocks; + } + + pwr->power_count = 1; + mutex_unlock(&pwr->power_lock); + return 0; +out_disable_clocks: + arm_smmu_disable_clocks(pwr); +out_unprepare_clocks: + arm_smmu_unprepare_clocks(pwr); +out_disable_regulators: + regulator_bulk_disable(pwr->num_gdscs, pwr->gdscs); +out_disable_bus: + arm_smmu_lower_interconnect_bw(pwr); +out_unlock: + mutex_unlock(&pwr->power_lock); + return ret; +} + +/* + * Needing to pass smmu to this api for arm_smmu_arch_write_sync is awkward. + */ +void arm_smmu_power_off(struct arm_smmu_device *smmu, + struct arm_smmu_power_resources *pwr) +{ + mutex_lock(&pwr->power_lock); + if (pwr->power_count == 0) { + WARN(1, "%s: Bad power count\n", dev_name(pwr->dev)); + mutex_unlock(&pwr->power_lock); + return; + + } else if (pwr->power_count > 1) { + pwr->power_count--; + mutex_unlock(&pwr->power_lock); + return; + } + + if (pwr->suspend) + pwr->suspend(pwr); + + arm_smmu_arch_write_sync(smmu); + arm_smmu_disable_clocks(pwr); + arm_smmu_unprepare_clocks(pwr); + regulator_bulk_disable(pwr->num_gdscs, pwr->gdscs); + arm_smmu_lower_interconnect_bw(pwr); + pwr->power_count = 0; + mutex_unlock(&pwr->power_lock); +} + +static int arm_smmu_init_clocks(struct arm_smmu_power_resources *pwr) +{ + const char *cname; + struct property *prop; + int i; + struct device *dev = pwr->dev; + + pwr->num_clocks = + of_property_count_strings(dev->of_node, "clock-names"); + + if (pwr->num_clocks < 1) { + pwr->num_clocks = 0; + return 0; + } + + pwr->clocks = devm_kzalloc( + dev, sizeof(*pwr->clocks) * pwr->num_clocks, + GFP_KERNEL); + + if (!pwr->clocks) + return -ENOMEM; + + i = 0; + of_property_for_each_string(dev->of_node, "clock-names", + prop, cname) { + struct clk *c = devm_clk_get(dev, cname); + + if (IS_ERR(c)) { + dev_err(dev, "Couldn't get clock: %s\n", + cname); + return PTR_ERR(c); + } + + if (clk_get_rate(c) == 0) { + long rate = clk_round_rate(c, 1000); + + clk_set_rate(c, rate); + } + + pwr->clocks[i] = c; + + ++i; + } + return 0; +} + +static int arm_smmu_init_regulators(struct arm_smmu_power_resources *pwr) +{ + const char *cname; + struct property *prop; + int i; + struct device *dev = pwr->dev; + + pwr->num_gdscs = + of_property_count_strings(dev->of_node, "qcom,regulator-names"); + + if (pwr->num_gdscs < 1) { + pwr->num_gdscs = 0; + return 0; + } + + pwr->gdscs = devm_kzalloc( + dev, sizeof(*pwr->gdscs) * pwr->num_gdscs, GFP_KERNEL); + + if (!pwr->gdscs) + return -ENOMEM; + + i = 0; + of_property_for_each_string(dev->of_node, "qcom,regulator-names", + prop, cname) + pwr->gdscs[i++].supply = cname; + + return devm_regulator_bulk_get(dev, pwr->num_gdscs, pwr->gdscs); +} + +static int arm_smmu_init_interconnect(struct arm_smmu_power_resources *pwr) +{ + struct device *dev = pwr->dev; + + /* We don't want the interconnect APIs to print an error message */ + if (!of_find_property(dev->of_node, "interconnects", NULL)) { + dev_dbg(dev, "No interconnect info\n"); + return 0; + } + + pwr->icc_path = devm_of_icc_get(dev, NULL); + if (IS_ERR_OR_NULL(pwr->icc_path)) { + if (PTR_ERR(pwr->icc_path) != -EPROBE_DEFER) + dev_err(dev, "Unable to read interconnect path from devicetree rc: %ld\n", + PTR_ERR(pwr->icc_path)); + return pwr->icc_path ? PTR_ERR(pwr->icc_path) : -EINVAL; + } + + if (of_property_read_bool(dev->of_node, "qcom,active-only")) + icc_set_tag(pwr->icc_path, ARM_SMMU_ICC_ACTIVE_ONLY_TAG); + + return 0; +} + +/* + * Cleanup done by devm. Any non-devm resources must clean up themselves. + */ +struct arm_smmu_power_resources *arm_smmu_init_power_resources( + struct device *dev) +{ + struct arm_smmu_power_resources *pwr; + int ret; + + pwr = devm_kzalloc(dev, sizeof(*pwr), GFP_KERNEL); + if (!pwr) + return ERR_PTR(-ENOMEM); + + pwr->dev = dev; + mutex_init(&pwr->power_lock); + + ret = arm_smmu_init_clocks(pwr); + if (ret) + return ERR_PTR(ret); + + ret = arm_smmu_init_regulators(pwr); + if (ret) + return ERR_PTR(ret); + + ret = arm_smmu_init_interconnect(pwr); + if (ret) + return ERR_PTR(ret); + + return pwr; +} diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index ba6298c7140e..3086b552c638 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -1,12 +1,19 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */ #include #include +#include +#include +#include +#include +#include #include #include +#include +#include #include "arm-smmu.h" @@ -81,6 +88,7 @@ static void qcom_adreno_smmu_resume_translation(const void *cookie, bool termina } #define QCOM_ADRENO_SMMU_GPU_SID 0 +#define QCOM_ADRENO_SMMU_GPU_LPAC_SID 1 static bool qcom_adreno_smmu_is_gpu_device(struct device *dev) { @@ -94,7 +102,8 @@ static bool qcom_adreno_smmu_is_gpu_device(struct device *dev) for (i = 0; i < fwspec->num_ids; i++) { u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); - if (sid == QCOM_ADRENO_SMMU_GPU_SID) + if (sid == QCOM_ADRENO_SMMU_GPU_SID || + sid == QCOM_ADRENO_SMMU_GPU_LPAC_SID) return true; } @@ -164,14 +173,14 @@ static int qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_doma int count; /* - * Assign context bank 0 to the GPU device so the GPU hardware can + * Assign context bank 0 and 1 to the GPU device so the GPU hardware can * switch pagetables */ if (qcom_adreno_smmu_is_gpu_device(dev)) { start = 0; - count = 1; + count = 2; } else { - start = 1; + start = 2; count = smmu->num_context_banks; } @@ -188,10 +197,12 @@ static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu) return true; } +static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[]; static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg *pgtbl_cfg, struct device *dev) { struct adreno_smmu_priv *priv; + const struct device_node *np = smmu_domain->smmu->dev->of_node; smmu_domain->cfg.flush_walk_prefer_tlbiasid = true; @@ -218,8 +229,19 @@ static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain, priv->get_ttbr1_cfg = qcom_adreno_smmu_get_ttbr1_cfg; priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg; priv->get_fault_info = qcom_adreno_smmu_get_fault_info; - priv->set_stall = qcom_adreno_smmu_set_stall; - priv->resume_translation = qcom_adreno_smmu_resume_translation; + + /* + * These functions are only compatible with the data structures used by the + * QCOM SMMU implementation hooks, and are thus not appropriate to set for other + * implementations (e.g. QSMMUV500). + * + * Providing these functions as part of the GPU interface also makes little sense + * as context banks are set to stall by default anyway. + */ + if (of_match_node(qcom_smmu_impl_of_match, np)) { + priv->set_stall = qcom_adreno_smmu_set_stall; + priv->resume_translation = qcom_adreno_smmu_resume_translation; + } return 0; } @@ -376,6 +398,1351 @@ static const struct arm_smmu_impl qcom_smmu_impl = { .write_s2cr = qcom_smmu_write_s2cr, }; +#define TBUID_SHIFT 10 + +#define DEBUG_SID_HALT_REG 0x0 +#define DEBUG_SID_HALT_REQ BIT(16) +#define DEBUG_SID_HALT_SID GENMASK(9, 0) + +#define DEBUG_VA_ADDR_REG 0x8 + +#define DEBUG_TXN_TRIGG_REG 0x18 +#define DEBUG_TXN_AXPROT GENMASK(8, 6) +#define DEBUG_TXN_AXCACHE GENMASK(5, 2) +#define DEBUG_TXN_WRITE BIT(1) +#define DEBUG_TXN_AXPROT_PRIV 0x1 +#define DEBUG_TXN_AXPROT_UNPRIV 0x0 +#define DEBUG_TXN_AXPROT_NSEC 0x2 +#define DEBUG_TXN_AXPROT_SEC 0x0 +#define DEBUG_TXN_AXPROT_INST 0x4 +#define DEBUG_TXN_AXPROT_DATA 0x0 +#define DEBUG_TXN_READ (0x0 << 1) +#define DEBUG_TXN_TRIGGER BIT(0) + +#define DEBUG_SR_HALT_ACK_REG 0x20 +#define DEBUG_SR_HALT_ACK_VAL (0x1 << 1) +#define DEBUG_SR_ECATS_RUNNING_VAL (0x1 << 0) + +#define DEBUG_PAR_REG 0x28 +#define DEBUG_PAR_PA GENMASK_ULL(47, 12) +#define DEBUG_PAR_FAULT_VAL BIT(0) + +#define DEBUG_AXUSER_REG 0x30 +#define DEBUG_AXUSER_CDMID GENMASK_ULL(43, 36) +#define DEBUG_AXUSER_CDMID_VAL 255 + +#define TBU_DBG_TIMEOUT_US 100 +#define TBU_MICRO_IDLE_DELAY_US 5 + +#define TNX_TCR_CNTL 0x130 +#define TNX_TCR_CNTL_TBU_OT_CAPTURE_EN BIT(18) +#define TNX_TCR_CNTL_ALWAYS_CAPTURE BIT(15) +#define TNX_TCR_CNTL_MATCH_MASK_UPD BIT(7) +#define TNX_TCR_CNTL_MATCH_MASK_VALID BIT(6) + +#define CAPTURE1_SNAPSHOT_1 0x138 + +#define TNX_TCR_CNTL_2 0x178 +#define TNX_TCR_CNTL_2_CAP1_VALID BIT(0) + +/* QTB constants */ +#define QTB_DBG_TIMEOUT_US 100 + +#define QTB_SWID_LOW 0x0 + +#define QTB_OVR_DBG_FENCEREQ 0x410 +#define QTB_OVR_DBG_FENCEREQ_HALT BIT(0) + +#define QTB_OVR_DBG_FENCEACK 0x418 +#define QTB_OVR_DBG_FENCEACK_ACK BIT(0) + +#define QTB_OVR_ECATS_INFLD0 0x430 +#define QTB_OVR_ECATS_INFLD0_PCIE_NO_SNOOP BIT(21) +#define QTB_OVR_ECATS_INFLD0_SEC_SID BIT(20) +#define QTB_OVR_ECATS_INFLD0_QAD GENMASK(19, 16) +#define QTB_OVR_ECATS_INFLD0_SID GENMASK(9, 0) + +#define QTB_OVR_ECATS_INFLD1 0x438 +#define QTB_OVR_ECATS_INFLD1_PNU BIT(13) +#define QTB_OVR_ECATS_INFLD1_IND BIT(12) +#define QTB_OVR_ECATS_INFLD1_DIRTY BIT(11) +#define QTB_OVR_ECATS_INFLD1_TR_TYPE GENMASK(10, 8) +#define QTB_OVR_ECATS_INFLD1_TR_TYPE_SHARED 4 +#define QTB_OVR_ECATS_INFLD1_ALLOC GENMASK(7, 4) +#define QTB_OVR_ECATS_INFLD1_NON_SEC BIT(3) +#define QTB_OVR_ECATS_INFLD1_OPC GENMASK(2, 0) +#define QTB_OVR_ECATS_INFLD1_OPC_WRI 1 + +#define QTB_OVR_ECATS_INFLD2 0x440 + +#define QTB_OVR_ECATS_TRIGGER 0x448 +#define QTB_OVR_ECATS_TRIGGER_START BIT(0) + +#define QTB_OVR_ECATS_STATUS 0x450 +#define QTB_OVR_ECATS_STATUS_DONE BIT(0) + +#define QTB_OVR_ECATS_OUTFLD0 0x458 +#define QTB_OVR_ECATS_OUTFLD0_PA GENMASK(63, 12) +#define QTB_OVR_ECATS_OUTFLD0_FAULT_TYPE GENMASK(5, 4) +#define QTB_OVR_ECATS_OUTFLD0_FAULT BIT(0) + +#define QTB_NS_DBG_PORT_N_OT_SNAPSHOT(port_num) (0xc10 + (0x10 * port_num)) + +struct actlr_setting { + struct arm_smmu_smr smr; + u32 actlr; +}; + +struct qsmmuv500_archdata { + struct arm_smmu_device smmu; + struct list_head tbus; + struct actlr_setting *actlrs; + u32 actlr_tbl_size; + struct work_struct outstanding_tnx_work; + spinlock_t atos_lock; +}; +#define to_qsmmuv500_archdata(smmu) \ + container_of(smmu, struct qsmmuv500_archdata, smmu) + +struct qsmmuv500_group_iommudata { + bool has_actlr; + u32 actlr; +}; +#define to_qsmmuv500_group_iommudata(group) \ + ((struct qsmmuv500_group_iommudata *) \ + (iommu_group_get_iommudata(group))) + +struct qsmmuv500_tbu_device { + struct list_head list; + struct device *dev; + struct arm_smmu_device *smmu; + void __iomem *base; + + const struct qsmmuv500_tbu_impl *impl; + struct arm_smmu_power_resources *pwr; + u32 sid_start; + u32 num_sids; + u32 iova_width; + + /* Protects halt count */ + spinlock_t halt_lock; + u32 halt_count; +}; + +struct qsmmuv500_tbu_impl { + int (*halt_req)(struct qsmmuv500_tbu_device *tbu); + int (*halt_poll)(struct qsmmuv500_tbu_device *tbu); + void (*resume)(struct qsmmuv500_tbu_device *tbu); + phys_addr_t (*trigger_atos)(struct qsmmuv500_tbu_device *tbu, dma_addr_t iova, u32 sid, + unsigned long trans_flags); + void (*write_sync)(struct qsmmuv500_tbu_device *tbu); + void (*log_outstanding_transactions)(struct qsmmuv500_tbu_device *tbu); +}; + +struct arm_tbu_device { + struct qsmmuv500_tbu_device tbu; + bool has_micro_idle; +}; + +#define to_arm_tbu(tbu) container_of(tbu, struct arm_tbu_device, tbu) + +struct qtb500_device { + struct qsmmuv500_tbu_device tbu; + bool no_halt; + u32 num_ports; +}; + +#define to_qtb500(tbu) container_of(tbu, struct qtb500_device, tbu) + +static int arm_tbu_halt_req(struct qsmmuv500_tbu_device *tbu) +{ + void __iomem *tbu_base = tbu->base; + u32 halt; + + halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); + halt |= DEBUG_SID_HALT_REQ; + writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); + + return 0; +} + +static int arm_tbu_halt_poll(struct qsmmuv500_tbu_device *tbu) +{ + void __iomem *tbu_base = tbu->base; + u32 halt, status; + + if (readl_poll_timeout_atomic(tbu_base + DEBUG_SR_HALT_ACK_REG, status, + (status & DEBUG_SR_HALT_ACK_VAL), + 0, TBU_DBG_TIMEOUT_US)) { + dev_err(tbu->dev, "Couldn't halt TBU!\n"); + + halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); + halt &= ~DEBUG_SID_HALT_REQ; + writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG); + + return -ETIMEDOUT; + } + + return 0; +} + +static void arm_tbu_resume(struct qsmmuv500_tbu_device *tbu) +{ + void __iomem *base = tbu->base; + u32 val; + + val = readl_relaxed(base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_REQ; + writel_relaxed(val, base + DEBUG_SID_HALT_REG); +} + +static phys_addr_t arm_tbu_trigger_atos(struct qsmmuv500_tbu_device *tbu, dma_addr_t iova, u32 sid, + unsigned long trans_flags) +{ + void __iomem *tbu_base = tbu->base; + phys_addr_t phys = 0; + u64 val; + ktime_t timeout; + bool ecats_timedout = false; + + /* Set address and stream-id */ + val = readq_relaxed(tbu_base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID; + val |= FIELD_PREP(DEBUG_SID_HALT_SID, sid); + writeq_relaxed(val, tbu_base + DEBUG_SID_HALT_REG); + writeq_relaxed(iova, tbu_base + DEBUG_VA_ADDR_REG); + val = FIELD_PREP(DEBUG_AXUSER_CDMID, DEBUG_AXUSER_CDMID_VAL); + writeq_relaxed(val, tbu_base + DEBUG_AXUSER_REG); + + /* Write-back Read and Write-Allocate */ + val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xF); + + /* Non-secure Access */ + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_NSEC); + + /* Write or Read Access */ + if (trans_flags & IOMMU_TRANS_WRITE) + val |= DEBUG_TXN_WRITE; + + /* Priviledged or Unpriviledged Access */ + if (trans_flags & IOMMU_TRANS_PRIV) + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_PRIV); + + /* Data or Instruction Access */ + if (trans_flags & IOMMU_TRANS_INST) + val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_INST); + + val |= DEBUG_TXN_TRIGGER; + writeq_relaxed(val, tbu_base + DEBUG_TXN_TRIGG_REG); + + timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US); + for (;;) { + val = readl_relaxed(tbu_base + DEBUG_SR_HALT_ACK_REG); + if (!(val & DEBUG_SR_ECATS_RUNNING_VAL)) + break; + val = readl_relaxed(tbu_base + DEBUG_PAR_REG); + if (val & DEBUG_PAR_FAULT_VAL) + break; + if (ktime_compare(ktime_get(), timeout) > 0) { + ecats_timedout = true; + break; + } + } + + val = readq_relaxed(tbu_base + DEBUG_PAR_REG); + if (val & DEBUG_PAR_FAULT_VAL) + dev_err(tbu->dev, "ECATS generated a fault interrupt! PAR = %llx, SID=0x%x\n", + val, sid); + else if (ecats_timedout) + dev_err_ratelimited(tbu->dev, "ECATS translation timed out!\n"); + else + phys = FIELD_GET(DEBUG_PAR_PA, val); + + /* Reset hardware */ + writeq_relaxed(0, tbu_base + DEBUG_TXN_TRIGG_REG); + writeq_relaxed(0, tbu_base + DEBUG_VA_ADDR_REG); + val = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG); + val &= ~DEBUG_SID_HALT_SID; + writel_relaxed(val, tbu_base + DEBUG_SID_HALT_REG); + + return phys; +} + +static void arm_tbu_write_sync(struct qsmmuv500_tbu_device *tbu) +{ + readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG); +} + +static void arm_tbu_log_outstanding_transactions(struct qsmmuv500_tbu_device *tbu) +{ + void __iomem *base = tbu->base; + u64 outstanding_tnxs; + u64 tcr_cntl_val, res; + + tcr_cntl_val = readq_relaxed(base + TNX_TCR_CNTL); + + /* Write 1 into MATCH_MASK_UPD of TNX_TCR_CNTL */ + writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_MATCH_MASK_UPD, + base + TNX_TCR_CNTL); + + /* + * Simultaneously write 0 into MATCH_MASK_UPD, 0 into + * ALWAYS_CAPTURE, 0 into MATCH_MASK_VALID, and 1 into + * TBU_OT_CAPTURE_EN of TNX_TCR_CNTL + */ + tcr_cntl_val &= ~(TNX_TCR_CNTL_MATCH_MASK_UPD | + TNX_TCR_CNTL_ALWAYS_CAPTURE | + TNX_TCR_CNTL_MATCH_MASK_VALID); + writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, + base + TNX_TCR_CNTL); + + /* Poll for CAPTURE1_VALID to become 1 on TNX_TCR_CNTL_2 */ + if (readq_poll_timeout_atomic(base + TNX_TCR_CNTL_2, res, + res & TNX_TCR_CNTL_2_CAP1_VALID, + 0, TBU_DBG_TIMEOUT_US)) { + dev_err_ratelimited(tbu->dev, + "Timeout on TNX snapshot poll\n"); + goto poll_timeout; + } + + /* Read Register CAPTURE1_SNAPSHOT_1 */ + outstanding_tnxs = readq_relaxed(base + CAPTURE1_SNAPSHOT_1); + dev_err_ratelimited(tbu->dev, + "Outstanding Transaction Bitmap: 0x%llx\n", + outstanding_tnxs); + +poll_timeout: + /* Write TBU_OT_CAPTURE_EN to 0 of TNX_TCR_CNTL */ + writeq_relaxed(tcr_cntl_val & ~TNX_TCR_CNTL_TBU_OT_CAPTURE_EN, + tbu->base + TNX_TCR_CNTL); +} + +static const struct qsmmuv500_tbu_impl arm_tbu_impl = { + .halt_req = arm_tbu_halt_req, + .halt_poll = arm_tbu_halt_poll, + .resume = arm_tbu_resume, + .trigger_atos = arm_tbu_trigger_atos, + .write_sync = arm_tbu_write_sync, + .log_outstanding_transactions = arm_tbu_log_outstanding_transactions, +}; + +/* + * Prior to accessing registers in the TBU local register space, + * TBU must be woken from micro idle. + */ +static int __arm_tbu_micro_idle_cfg(struct arm_smmu_device *smmu, + u32 val, u32 mask) +{ + void __iomem *reg; + u32 tmp, new; + unsigned long flags; + int ret; + + /* Protect APPS_SMMU_TBU_REG_ACCESS register. */ + spin_lock_irqsave(&smmu->global_sync_lock, flags); + new = arm_smmu_readl(smmu, ARM_SMMU_IMPL_DEF5, + APPS_SMMU_TBU_REG_ACCESS_REQ_NS); + new &= ~mask; + new |= val; + arm_smmu_writel(smmu, ARM_SMMU_IMPL_DEF5, + APPS_SMMU_TBU_REG_ACCESS_REQ_NS, + new); + + reg = arm_smmu_page(smmu, ARM_SMMU_IMPL_DEF5); + reg += APPS_SMMU_TBU_REG_ACCESS_ACK_NS; + ret = readl_poll_timeout_atomic(reg, tmp, ((tmp & mask) == val), 0, 200); + if (ret) + dev_WARN(smmu->dev, "Timed out configuring micro idle! %x instead of %x\n", tmp, + new); + /* + * While the micro-idle guard sequence registers may have been configured + * properly, it is possible that the intended effect has not been realized + * by the power management hardware due to delays in the system. + * + * Spin for a short amount of time to allow for the desired configuration to + * take effect before proceeding. + */ + udelay(TBU_MICRO_IDLE_DELAY_US); + spin_unlock_irqrestore(&smmu->global_sync_lock, flags); + return ret; +} + +int arm_tbu_micro_idle_wake(struct arm_smmu_power_resources *pwr) +{ + struct qsmmuv500_tbu_device *tbu = dev_get_drvdata(pwr->dev); + struct arm_tbu_device *arm_tbu = to_arm_tbu(tbu); + u32 val; + + if (!arm_tbu->has_micro_idle) + return 0; + + val = tbu->sid_start >> 10; + val = 1 << val; + return __arm_tbu_micro_idle_cfg(tbu->smmu, val, val); +} + +void arm_tbu_micro_idle_allow(struct arm_smmu_power_resources *pwr) +{ + struct qsmmuv500_tbu_device *tbu = dev_get_drvdata(pwr->dev); + struct arm_tbu_device *arm_tbu = to_arm_tbu(tbu); + u32 val; + + if (!arm_tbu->has_micro_idle) + return; + + val = tbu->sid_start >> 10; + val = 1 << val; + __arm_tbu_micro_idle_cfg(tbu->smmu, 0, val); +} + +static struct qsmmuv500_tbu_device *arm_tbu_impl_init(struct qsmmuv500_tbu_device *tbu) +{ + struct arm_tbu_device *arm_tbu; + struct device *dev = tbu->dev; + + arm_tbu = devm_krealloc(dev, tbu, sizeof(*arm_tbu), GFP_KERNEL); + if (!arm_tbu) + return ERR_PTR(-ENOMEM); + + arm_tbu->tbu.impl = &arm_tbu_impl; + arm_tbu->has_micro_idle = of_property_read_bool(dev->of_node, "qcom,micro-idle"); + + if (arm_tbu->has_micro_idle) { + arm_tbu->tbu.pwr->resume = arm_tbu_micro_idle_wake; + arm_tbu->tbu.pwr->suspend = arm_tbu_micro_idle_allow; + } + + return &arm_tbu->tbu; +} + +static int qtb500_tbu_halt_req(struct qsmmuv500_tbu_device *tbu) +{ + void __iomem *qtb_base = tbu->base; + struct qtb500_device *qtb = to_qtb500(tbu); + u64 val; + + if (qtb->no_halt) + return 0; + + val = readq_relaxed(qtb_base + QTB_OVR_DBG_FENCEREQ); + val |= QTB_OVR_DBG_FENCEREQ_HALT; + writeq_relaxed(val, qtb_base + QTB_OVR_DBG_FENCEREQ); + + return 0; +} + +static int qtb500_tbu_halt_poll(struct qsmmuv500_tbu_device *tbu) +{ + void __iomem *qtb_base = tbu->base; + struct qtb500_device *qtb = to_qtb500(tbu); + u64 val, status; + + if (qtb->no_halt) + return 0; + + if (readq_poll_timeout_atomic(qtb_base + QTB_OVR_DBG_FENCEACK, status, + (status & QTB_OVR_DBG_FENCEACK_ACK), 0, + QTB_DBG_TIMEOUT_US)) { + dev_err(tbu->dev, "Couldn't halt QTB\n"); + + val = readq_relaxed(qtb_base + QTB_OVR_DBG_FENCEREQ); + val &= ~QTB_OVR_DBG_FENCEREQ_HALT; + writeq_relaxed(val, qtb_base + QTB_OVR_DBG_FENCEREQ); + + return -ETIMEDOUT; + } + + return 0; +} + +static void qtb500_tbu_resume(struct qsmmuv500_tbu_device *tbu) +{ + void __iomem *qtb_base = tbu->base; + struct qtb500_device *qtb = to_qtb500(tbu); + u64 val; + + if (qtb->no_halt) + return; + + val = readq_relaxed(qtb_base + QTB_OVR_DBG_FENCEREQ); + val &= ~QTB_OVR_DBG_FENCEREQ_HALT; + writeq_relaxed(val, qtb_base + QTB_OVR_DBG_FENCEREQ); +} + +static phys_addr_t qtb500_trigger_atos(struct qsmmuv500_tbu_device *tbu, dma_addr_t iova, + u32 sid, unsigned long trans_flags) +{ + void __iomem *qtb_base = tbu->base; + u64 infld0, infld1, infld2, val; + phys_addr_t phys = 0; + ktime_t timeout; + bool ecats_timedout = false; + + /* + * Recommended to set: + * + * QTB_OVR_ECATS_INFLD0.QAD == 0 (AP Access Domain) + * QTB_OVR_EACTS_INFLD0.PCIE_NO_SNOOP == 0 (IO-Coherency enabled) + */ + infld0 = FIELD_PREP(QTB_OVR_ECATS_INFLD0_SID, sid); + if (trans_flags & IOMMU_TRANS_SEC) + infld0 |= QTB_OVR_ECATS_INFLD0_SEC_SID; + + infld1 = 0; + if (trans_flags & IOMMU_TRANS_PRIV) + infld1 |= QTB_OVR_ECATS_INFLD1_PNU; + if (trans_flags & IOMMU_TRANS_INST) + infld1 |= QTB_OVR_ECATS_INFLD1_IND; + /* + * Recommended to set: + * + * QTB_OVR_ECATS_INFLD1.DIRTY == 0, + * QTB_OVR_ECATS_INFLD1.TR_TYPE == 4 (Cacheable and Shareable memory) + * QTB_OVR_ECATS_INFLD1.ALLOC == 0 (No allocation in TLB/caches) + */ + infld1 |= FIELD_PREP(QTB_OVR_ECATS_INFLD1_TR_TYPE, QTB_OVR_ECATS_INFLD1_TR_TYPE_SHARED); + if (!(trans_flags & IOMMU_TRANS_SEC)) + infld1 |= QTB_OVR_ECATS_INFLD1_NON_SEC; + if (trans_flags & IOMMU_TRANS_WRITE) + infld1 |= FIELD_PREP(QTB_OVR_ECATS_INFLD1_OPC, QTB_OVR_ECATS_INFLD1_OPC_WRI); + + infld2 = iova; + + writeq_relaxed(infld0, qtb_base + QTB_OVR_ECATS_INFLD0); + writeq_relaxed(infld1, qtb_base + QTB_OVR_ECATS_INFLD1); + writeq_relaxed(infld2, qtb_base + QTB_OVR_ECATS_INFLD2); + writeq_relaxed(QTB_OVR_ECATS_TRIGGER_START, qtb_base + QTB_OVR_ECATS_TRIGGER); + + timeout = ktime_add_us(ktime_get(), QTB_DBG_TIMEOUT_US); + for (;;) { + val = readq_relaxed(qtb_base + QTB_OVR_ECATS_STATUS); + if (val & QTB_OVR_ECATS_STATUS_DONE) + break; + val = readq_relaxed(qtb_base + QTB_OVR_ECATS_OUTFLD0); + if (val & QTB_OVR_ECATS_OUTFLD0_FAULT) + break; + if (ktime_compare(ktime_get(), timeout) > 0) { + ecats_timedout = true; + break; + } + } + + val = readq_relaxed(qtb_base + QTB_OVR_ECATS_OUTFLD0); + if (val & QTB_OVR_ECATS_OUTFLD0_FAULT) + dev_err(tbu->dev, "ECATS generated a fault interrupt! OUTFLD0 = 0x%llx SID = 0x%x\n", + val, sid); + else if (ecats_timedout) + dev_err_ratelimited(tbu->dev, "ECATS translation timed out!\n"); + else + phys = FIELD_GET(QTB_OVR_ECATS_OUTFLD0_PA, val); + + /* Reset hardware for next transaction. */ + writeq_relaxed(0, qtb_base + QTB_OVR_ECATS_TRIGGER); + + return phys; +} + +static void qtb500_tbu_write_sync(struct qsmmuv500_tbu_device *tbu) +{ + readl_relaxed(tbu->base + QTB_SWID_LOW); +} + +static void qtb500_log_outstanding_transactions(struct qsmmuv500_tbu_device *tbu) +{ + void __iomem *qtb_base = tbu->base; + struct qtb500_device *qtb = to_qtb500(tbu); + u64 outstanding_tnx; + int i; + + for (i = 0; i < qtb->num_ports; i++) { + outstanding_tnx = readq_relaxed(qtb_base + QTB_NS_DBG_PORT_N_OT_SNAPSHOT(i)); + dev_err(tbu->dev, "port %d outstanding transactions bitmap: 0x%llx\n", i, + outstanding_tnx); + } +} + +static const struct qsmmuv500_tbu_impl qtb500_impl = { + .halt_req = qtb500_tbu_halt_req, + .halt_poll = qtb500_tbu_halt_poll, + .resume = qtb500_tbu_resume, + .trigger_atos = qtb500_trigger_atos, + .write_sync = qtb500_tbu_write_sync, + .log_outstanding_transactions = qtb500_log_outstanding_transactions, +}; + +static struct qsmmuv500_tbu_device *qtb500_impl_init(struct qsmmuv500_tbu_device *tbu) +{ + struct qtb500_device *qtb; + struct device *dev = tbu->dev; + int ret; + + qtb = devm_krealloc(dev, tbu, sizeof(*qtb), GFP_KERNEL); + if (!qtb) + return ERR_PTR(-ENOMEM); + + qtb->tbu.impl = &qtb500_impl; + + ret = of_property_read_u32(dev->of_node, "qcom,num-qtb-ports", &qtb->num_ports); + if (ret) + return ERR_PTR(ret); + + qtb->no_halt = of_property_read_bool(dev->of_node, "qcom,no-qtb-atos-halt"); + + return &qtb->tbu; +} + +static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu(struct arm_smmu_device *smmu, u32 sid) +{ + struct qsmmuv500_tbu_device *tbu = NULL; + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); + + list_for_each_entry(tbu, &data->tbus, list) { + if (tbu->sid_start <= sid && + sid < tbu->sid_start + tbu->num_sids) + return tbu; + } + + return NULL; +} + +static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu, + struct arm_smmu_domain *smmu_domain) +{ + unsigned long flags; + struct arm_smmu_device *smmu = smmu_domain->smmu; + int ret = 0, idx = smmu_domain->cfg.cbndx; + u32 fsr; + + if (of_property_read_bool(tbu->dev->of_node, "qcom,opt-out-tbu-halting")) { + dev_notice(tbu->dev, "TBU opted-out for halting!\n"); + return -EBUSY; + } + + spin_lock_irqsave(&tbu->halt_lock, flags); + if (tbu->halt_count) { + tbu->halt_count++; + goto out; + } + + ret = tbu->impl->halt_req(tbu); + if (ret) + goto out; + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if ((fsr & ARM_SMMU_FSR_FAULT) && (fsr & ARM_SMMU_FSR_SS)) { + u32 sctlr_orig, sctlr; + /* + * We are in a fault; Our request to halt the bus will not + * complete until transactions in front of us (such as the fault + * itself) have completed. Disable iommu faults and terminate + * any existing transactions. + */ + sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(ARM_SMMU_SCTLR_CFCFG | ARM_SMMU_SCTLR_CFIE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation + */ + wmb(); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + ARM_SMMU_RESUME_TERMINATE); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); + } + + ret = tbu->impl->halt_poll(tbu); + if (ret) + goto out; + + tbu->halt_count = 1; +out: + spin_unlock_irqrestore(&tbu->halt_lock, flags); + return ret; +} + +static void qsmmuv500_tbu_resume(struct qsmmuv500_tbu_device *tbu) +{ + unsigned long flags; + + spin_lock_irqsave(&tbu->halt_lock, flags); + if (WARN(!tbu->halt_count, "%s bad tbu->halt_count", dev_name(tbu->dev))) { + goto out; + + } else if (tbu->halt_count > 1) { + tbu->halt_count--; + goto out; + } + + tbu->impl->resume(tbu); + + tbu->halt_count = 0; +out: + spin_unlock_irqrestore(&tbu->halt_lock, flags); +} + +/* + * Provides mutually exclusive access to the registers used by the + * outstanding transaction snapshot feature and the transaction + * snapshot capture feature. + */ +static DEFINE_MUTEX(capture_reg_lock); + +static void qsmmuv500_log_outstanding_transactions(struct work_struct *work) +{ + struct qsmmuv500_tbu_device *tbu = NULL; + struct qsmmuv500_archdata *data = container_of(work, + struct qsmmuv500_archdata, + outstanding_tnx_work); + struct arm_smmu_device *smmu = &data->smmu; + + if (!mutex_trylock(&capture_reg_lock)) { + dev_warn_ratelimited(smmu->dev, + "Tnx snapshot regs in use, not dumping OT tnxs.\n"); + goto bug; + } + + if (arm_smmu_power_on(smmu->pwr)) { + dev_err_ratelimited(smmu->dev, + "%s: Failed to power on SMMU.\n", + __func__); + goto unlock; + } + + list_for_each_entry(tbu, &data->tbus, list) { + if (arm_smmu_power_on(tbu->pwr)) { + dev_err_ratelimited(tbu->dev, "%s: Failed to power on TBU.\n", __func__); + continue; + } + + tbu->impl->log_outstanding_transactions(tbu); + + arm_smmu_power_off(smmu, tbu->pwr); + } + + arm_smmu_power_off(smmu, smmu->pwr); +unlock: + mutex_unlock(&capture_reg_lock); +bug: + BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); +} + +static struct qsmmuv500_tbu_device *qsmmuv500_tbu_impl_init(struct qsmmuv500_tbu_device *tbu) +{ + if (of_device_is_compatible(tbu->dev->of_node, "qcom,qtb500")) + return qtb500_impl_init(tbu); + + return arm_tbu_impl_init(tbu); +} + +static int qsmmuv500_tbu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct qsmmuv500_tbu_device *tbu; + struct resource *res; + const __be32 *cell; + int ret, len; + + tbu = devm_kzalloc(dev, sizeof(*tbu), GFP_KERNEL); + if (!tbu) + return -ENOMEM; + + tbu->dev = dev; + + /* + * ARM TBUs need to have power resources initialized before its + * implementation defined initialization occurs to setup the + * suspend and resure power callbacks. + */ + tbu->pwr = arm_smmu_init_power_resources(dev); + if (IS_ERR(tbu->pwr)) + return PTR_ERR(tbu->pwr); + + tbu = qsmmuv500_tbu_impl_init(tbu); + if (IS_ERR(tbu)) + return PTR_ERR(tbu); + + INIT_LIST_HEAD(&tbu->list); + + spin_lock_init(&tbu->halt_lock); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -EINVAL; + + tbu->base = devm_ioremap(dev, res->start, resource_size(res)); + if (!tbu->base) + return -ENOMEM; + + cell = of_get_property(dev->of_node, "qcom,stream-id-range", &len); + if (!cell || len < 8) + return -EINVAL; + + tbu->sid_start = of_read_number(cell, 1); + tbu->num_sids = of_read_number(cell + 1, 1); + + ret = of_property_read_u32(dev->of_node, "qcom,iova-width", &tbu->iova_width); + if (ret < 0) + return ret; + + dev_set_drvdata(dev, tbu); + return 0; +} + +static const struct of_device_id qsmmuv500_tbu_of_match[] = { + {.compatible = "qcom,qsmmuv500-tbu"}, + {} +}; + +struct platform_driver qsmmuv500_tbu_driver = { + .driver = { + .name = "qsmmuv500-tbu", + .of_match_table = of_match_ptr(qsmmuv500_tbu_of_match), + }, + .probe = qsmmuv500_tbu_probe, +}; + +static void qsmmuv500_tlb_sync_timeout(struct arm_smmu_device *smmu) +{ + u32 sync_inv_ack, tbu_pwr_status, sync_inv_progress; + u32 tbu_inv_pending = 0, tbu_sync_pending = 0; + u32 tbu_inv_acked = 0, tbu_sync_acked = 0; + u32 tcu_inv_pending = 0, tcu_sync_pending = 0; + unsigned long tbu_ids = 0; + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); + int ret; + + static DEFINE_RATELIMIT_STATE(_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + dev_err_ratelimited(smmu->dev, + "TLB sync timed out -- SMMU may be deadlocked\n"); + + sync_inv_ack = arm_smmu_readl(smmu, + ARM_SMMU_IMPL_DEF5, + ARM_SMMU_STATS_SYNC_INV_TBU_ACK); + + if (sync_inv_ack) { + tbu_inv_pending = FIELD_GET(TBU_INV_REQ, sync_inv_ack); + tbu_inv_acked = FIELD_GET(TBU_INV_ACK, sync_inv_ack); + tbu_sync_pending = FIELD_GET(TBU_SYNC_REQ, sync_inv_ack); + tbu_sync_acked = FIELD_GET(TBU_SYNC_ACK, sync_inv_ack); + } + + ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + + ARM_SMMU_TBU_PWR_STATUS), &tbu_pwr_status); + if (ret) { + dev_err_ratelimited(smmu->dev, + "SCM read of TBU power status fails: %d\n", + ret); + goto out; + } + + ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr + + ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR), + &sync_inv_progress); + if (ret) { + dev_err_ratelimited(smmu->dev, + "SCM read of TBU sync/inv prog fails: %d\n", + ret); + goto out; + } + + if (tbu_pwr_status) { + if (tbu_sync_pending) + tbu_ids = tbu_pwr_status & ~tbu_sync_acked; + else if (tbu_inv_pending) + tbu_ids = tbu_pwr_status & ~tbu_inv_acked; + } + + tcu_inv_pending = FIELD_GET(TCU_INV_IN_PRGSS, sync_inv_progress); + tcu_sync_pending = FIELD_GET(TCU_SYNC_IN_PRGSS, sync_inv_progress); + + if (__ratelimit(&_rs)) { + unsigned long tbu_id; + + dev_err(smmu->dev, + "TBU ACK 0x%x TBU PWR 0x%x TCU sync_inv 0x%x\n", + sync_inv_ack, tbu_pwr_status, sync_inv_progress); + dev_err(smmu->dev, + "TCU invalidation %s, TCU sync %s\n", + tcu_inv_pending?"pending":"completed", + tcu_sync_pending?"pending":"completed"); + + for_each_set_bit(tbu_id, &tbu_ids, sizeof(tbu_ids) * + BITS_PER_BYTE) { + + struct qsmmuv500_tbu_device *tbu; + + tbu = qsmmuv500_find_tbu(smmu, + (u16)(tbu_id << TBUID_SHIFT)); + if (tbu) { + dev_err(smmu->dev, + "TBU %s ack pending for TBU %s, %s\n", + tbu_sync_pending?"sync" : "inv", + dev_name(tbu->dev), + tbu_sync_pending ? + "check pending transactions on TBU" + : "check for TBU power status"); + } + } + } + + if (tcu_sync_pending) { + schedule_work(&data->outstanding_tnx_work); + return; + } +out: + if (ret) { + if (sync_inv_ack) { + /* TBU PWR status is not available so just dump raw + * fields + */ + dev_err(smmu->dev, + "TBU %s ack pending, got ack for TBUs %d, %s\n", + tbu_sync_pending ? "sync" : "inv", + tbu_sync_pending ? tbu_sync_acked:tbu_inv_acked, + tbu_sync_pending ? + "check pending transactions on TBU" + : "check for TBU power status"); + + } + + dev_err(smmu->dev, "TBU SYNC_INV_ACK reg 0x%x\n", sync_inv_ack); + } + + BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG)); +} + +static void qsmmuv500_device_remove(struct arm_smmu_device *smmu) +{ + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); + + cancel_work_sync(&data->outstanding_tnx_work); +} + +/* + * Checks whether smr2 is a subset of smr + */ +static bool smr_is_subset(struct arm_smmu_smr *smr2, struct arm_smmu_smr *smr) +{ + return (smr->mask & smr2->mask) == smr2->mask && + !((smr->id ^ smr2->id) & ~smr->mask); +} + +/* + * Zero means failure. + */ +static phys_addr_t qsmmuv500_iova_to_phys(struct arm_smmu_domain *smmu_domain, dma_addr_t iova, + u32 sid, unsigned long trans_flags) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct qsmmuv500_archdata *data = to_qsmmuv500_archdata(smmu); + struct qsmmuv500_tbu_device *tbu; + phys_addr_t phys = 0; + int idx = cfg->cbndx; + int needs_redo = 0; + u32 sctlr_orig, sctlr, fsr; + unsigned long spinlock_flags; + + tbu = qsmmuv500_find_tbu(smmu, sid); + if (!tbu) + return 0; + + if (arm_smmu_power_on(tbu->pwr)) + return 0; + + if (iova >= (1ULL << tbu->iova_width)) { + dev_err_ratelimited(tbu->dev, "ECATS: address too large: %pad\n", &iova); + return 0; + } + + if (qsmmuv500_tbu_halt(tbu, smmu_domain)) + goto out_power_off; + + /* + * ECATS can trigger the fault interrupt, so disable it temporarily + * and check for an interrupt manually. + */ + sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR); + sctlr = sctlr_orig & ~(ARM_SMMU_SCTLR_CFCFG | ARM_SMMU_SCTLR_CFIE); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (fsr & ARM_SMMU_FSR_FAULT) { + /* Clear pending interrupts */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation. + */ + wmb(); + + /* + * TBU halt takes care of resuming any stalled transcation. + * Kept it here for completeness sake. + */ + if (fsr & ARM_SMMU_FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + ARM_SMMU_RESUME_TERMINATE); + } + + /* Only one concurrent atos operation */ + spin_lock_irqsave(&data->atos_lock, spinlock_flags); + + /* + * After a failed translation, the next successful translation will + * incorrectly be reported as a failure. + */ + do { + phys = tbu->impl->trigger_atos(tbu, iova, sid, trans_flags); + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (fsr & ARM_SMMU_FSR_FAULT) { + /* Clear pending interrupts */ + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation. + */ + wmb(); + + if (fsr & ARM_SMMU_FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + ARM_SMMU_RESUME_TERMINATE); + } + } while (!phys && needs_redo++ < 2); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig); + spin_unlock_irqrestore(&data->atos_lock, spinlock_flags); + qsmmuv500_tbu_resume(tbu); + +out_power_off: + /* Read to complete prior write transcations */ + tbu->impl->write_sync(tbu); + + /* Wait for read to complete before off */ + rmb(); + + arm_smmu_power_off(tbu->smmu, tbu->pwr); + + return phys; +} + +static phys_addr_t qsmmuv500_iova_to_phys_hard( + struct arm_smmu_domain *smmu_domain, + struct qcom_iommu_atos_txn *txn) +{ + return qsmmuv500_iova_to_phys(smmu_domain, txn->addr, txn->id, + txn->flags); +} + +static void qsmmuv500_release_group_iommudata(void *data) +{ + kfree(data); +} + +/* If a device has a valid actlr, it must match */ +static int qsmmuv500_device_group(struct device *dev, + struct iommu_group *group) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); + struct arm_smmu_device *smmu; + struct qsmmuv500_archdata *data; + struct qsmmuv500_group_iommudata *iommudata; + u32 actlr, i, j, idx; + struct arm_smmu_smr *smr, *smr2; + + if (!fwspec || !cfg) + return -EINVAL; + + smmu = cfg->smmu; + data = to_qsmmuv500_archdata(smmu); + + iommudata = to_qsmmuv500_group_iommudata(group); + if (!iommudata) { + iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL); + if (!iommudata) + return -ENOMEM; + + iommu_group_set_iommudata(group, iommudata, + qsmmuv500_release_group_iommudata); + } + + for (i = 0; i < data->actlr_tbl_size; i++) { + smr = &data->actlrs[i].smr; + actlr = data->actlrs[i].actlr; + + for_each_cfg_sme(cfg, fwspec, j, idx) { + smr2 = &smmu->smrs[idx]; + if (!smr_is_subset(smr2, smr)) + continue; + + dev_dbg(dev, "Matched actlr sid=%x mask=%x actlr=%x\n", + smr->id, smr->mask, actlr); + + if (!iommudata->has_actlr) { + iommudata->actlr = actlr; + iommudata->has_actlr = true; + } else if (iommudata->actlr != actlr) { + dev_err(dev, "Invalid actlr setting\n"); + return -EINVAL; + } + } + } + + return 0; +} + +static void qsmmuv500_init_cb(struct arm_smmu_domain *smmu_domain, + struct device *dev) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct qsmmuv500_group_iommudata *iommudata = + to_qsmmuv500_group_iommudata(dev->iommu_group); + int idx = smmu_domain->cfg.cbndx; + + if (!iommudata->has_actlr) + return; + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ACTLR, iommudata->actlr); + + /* + * Flush the context bank after modifying ACTLR to ensure there + * are no cache entries with stale state + */ + iommu_flush_iotlb_all(&smmu_domain->domain); +} + +static int qsmmuv500_tbu_register(struct device *dev, void *cookie) +{ + struct qsmmuv500_tbu_device *tbu; + struct qsmmuv500_archdata *data = cookie; + + if (!dev->driver) { + dev_err(dev, "TBU failed probe, QSMMUV500 cannot continue!\n"); + return -EINVAL; + } + + tbu = dev_get_drvdata(dev); + + INIT_LIST_HEAD(&tbu->list); + tbu->smmu = &data->smmu; + list_add(&tbu->list, &data->tbus); + return 0; +} + +static int qsmmuv500_read_actlr_tbl(struct qsmmuv500_archdata *data) +{ + int len, i; + struct device *dev = data->smmu.dev; + struct actlr_setting *actlrs; + const __be32 *cell; + + cell = of_get_property(dev->of_node, "qcom,actlr", NULL); + if (!cell) + return 0; + + len = of_property_count_elems_of_size(dev->of_node, "qcom,actlr", + sizeof(u32) * 3); + if (len < 0) + return 0; + + actlrs = devm_kzalloc(dev, sizeof(*actlrs) * len, GFP_KERNEL); + if (!actlrs) + return -ENOMEM; + + for (i = 0; i < len; i++) { + actlrs[i].smr.id = of_read_number(cell++, 1); + actlrs[i].smr.mask = of_read_number(cell++, 1); + actlrs[i].actlr = of_read_number(cell++, 1); + } + + data->actlrs = actlrs; + data->actlr_tbl_size = len; + return 0; +} + +static int qsmmuv500_cfg_probe(struct arm_smmu_device *smmu) +{ + u32 val; + + val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); + val &= ~ARM_MMU500_ACR_CACHE_LOCK; + arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, val); + val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR); + /* + * Modifiying the nonsecure copy of the sACR register is only + * allowed if permission is given in the secure sACR register. + * Attempt to detect if we were able to update the value. + */ + WARN_ON(val & ARM_MMU500_ACR_CACHE_LOCK); + + return 0; +} + +/* + * Case 1) + * Client wants to use the standard upstream dma ops from + * drivers/iommu/dma-iommu.c + * + * This requires domain->type == IOMMU_DOMAIN_DMA. + * + * Case 2) + * Client doesn't want to use the default dma domain, and wants to + * allocate their own via iommu_domain_alloc() + * + * There are insufficient context banks to "waste" one on a default + * dma domain that isn't going to be used. Therefore, give it + * IOMMU_DOMAIN_IDENTITY. Note that IOMMU_DOMAIN_IDENTITY is treated as + * IOMMU_DOMAIN_BLOCKED by our hypervisor, which doesn't allow setting + * S2CR.TYPE = 0x1. + * + * Case 3) + * Client wants to use our fastmap dma ops + * + * Per case 1) we cannot use IOMMU_DOMAIN_DMA, since those imply using + * the standard upstream dma ops. So use IOMMU_DOMAIN_UNMANAGED instead. + * + * Case 4) + * Client wants to use S1 bypass + * + * Same as Case 3, except use the platform dma ops. + */ +static int qsmmuv500_def_domain_type(struct device *dev) +{ + const char *str; + struct device_node *np; + + /* Default to iommu_def_domain_type */ + np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); + if (!np) + np = dev->of_node; + + if (of_property_read_string(np, "qcom,iommu-dma", &str)) + str = "default"; + + if (!strcmp(str, "fastmap") && + IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_FAST)) + return IOMMU_DOMAIN_UNMANAGED; + if (!strcmp(str, "bypass")) + return IOMMU_DOMAIN_UNMANAGED; + if (!strcmp(str, "atomic")) + return IOMMU_DOMAIN_DMA; + if (!strcmp(str, "disabled")) + return IOMMU_DOMAIN_IDENTITY; + + return IOMMU_DOMAIN_DMA; +} + +static const struct arm_smmu_impl qsmmuv500_impl = { + .cfg_probe = qsmmuv500_cfg_probe, + .init_context_bank = qsmmuv500_init_cb, + .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, + .tlb_sync_timeout = qsmmuv500_tlb_sync_timeout, + .device_remove = qsmmuv500_device_remove, + .device_group = qsmmuv500_device_group, + .def_domain_type = qsmmuv500_def_domain_type, +}; + +static const struct arm_smmu_impl qsmmuv500_adreno_impl = { + .init_context = qcom_adreno_smmu_init_context, + .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank, + .cfg_probe = qsmmuv500_cfg_probe, + .init_context_bank = qsmmuv500_init_cb, + .iova_to_phys_hard = qsmmuv500_iova_to_phys_hard, + .tlb_sync_timeout = qsmmuv500_tlb_sync_timeout, + .device_remove = qsmmuv500_device_remove, + .device_group = qsmmuv500_device_group, + .def_domain_type = qsmmuv500_def_domain_type, +}; + +/* We only have access to arm-architected registers */ +static const struct arm_smmu_impl qsmmuv500_virt_impl = { + .cfg_probe = qsmmuv500_cfg_probe, + .init_context_bank = qsmmuv500_init_cb, + .device_group = qsmmuv500_device_group, + .def_domain_type = qsmmuv500_def_domain_type, +}; + +struct arm_smmu_device *qsmmuv500_create(struct arm_smmu_device *smmu, + const struct arm_smmu_impl *impl) +{ + struct device *dev = smmu->dev; + struct qsmmuv500_archdata *data; + int ret; + + /* + * devm_krealloc() invokes devm_kmalloc(), so we pass __GFP_ZERO + * to ensure that fields after smmu are initialized, even if we don't + * initialize them (e.g. ACTLR related fields). + */ + data = devm_krealloc(dev, smmu, sizeof(*data), GFP_KERNEL | __GFP_ZERO); + if (!data) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&data->tbus); + spin_lock_init(&data->atos_lock); + INIT_WORK(&data->outstanding_tnx_work, + qsmmuv500_log_outstanding_transactions); + data->smmu.impl = impl; + + ret = qsmmuv500_read_actlr_tbl(data); + if (ret) + return ERR_PTR(ret); + + ret = of_platform_populate(dev->of_node, NULL, NULL, dev); + if (ret) + return ERR_PTR(ret); + + /* Attempt to register child devices */ + ret = device_for_each_child(dev, data, qsmmuv500_tbu_register); + if (ret) + return ERR_PTR(-EPROBE_DEFER); + + return &data->smmu; +} + +static struct arm_smmu_device *qsmmuv500_virt_create(struct arm_smmu_device *smmu, + const struct arm_smmu_impl *impl) +{ + struct device *dev = smmu->dev; + struct qsmmuv500_archdata *data; + int ret; + + data = devm_krealloc(dev, smmu, sizeof(*data), GFP_KERNEL | __GFP_ZERO); + if (!data) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&data->tbus); + spin_lock_init(&data->atos_lock); + INIT_WORK(&data->outstanding_tnx_work, + qsmmuv500_log_outstanding_transactions); + data->smmu.impl = impl; + + ret = qsmmuv500_read_actlr_tbl(data); + if (ret) + return ERR_PTR(ret); + + return &data->smmu; +} + +struct arm_smmu_device *qsmmuv500_impl_init(struct arm_smmu_device *smmu) +{ + if (of_device_is_compatible(smmu->dev->of_node, "qcom,adreno-smmu")) + return qsmmuv500_create(smmu, &qsmmuv500_adreno_impl); + + if (of_device_is_compatible(smmu->dev->of_node, "qcom,virt-smmu")) + return qsmmuv500_virt_create(smmu, &qsmmuv500_virt_impl); + + return qsmmuv500_create(smmu, &qsmmuv500_impl); +} + static const struct arm_smmu_impl qcom_adreno_smmu_impl = { .init_context = qcom_adreno_smmu_init_context, .def_domain_type = qcom_smmu_def_domain_type, diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-trace.h b/drivers/iommu/arm/arm-smmu/arm-smmu-trace.h new file mode 100644 index 000000000000..c6786808c6d7 --- /dev/null +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-trace.h @@ -0,0 +1,224 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019, 2021 The Linux Foundation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM arm_smmu + +#if !defined(_TRACE_ARM_SMMU_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_ARM_SMMU_H + +#include +#include +#include +#include "arm-smmu.h" + +struct device; + +DECLARE_EVENT_CLASS(iommu_tlbi, + + TP_PROTO(struct arm_smmu_domain *domain), + + TP_ARGS(domain), + + TP_STRUCT__entry( + __string(group_name, dev_name(domain->dev)) + ), + + TP_fast_assign( + __assign_str(group_name, dev_name(domain->dev)); + ), + + TP_printk("group=%s", + __get_str(group_name) + ) +); + +DEFINE_EVENT(iommu_tlbi, tlbi_start, + + TP_PROTO(struct arm_smmu_domain *domain), + + TP_ARGS(domain) +); + +DEFINE_EVENT(iommu_tlbi, tlbi_end, + + TP_PROTO(struct arm_smmu_domain *domain), + + TP_ARGS(domain) +); + +DECLARE_EVENT_CLASS(iommu_pgtable, + + TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, + unsigned long long ipa, size_t granule), + + TP_ARGS(domain, iova, ipa, granule), + + TP_STRUCT__entry( + __string(group_name, dev_name(domain->dev)) + __field(unsigned long, iova) + __field(unsigned long long, ipa) + __field(size_t, granule) + ), + + TP_fast_assign( + __assign_str(group_name, dev_name(domain->dev)); + __entry->iova = iova; + __entry->ipa = ipa; + __entry->granule = granule; + ), + + TP_printk("group=%s table_base_iova=%lx table_ipa=%llx table_size=%zx", + __get_str(group_name), __entry->iova, + __entry->ipa, __entry->granule + ) +); + +DEFINE_EVENT(iommu_pgtable, iommu_pgtable_add, + + TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, + unsigned long long ipa, size_t granule), + + TP_ARGS(domain, iova, ipa, granule) +); + +DEFINE_EVENT(iommu_pgtable, iommu_pgtable_remove, + + TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, + unsigned long long ipa, size_t granule), + + TP_ARGS(domain, iova, ipa, granule) +); + +DECLARE_EVENT_CLASS(iommu_map_pages, + + TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount), + + TP_ARGS(domain, iova, pgsize, pgcount), + + TP_STRUCT__entry( + __string(group_name, dev_name(domain->dev)) + __field(unsigned long, iova) + __field(size_t, pgsize) + __field(size_t, pgcount) + ), + + TP_fast_assign( + __assign_str(group_name, dev_name(domain->dev)); + __entry->iova = iova; + __entry->pgsize = pgsize; + __entry->pgcount = pgcount; + ), + + TP_printk("group=%s iova=%lx size=%zx pgsize=%zx pgcount=%zx", + __get_str(group_name), __entry->iova, + __entry->pgsize * __entry->pgcount, + __entry->pgsize, __entry->pgcount + ) +); + +DEFINE_EVENT(iommu_map_pages, map_pages, + + TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount), + + TP_ARGS(domain, iova, pgsize, pgcount) +); + +DEFINE_EVENT(iommu_map_pages, unmap_pages, + + TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount), + + TP_ARGS(domain, iova, pgsize, pgcount) +); + +/* Refer to samples/ftrace_events */ +#ifndef __TRACE_EVENT_ARM_SMMU_HELPER_FUNCTIONS +#define __TRACE_EVENT_ARM_SMMU_HELPER_FUNCTIONS +static inline unsigned long sum_scatterlist_length(struct scatterlist *sgl, + unsigned int nents) +{ + int i = 0; + unsigned long sum = 0; + + for (i = 0; i < nents; i++, sgl = sg_next(sgl)) + sum += sgl->length; + + return sum; +} +#endif + +TRACE_EVENT(map_sg, + + TP_PROTO(struct arm_smmu_domain *domain, unsigned long iova, + struct scatterlist *sgl, unsigned int nents), + + TP_ARGS(domain, iova, sgl, nents), + + TP_STRUCT__entry( + __string(group_name, dev_name(domain->dev)) + __field(unsigned long, iova) + __field(unsigned long, size) + ), + + TP_fast_assign( + __assign_str(group_name, dev_name(domain->dev)); + __entry->iova = iova; + __entry->size = sum_scatterlist_length(sgl, nents); + ), + + TP_printk("group=%s iova=%lx size=%lx", + __get_str(group_name), __entry->iova, + __entry->size + ) +); + +TRACE_EVENT(tlbsync_timeout, + + TP_PROTO(struct device *dev), + + TP_ARGS(dev), + + TP_STRUCT__entry( + __string(device, dev_name(dev)) + ), + + TP_fast_assign( + __assign_str(device, dev_name(dev)); + ), + + TP_printk("smmu=%s", + __get_str(device) + ) +); + +TRACE_EVENT(smmu_init, + + TP_PROTO(u64 time), + + TP_ARGS(time), + + TP_STRUCT__entry( + __field(u64, time) + ), + + TP_fast_assign( + __entry->time = time; + ), + + TP_printk("ARM SMMU init latency: %lld us", __entry->time) +); +#endif /* _TRACE_ARM_SMMU_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../drivers/iommu/arm/arm-smmu + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE arm-smmu-trace + +/* This part must be outside protection */ +#include diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 004be15302ce..24a3887a5fd4 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -13,6 +13,8 @@ * - Non-secure access to the SMMU * - Context fault reporting * - Extended Stream ID (16 bit) + * + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #define pr_fmt(fmt) "arm-smmu: " fmt @@ -23,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -36,11 +40,22 @@ #include #include #include +#include +#include +#include #include #include #include "arm-smmu.h" +#include "../../iommu-logger.h" +#include "../../qcom-dma-iommu-generic.h" +#include "../../qcom-io-pgtable.h" +#include "../../qcom-io-pgtable-alloc.h" +#include + +#define CREATE_TRACE_POINTS +#include "arm-smmu-trace.h" /* * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU @@ -70,6 +85,30 @@ MODULE_PARM_DESC(disable_bypass, static bool using_legacy_binding, using_generic_binding; +struct arm_smmu_option_prop { + u32 opt; + const char *prop; +}; + +static struct arm_smmu_option_prop arm_smmu_options[] = { + { ARM_SMMU_OPT_FATAL_ASF, "qcom,fatal-asf" }, + { ARM_SMMU_OPT_3LVL_TABLES, "qcom,use-3-lvl-tables" }, + { ARM_SMMU_OPT_NO_ASID_RETENTION, "qcom,no-asid-retention" }, + { ARM_SMMU_OPT_DISABLE_ATOS, "qcom,disable-atos" }, + { ARM_SMMU_OPT_CONTEXT_FAULT_RETRY, "qcom,context-fault-retry" }, + { 0, NULL}, +}; + +static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova); +static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, + struct qcom_iommu_atos_txn *txn); +static void arm_smmu_destroy_domain_context(struct iommu_domain *domain); + +static int arm_smmu_setup_default_domain(struct device *dev, + struct iommu_domain *domain); +static void __arm_smmu_qcom_tlb_sync(struct iommu_domain *domain); + static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) { if (pm_runtime_enabled(smmu->dev)) @@ -80,8 +119,10 @@ static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu) { - if (pm_runtime_enabled(smmu->dev)) + if (pm_runtime_enabled(smmu->dev)) { + pm_runtime_mark_last_busy(smmu->dev); pm_runtime_put_autosuspend(smmu->dev); + } } static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) @@ -89,8 +130,160 @@ static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) return container_of(dom, struct arm_smmu_domain, domain); } +static void parse_driver_options(struct arm_smmu_device *smmu) +{ + int i = 0; + + do { + if (of_property_read_bool(smmu->dev->of_node, + arm_smmu_options[i].prop)) { + smmu->options |= arm_smmu_options[i].opt; + dev_dbg(smmu->dev, "option %s\n", + arm_smmu_options[i].prop); + } + } while (arm_smmu_options[++i].opt); +} + +static bool is_iommu_pt_coherent(struct arm_smmu_domain *smmu_domain) +{ + if (smmu_domain->force_coherent_walk) + return true; + else if (smmu_domain->smmu && smmu_domain->smmu->dev) + return dev_is_dma_coherent(smmu_domain->smmu->dev); + return false; +} + +static bool arm_smmu_has_secure_vmid(struct arm_smmu_domain *smmu_domain) +{ + return (smmu_domain->secure_vmid != VMID_INVAL); +} + +#ifdef CONFIG_ARM_SMMU_SELFTEST + +static int selftest; +module_param_named(selftest, selftest, int, 0644); +static int irq_count; + +struct arm_smmu_cf_selftest_data { + struct arm_smmu_device *smmu; + int cbndx; +}; + +static DECLARE_WAIT_QUEUE_HEAD(wait_int); +static irqreturn_t arm_smmu_cf_selftest(int irq, void *data) +{ + u32 fsr; + struct irq_data *irq_data = irq_get_irq_data(irq); + struct arm_smmu_cf_selftest_data *cb_data = data; + struct arm_smmu_device *smmu = cb_data->smmu; + int idx = cb_data->cbndx; + unsigned long hwirq = ULONG_MAX; + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + + irq_count++; + if (irq_data) + hwirq = irq_data->hwirq; + pr_info("Interrupt (irq:%d hwirq:%ld) received, fsr:0x%x\n", + irq, hwirq, fsr); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + + wake_up(&wait_int); + return IRQ_HANDLED; +} + +static void arm_smmu_interrupt_selftest(struct arm_smmu_device *smmu) +{ + int cb; + int cb_count = 0; + struct arm_smmu_cf_selftest_data *cb_data; + + if (!selftest) + return; + + cb = smmu->num_s2_context_banks; + + if (smmu->version < ARM_SMMU_V2) + return; + + cb_data = kmalloc(sizeof(*cb_data), GFP_KERNEL); + if (!cb_data) + return; + cb_data->smmu = smmu; + + for_each_clear_bit_from(cb, smmu->context_map, + smmu->num_context_banks) { + int irq; + int ret; + u32 reg; + u32 reg_orig; + int irq_cnt; + + irq = smmu->irqs[smmu->num_global_irqs + cb]; + cb_data->cbndx = cb; + + ret = request_threaded_irq(irq, NULL, arm_smmu_cf_selftest, + IRQF_ONESHOT | IRQF_SHARED, + "arm-smmu-context-fault", cb_data); + if (ret < 0) { + dev_err(smmu->dev, + "Failed to request cntx IRQ %d (%u)\n", + cb, irq); + continue; + } + + cb_count++; + irq_cnt = irq_count; + + reg_orig = arm_smmu_cb_read(smmu, cb, ARM_SMMU_CB_SCTLR); + reg = reg_orig | ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE; + + arm_smmu_cb_write(smmu, cb, ARM_SMMU_CB_SCTLR, reg); + dev_info(smmu->dev, "Testing cntx %d irq %d\n", cb, irq); + + /* Make sure ARM_SMMU_CB_SCTLR is configured */ + wmb(); + arm_smmu_cb_write(smmu, cb, ARM_SMMU_CB_FSRRESTORE, + ARM_SMMU_FSR_TF); + + if (!wait_event_timeout(wait_int, (irq_count > irq_cnt), + msecs_to_jiffies(1000))) { + u32 fsr; + + fsr = arm_smmu_cb_read(smmu, cb, ARM_SMMU_CB_FSR); + dev_info(smmu->dev, "timeout cb:%d, irq:%d, fsr:0x%x\n", + cb, irq_cnt, fsr); + + if (!fsr) + dev_err(smmu->dev, "SCTLR = 0x%08x\n", + arm_smmu_cb_read(smmu, cb, + ARM_SMMU_CB_SCTLR)); + } + + /* Make sure ARM_SMMU_CB_FSRRESTORE is written to */ + wmb(); + arm_smmu_cb_write(smmu, cb, ARM_SMMU_CB_SCTLR, reg_orig); + free_irq(irq, cb_data); + } + + kfree(cb_data); + dev_info(smmu->dev, + "Interrupt selftest completed...\n"); + dev_info(smmu->dev, + "Tested %d contexts, received %d interrupts\n", + cb_count, irq_count); + WARN_ON(cb_count != irq_count); + irq_count = 0; +} +#else +static void arm_smmu_interrupt_selftest(struct arm_smmu_device *smmu) +{ +} +#endif + static struct platform_driver arm_smmu_driver; -static struct iommu_ops arm_smmu_ops; +static struct qcom_iommu_ops arm_smmu_ops; #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS static int arm_smmu_bus_init(struct iommu_ops *ops); @@ -166,7 +359,7 @@ static int arm_smmu_register_legacy_master(struct device *dev, } err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode, - &arm_smmu_ops); + &arm_smmu_ops.iommu_ops); if (err) return err; @@ -190,7 +383,7 @@ static int arm_smmu_register_legacy_master(struct device *dev, static int arm_smmu_legacy_bus_init(void) { if (using_legacy_binding) - return arm_smmu_bus_init(&arm_smmu_ops); + return arm_smmu_bus_init(&arm_smmu_ops.iommu_ops); return 0; } device_initcall_sync(arm_smmu_legacy_bus_init); @@ -202,33 +395,68 @@ static int arm_smmu_register_legacy_master(struct device *dev, } #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */ +static int __arm_smmu_alloc_cb(unsigned long *map, int start, int end, + struct device *dev, + struct arm_smmu_domain *smmu_domain) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); + struct arm_smmu_device *smmu = cfg->smmu; + int idx; + int i; + + for_each_cfg_sme(cfg, fwspec, i, idx) { + if (smmu->s2crs[idx].pinned) + return smmu->s2crs[idx].cbndx; + } + + return __arm_smmu_alloc_bitmap(map, start, end); +} + static void __arm_smmu_free_bitmap(unsigned long *map, int idx) { clear_bit(idx, map); } /* Wait for any pending TLB invalidations to complete */ -static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, +static int __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, int sync, int status) { - unsigned int spin_cnt, delay; + unsigned int inc, delay; u32 reg; - if (smmu->impl && unlikely(smmu->impl->tlb_sync)) - return smmu->impl->tlb_sync(smmu, page, sync, status); + /* + * Allowing an unbounded number of sync requests to be submitted when a + * TBU is not processing sync requests can cause a TBU's command queue + * to fill up. Once the queue is full, subsequent sync requests can + * stall the CPU indefinitely. Avoid this by gating subsequent sync + * requests after the first sync timeout on an SMMU. + */ + if (IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG) && + test_bit(0, &smmu->sync_timed_out)) + return -EINVAL; arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL); - for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { - for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { - reg = arm_smmu_readl(smmu, page, status); - if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) - return; - cpu_relax(); - } - udelay(delay); + for (delay = 1, inc = 1; delay < TLB_LOOP_TIMEOUT; delay += inc) { + reg = arm_smmu_readl(smmu, page, status); + if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) + return 0; + + cpu_relax(); + udelay(inc); + if (inc < TLB_LOOP_INC_MAX) + inc *= 2; } - dev_err_ratelimited(smmu->dev, - "TLB sync timed out -- SMMU may be deadlocked\n"); + + if (IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG) && + test_and_set_bit_lock(0, &smmu->sync_timed_out)) + goto out; + + trace_tlbsync_timeout(smmu->dev); + if (smmu->impl && smmu->impl->tlb_sync_timeout) + smmu->impl->tlb_sync_timeout(smmu); +out: + return -EINVAL; } static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) @@ -236,8 +464,10 @@ static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) unsigned long flags; spin_lock_irqsave(&smmu->global_sync_lock, flags); - __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC, - ARM_SMMU_GR0_sTLBGSTATUS); + if (__arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC, + ARM_SMMU_GR0_sTLBGSTATUS)) + dev_err_ratelimited(smmu->dev, + "TLB global sync failed!\n"); spin_unlock_irqrestore(&smmu->global_sync_lock, flags); } @@ -246,10 +476,14 @@ static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain) struct arm_smmu_device *smmu = smmu_domain->smmu; unsigned long flags; - spin_lock_irqsave(&smmu_domain->cb_lock, flags); - __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx), - ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS); - spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); + spin_lock_irqsave(&smmu_domain->sync_lock, flags); + if (__arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx), + ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS)) + dev_err_ratelimited(smmu->dev, + "TLB sync on cb%d failed for device %s\n", + smmu_domain->cfg.cbndx, + dev_name(smmu_domain->dev)); + spin_unlock_irqrestore(&smmu_domain->sync_lock, flags); } static void arm_smmu_tlb_inv_context_s1(void *cookie) @@ -260,9 +494,11 @@ static void arm_smmu_tlb_inv_context_s1(void *cookie) * current CPU are visible beforehand. */ wmb(); + trace_tlbi_start(smmu_domain); arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx, ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid); arm_smmu_tlb_sync_context(smmu_domain); + trace_tlbi_end(smmu_domain); } static void arm_smmu_tlb_inv_context_s2(void *cookie) @@ -406,34 +642,343 @@ static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = { .tlb_add_page = arm_smmu_tlb_add_page_s2_v1, }; -static irqreturn_t arm_smmu_context_fault(int irq, void *dev) +static void print_fault_regs(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_device *smmu, int idx) { - u32 fsr, fsynr, cbfrsynra; + u32 fsr, fsynr0, fsynr1, cbfrsynra; unsigned long iova; - struct iommu_domain *domain = dev; - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_device *smmu = smmu_domain->smmu; - int idx = smmu_domain->cfg.cbndx; - int ret; + struct arm_smmu_cfg *cfg = smmu->cbs[idx].cfg; + bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); - if (!(fsr & ARM_SMMU_FSR_FAULT)) - return IRQ_NONE; + fsynr0 = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); + fsynr1 = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR1); + iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); + cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); + + dev_err(smmu->dev, "Unhandled arm-smmu context fault from %s!\n", + dev_name(smmu_domain->dev)); + dev_err(smmu->dev, "FAR = 0x%016lx\n", + iova); + dev_err(smmu->dev, "PAR = 0x%pK\n", + (void *) arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR)); + + dev_err(smmu->dev, + "FSR = 0x%08x [%s%s%s%s%s%s%s%s%s%s]\n", + fsr, + (fsr & ARM_SMMU_FSR_TF) ? (fsynr0 & ARM_SMMU_FSYNR0_WNR ? + "TF W " : "TF R ") : "", + (fsr & ARM_SMMU_FSR_AFF) ? "AFF " : "", + (fsr & ARM_SMMU_FSR_PF) ? (fsynr0 & ARM_SMMU_FSYNR0_WNR ? + "PF W " : "PF R ") : "", + (fsr & ARM_SMMU_FSR_EF) ? "EF " : "", + (fsr & ARM_SMMU_FSR_TLBMCF) ? "TLBMCF " : "", + (fsr & ARM_SMMU_FSR_TLBLKF) ? "TLBLKF " : "", + (fsr & ARM_SMMU_FSR_ASF) ? "ASF " : "", + (fsr & ARM_SMMU_FSR_UUT) ? "UUT " : "", + (fsr & ARM_SMMU_FSR_SS) ? "SS " : "", + (fsr & ARM_SMMU_FSR_MULTI) ? "MULTI " : ""); + + dev_err(smmu->dev, "FSYNR0 = 0x%x\n", fsynr0); + dev_err(smmu->dev, "FSYNR1 = 0x%x\n", fsynr1); + dev_err(smmu->dev, "context bank# = 0x%x\n", idx); + + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { + dev_err(smmu->dev, "TTBR0 = 0x%pK\n", + (void *) (unsigned long) + arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_TTBR0)); + dev_err(smmu->dev, "TTBR1 = 0x%pK\n", + (void *) (unsigned long) + arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_TTBR1)); + } else { + dev_err(smmu->dev, "TTBR0 = 0x%pK\n", + (void *) arm_smmu_cb_readq(smmu, idx, + ARM_SMMU_CB_TTBR0)); + if (stage1) + dev_err(smmu->dev, "TTBR1 = 0x%pK\n", + (void *) arm_smmu_cb_readq(smmu, idx, + ARM_SMMU_CB_TTBR1)); + } + + dev_err(smmu->dev, "SCTLR = 0x%08x ACTLR = 0x%08x\n", + arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR), + arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_ACTLR)); + dev_err(smmu->dev, "CBAR = 0x%08x\n", + arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBAR(idx))); + dev_err(smmu->dev, "MAIR0 = 0x%08x MAIR1 = 0x%08x\n", + arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_S1_MAIR0), + arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_S1_MAIR1)); + + dev_err(smmu->dev, "SID = 0x%x\n", + cbfrsynra & CBFRSYNRA_SID_MASK); + dev_err(smmu->dev, "Client info: BID=0x%lx, PID=0x%lx, MID=0x%lx\n", + FIELD_GET(ARM_SMMU_FSYNR1_BID, fsynr1), + FIELD_GET(ARM_SMMU_FSYNR1_PID, fsynr1), + FIELD_GET(ARM_SMMU_FSYNR1_MID, fsynr1)); +} + +/* + * Iommu HW has generated a fault. If HW and SW states are in sync, + * then a SW page table walk should yield 0. + * + * WARNING!!! This check is racy!!!! + * For a faulting address x, the dma layer mapping may map address x + * into the iommu page table in parallel to the fault handler running. + * This is frequently seen due to dma-iommu's address reuse policy. + * Thus, arm_smmu_iova_to_phys() returning nonzero is not necessarily + * indicative of an issue. + */ +static void arm_smmu_verify_fault(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_device *smmu, int idx) +{ + u32 fsynr, cbfrsynra; + unsigned long iova; + struct iommu_domain *domain = &smmu_domain->domain; + phys_addr_t phys_soft; + phys_addr_t phys_stimu, phys_hard_priv, phys_stimu_post_tlbiall; + unsigned long flags = 0; + struct qcom_iommu_atos_txn txn = {0}; fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); - ret = report_iommu_fault(domain, NULL, iova, - fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); + phys_soft = arm_smmu_iova_to_phys(domain, iova); + dev_err(smmu->dev, "soft iova-to-phys=%pa\n", &phys_soft); - if (ret == -ENOSYS) - dev_err_ratelimited(smmu->dev, - "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", - fsr, iova, fsynr, cbfrsynra, idx); + /* Get the transaction type */ + if (fsynr & ARM_SMMU_FSYNR0_WNR) + flags |= IOMMU_TRANS_WRITE; + if (fsynr & ARM_SMMU_FSYNR0_PNU) + flags |= IOMMU_TRANS_PRIV; + if (fsynr & ARM_SMMU_FSYNR0_IND) + flags |= IOMMU_TRANS_INST; - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); - return IRQ_HANDLED; + txn.addr = iova; + txn.flags = flags; + txn.id = cbfrsynra & CBFRSYNRA_SID_MASK; + + /* Now replicate the faulty transaction */ + phys_stimu = arm_smmu_iova_to_phys_hard(domain, &txn); + + /* + * If the replicated transaction fails, it could be due to legitimate + * unmapped access (translation fault) or stale TLB with insufficient + * privileges (permission fault). Try ATOS operation with full access + * privileges to rule out stale entry with insufficient privileges case. + */ + if (!phys_stimu) { + txn.flags = QCOM_IOMMU_ATOS_TRANS_DEFAULT | + QCOM_IOMMU_ATOS_TRANS_PRIV; + phys_hard_priv = arm_smmu_iova_to_phys_hard(domain, &txn); + } + + /* Now replicate the faulty transaction post tlbiall */ + iommu_flush_iotlb_all(domain); + txn.flags = flags; + phys_stimu_post_tlbiall = arm_smmu_iova_to_phys_hard(domain, &txn); + + if (!phys_stimu && phys_hard_priv) { + dev_err(smmu->dev, + "ATOS results differed across access privileges...\n" + "Before: %pa After: %pa\n", + &phys_stimu, &phys_hard_priv); + } + + if (phys_stimu != phys_stimu_post_tlbiall) { + dev_err(smmu->dev, + "ATOS results differed across TLBIALL...\n" + "Before: %pa After: %pa\n", &phys_stimu, + &phys_stimu_post_tlbiall); + } + + dev_err(smmu->dev, "hard iova-to-phys (ATOS)=%pa\n", + phys_stimu ? &phys_stimu : &phys_stimu_post_tlbiall); +} + +static int report_iommu_fault_helper(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_device *smmu, int idx) +{ + u32 fsr, fsynr; + unsigned long iova; + int flags; + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); + iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); + + flags = fsynr & ARM_SMMU_FSYNR0_WNR ? + IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; + if (fsr & ARM_SMMU_FSR_TF) + flags |= IOMMU_FAULT_TRANSLATION; + if (fsr & ARM_SMMU_FSR_PF) + flags |= IOMMU_FAULT_PERMISSION; + if (fsr & ARM_SMMU_FSR_EF) + flags |= IOMMU_FAULT_EXTERNAL; + if (fsr & ARM_SMMU_FSR_SS) + flags |= IOMMU_FAULT_TRANSACTION_STALLED; + + + return report_iommu_fault(&smmu_domain->domain, + smmu->dev, iova, flags); +} + +static int arm_smmu_get_fault_ids(struct iommu_domain *domain, + struct qcom_iommu_fault_ids *f_ids) +{ + struct arm_smmu_domain *smmu_domain; + struct arm_smmu_device *smmu; + u32 fsr, fsynr1; + int idx, ret; + + if (!domain || !f_ids) + return -EINVAL; + + smmu_domain = to_smmu_domain(domain); + smmu = smmu_domain->smmu; + idx = smmu_domain->cfg.cbndx; + + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + return ret; + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + + if (!(fsr & ARM_SMMU_FSR_FAULT)) { + arm_smmu_rpm_put(smmu); + return -EINVAL; + } + + fsynr1 = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR1); + arm_smmu_rpm_put(smmu); + + f_ids->bid = FIELD_GET(ARM_SMMU_FSYNR1_BID, fsynr1); + f_ids->pid = FIELD_GET(ARM_SMMU_FSYNR1_PID, fsynr1); + f_ids->mid = FIELD_GET(ARM_SMMU_FSYNR1_MID, fsynr1); + + return 0; +} + +#ifdef CONFIG_ARM_SMMU_CONTEXT_FAULT_RETRY +/* + * Retry faulting address after tlb invalidate. + * Applicable to: Waipio + */ +static irqreturn_t arm_smmu_context_fault_retry(struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + int idx = smmu_domain->cfg.cbndx; + u64 iova; + u32 fsr; + + if (!(smmu->options & ARM_SMMU_OPT_CONTEXT_FAULT_RETRY) || + (smmu_domain->fault_model.no_stall)) + return IRQ_NONE; + + iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + + if (iova != smmu_domain->prev_fault_address || + !smmu_domain->fault_retry_counter) { + smmu_domain->prev_fault_address = iova; + smmu_domain->fault_retry_counter++; + arm_smmu_tlb_inv_context_s1(smmu_domain); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation + */ + wmb(); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + ARM_SMMU_RESUME_RESUME); + return IRQ_HANDLED; + } + + return IRQ_NONE; +} +#else +static irqreturn_t arm_smmu_context_fault_retry(struct arm_smmu_domain *smmu_domain) +{ + return IRQ_NONE; +} +#endif + +static irqreturn_t arm_smmu_context_fault(int irq, void *dev) +{ + u32 fsr; + struct iommu_domain *domain = dev; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + int idx = smmu_domain->cfg.cbndx; + static DEFINE_RATELIMIT_STATE(_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + int ret; + + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + return IRQ_NONE; + + fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); + if (!(fsr & ARM_SMMU_FSR_FAULT)) { + ret = IRQ_NONE; + goto out_power_off; + } + + if ((smmu->options & ARM_SMMU_OPT_FATAL_ASF) && + (fsr & ARM_SMMU_FSR_ASF)) { + dev_err(smmu->dev, + "Took an address size fault. Refusing to recover.\n"); + BUG(); + } + + ret = arm_smmu_context_fault_retry(smmu_domain); + if (ret == IRQ_HANDLED) + goto out_power_off; + + /* + * If the fault helper returns -ENOSYS, then no client fault helper was + * registered. In that case, print the default report. + * + * If the client returns -EBUSY, do not clear FSR and do not RESUME + * if stalled. This is required to keep the IOMMU client stalled on + * the outstanding fault. This gives the client a chance to take any + * debug action and then terminate the stalled transaction. + * So, the sequence in case of stall on fault should be: + * 1) Do not clear FSR or write to RESUME here + * 2) Client takes any debug action + * 3) Client terminates the stalled transaction and resumes the IOMMU + * 4) Client clears FSR. The FSR should only be cleared after 3) and + * not before so that the fault remains outstanding. This ensures + * SCTLR.HUPCF has the desired effect if subsequent transactions also + * need to be terminated. + */ + ret = report_iommu_fault_helper(smmu_domain, smmu, idx); + if (ret == -ENOSYS) { + if (__ratelimit(&_rs)) { + print_fault_regs(smmu_domain, smmu, idx); + arm_smmu_verify_fault(smmu_domain, smmu, idx); + } + BUG_ON(!smmu_domain->fault_model.non_fatal); + } + if (ret != -EBUSY) { + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); + + /* + * Barrier required to ensure that the FSR is cleared + * before resuming SMMU operation + */ + wmb(); + + if (fsr & ARM_SMMU_FSR_SS) + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, + ARM_SMMU_RESUME_TERMINATE); + } + + ret = IRQ_HANDLED; +out_power_off: + arm_smmu_rpm_put(smmu); + return ret; } static irqreturn_t arm_smmu_global_fault(int irq, void *dev) @@ -442,14 +987,21 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) struct arm_smmu_device *smmu = dev; static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); + int ret; + + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + return IRQ_NONE; gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0); gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1); gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2); - if (!gfsr) + if (!gfsr) { + arm_smmu_rpm_put(smmu); return IRQ_NONE; + } if (__ratelimit(&rs)) { if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) && @@ -466,6 +1018,7 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) } arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr); + arm_smmu_rpm_put(smmu); return IRQ_HANDLED; } @@ -524,6 +1077,33 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32; } } + + memset(&cfg->sctlr, 0, sizeof(cfg->sctlr)); + /* + * Override cacheability, shareability, r/w allocation for + * clients who are io-coherent. Otherwise set NSH to force io-coherency + * to be disabled. + * These settings only take effect during bypass mode, when sctlr.M = 0 + */ + if (of_dma_is_coherent(smmu_domain->dev->of_node)) { + cfg->sctlr.wacfg = ARM_SMMU_SCTLR_WACFG_WA; + cfg->sctlr.racfg = ARM_SMMU_SCTLR_RACFG_RA; + cfg->sctlr.shcfg = ARM_SMMU_SCTLR_SHCFG_OSH; + cfg->sctlr.mtcfg = 1; + cfg->sctlr.memattr = ARM_SMMU_SCTLR_MEM_ATTR_OISH_WB_CACHE; + } else { + cfg->sctlr.shcfg = ARM_SMMU_SCTLR_SHCFG_NSH; + } + + cfg->sctlr.cfre = !smmu_domain->fault_model.no_cfre; + cfg->sctlr.cfcfg = !smmu_domain->fault_model.no_stall; + cfg->sctlr.hupcf = smmu_domain->fault_model.hupcf; + + if ((!smmu_domain->mapping_cfg.s1_bypass && !smmu_domain->delayed_s1_trans_enable) || + !stage1) + cfg->sctlr.m = 1; + + cb->sctlr = arm_smmu_lpae_sctlr(cfg); } void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) @@ -602,19 +1182,87 @@ void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) } /* SCTLR */ - reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE | - ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M; - if (stage1) - reg |= ARM_SMMU_SCTLR_S1_ASIDPNE; - if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) - reg |= ARM_SMMU_SCTLR_E; - if (smmu->impl && smmu->impl->write_sctlr) - smmu->impl->write_sctlr(smmu, idx, reg); + smmu->impl->write_sctlr(smmu, idx, cb->sctlr); else - arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg); + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, cb->sctlr); } +/* This function assumes that the domain's init mutex is held */ +static int arm_smmu_get_dma_cookie(struct device *dev, + struct arm_smmu_domain *smmu_domain, + struct io_pgtable_ops *pgtbl_ops) +{ + bool fast = smmu_domain->mapping_cfg.fast; + + /* DMA cookie is allocated by the IOMMU core for DMA domains. */ + return fast ? fast_smmu_init_mapping(dev, &smmu_domain->domain, pgtbl_ops) : 0; +} + +static void arm_smmu_put_dma_cookie(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + /* DMA cookie is freed by the IOMMU core for DMA domains. */ + if (smmu_domain->mapping_cfg.fast) + fast_smmu_put_dma_cookie(domain); +} + +static void arm_smmu_log_new_table(void *cookie, void *virt, unsigned long iova, size_t granule) +{ + struct arm_smmu_domain *smmu_domain = cookie; + + trace_iommu_pgtable_add(smmu_domain, iova, __pa(virt), granule); +} + +static void arm_smmu_log_remove_table(void *cookie, void *virt, unsigned long iova, size_t granule) +{ + struct arm_smmu_domain *smmu_domain = cookie; + + trace_iommu_pgtable_remove(smmu_domain, iova, __pa(virt), granule); +} + +static void arm_smmu_tlb_add_walk_page(void *cookie, void *virt) +{ + struct arm_smmu_domain *smmu_domain = cookie; + struct page *page = virt_to_page(virt); + unsigned long flags; + + spin_lock_irqsave(&smmu_domain->iotlb_gather_lock, flags); + smmu_domain->deferred_sync = true; + page->freelist = smmu_domain->freelist; + smmu_domain->freelist = page; + spin_unlock_irqrestore(&smmu_domain->iotlb_gather_lock, flags); +} + +static void arm_smmu_qcom_tlb_add_inv(void *cookie) +{ + struct arm_smmu_domain *smmu_domain = cookie; + unsigned long flags; + + spin_lock_irqsave(&smmu_domain->iotlb_gather_lock, flags); + smmu_domain->deferred_sync = true; + spin_unlock_irqrestore(&smmu_domain->iotlb_gather_lock, flags); +} + +static void arm_smmu_qcom_tlb_sync(void *cookie) +{ + struct arm_smmu_domain *smmu_domain = cookie; + + __arm_smmu_qcom_tlb_sync(&smmu_domain->domain); +} + +static const struct qcom_iommu_pgtable_log_ops arm_smmu_pgtable_log_ops = { + .log_new_table = arm_smmu_log_new_table, + .log_remove_table = arm_smmu_log_remove_table, +}; + +static const struct qcom_iommu_flush_ops arm_smmu_iotlb_ops = { + .tlb_add_walk_page = arm_smmu_tlb_add_walk_page, + .tlb_add_inv = arm_smmu_qcom_tlb_add_inv, + .tlb_sync = arm_smmu_qcom_tlb_sync, +}; + static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain, struct arm_smmu_device *smmu, struct device *dev, unsigned int start) @@ -622,7 +1270,8 @@ static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain, if (smmu->impl && smmu->impl->alloc_context_bank) return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start); - return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); + return __arm_smmu_alloc_cb(smmu->context_map, start, smmu->num_context_banks, dev, + smmu_domain); } static int arm_smmu_init_domain_context(struct iommu_domain *domain, @@ -632,16 +1281,26 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, int irq, start, ret = 0; unsigned long ias, oas; struct io_pgtable_ops *pgtbl_ops; - struct io_pgtable_cfg pgtbl_cfg; + struct qcom_io_pgtable_info pgtbl_info = {}; + struct io_pgtable_cfg *pgtbl_cfg = &pgtbl_info.cfg; enum io_pgtable_fmt fmt; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_cfg *cfg = &smmu_domain->cfg; irqreturn_t (*context_fault)(int irq, void *dev); + struct io_pgtable *iop; mutex_lock(&smmu_domain->init_mutex); if (smmu_domain->smmu) goto out_unlock; + smmu_domain->dev = dev; + ret = arm_smmu_setup_default_domain(dev, domain); + if (ret) { + dev_err(dev, "%s: default domain setup failed\n", + __func__); + goto out_unlock; + } + if (domain->type == IOMMU_DOMAIN_IDENTITY) { smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS; smmu_domain->smmu = smmu; @@ -704,7 +1363,9 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, ias = smmu->va_size; oas = smmu->ipa_size; if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { - fmt = ARM_64_LPAE_S1; + fmt = QCOM_ARM_64_LPAE_S1; + if (smmu->options & ARM_SMMU_OPT_3LVL_TABLES) + ias = min(ias, 39UL); } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { fmt = ARM_32_LPAE_S1; ias = min(ias, 32UL); @@ -743,6 +1404,17 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, goto out_unlock; } + if (smmu_domain->mapping_cfg.fast) { + fmt = ARM_V8L_FAST; + ret = qcom_iommu_get_fast_iova_range(dev, + &pgtbl_info.iova_base, + &pgtbl_info.iova_end); + if (ret < 0) + goto out_unlock; + } else if (arm_smmu_has_secure_vmid(smmu_domain)) { + pgtbl_info.vmid = smmu_domain->secure_vmid; + } + ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start); if (ret < 0) { goto out_unlock; @@ -763,34 +1435,44 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, else cfg->asid = cfg->cbndx; - pgtbl_cfg = (struct io_pgtable_cfg) { + pgtbl_info.iommu_tlb_ops = &arm_smmu_iotlb_ops; + pgtbl_info.pgtable_log_ops = &arm_smmu_pgtable_log_ops; + pgtbl_info.cfg = (struct io_pgtable_cfg) { .pgsize_bitmap = smmu->pgsize_bitmap, .ias = ias, .oas = oas, - .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK, + .coherent_walk = is_iommu_pt_coherent(smmu_domain), .tlb = smmu_domain->flush_ops, .iommu_dev = smmu->dev, }; if (smmu->impl && smmu->impl->init_context) { - ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev); + ret = smmu->impl->init_context(smmu_domain, pgtbl_cfg, dev); if (ret) goto out_clear_smmu; } if (smmu_domain->pgtbl_quirks) - pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks; + pgtbl_cfg->quirks |= smmu_domain->pgtbl_quirks; - pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); + pgtbl_ops = qcom_alloc_io_pgtable_ops(fmt, &pgtbl_info, smmu_domain); if (!pgtbl_ops) { ret = -ENOMEM; goto out_clear_smmu; } - /* Update the domain's page sizes to reflect the page table format */ - domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + iop = container_of(pgtbl_ops, struct io_pgtable, ops); + ret = iommu_logger_register(&smmu_domain->logger, domain, + smmu_domain->dev, iop); + if (ret) { + dev_err(dev, "Log registration failed\n"); + goto out_free_io_pgtable; + } - if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) { + /* Update the domain's page sizes to reflect the page table format */ + domain->pgsize_bitmap = pgtbl_cfg->pgsize_bitmap; + + if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) { domain->geometry.aperture_start = ~0UL << ias; domain->geometry.aperture_end = ~0UL; } else { @@ -799,8 +1481,23 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, domain->geometry.force_aperture = true; + ret = arm_smmu_get_dma_cookie(dev, smmu_domain, pgtbl_ops); + if (ret) + goto out_logger; + + /* + * Matches with call to arm_smmu_rpm_put in + * arm_smmu_destroy_domain_context. + */ + if (smmu_domain->mapping_cfg.atomic) { + smmu_domain->rpm_always_on = true; + arm_smmu_rpm_get(smmu); + } + /* Initialise the context bank with our page table cfg */ - arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); + arm_smmu_init_context_bank(smmu_domain, pgtbl_cfg); + if (smmu->impl && smmu->impl->init_context_bank) + smmu->impl->init_context_bank(smmu_domain, dev); arm_smmu_write_context_bank(smmu, cfg->cbndx); /* @@ -814,8 +1511,9 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, else context_fault = arm_smmu_context_fault; - ret = devm_request_irq(smmu->dev, irq, context_fault, - IRQF_SHARED, "arm-smmu-context-fault", domain); + ret = devm_request_threaded_irq(smmu->dev, irq, NULL, + context_fault, IRQF_ONESHOT | IRQF_SHARED, + "arm-smmu-context-fault", domain); if (ret < 0) { dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", cfg->irptndx, irq); @@ -828,6 +1526,11 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, smmu_domain->pgtbl_ops = pgtbl_ops; return 0; +out_logger: + iommu_logger_unregister(smmu_domain->logger); + smmu_domain->logger = NULL; +out_free_io_pgtable: + qcom_free_io_pgtable_ops(smmu_domain->pgtbl_ops); out_clear_smmu: __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); smmu_domain->smmu = NULL; @@ -841,7 +1544,8 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - int ret, irq; + int ret, irq, i; + bool pinned = false; if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY) return; @@ -850,6 +1554,13 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) if (ret < 0) return; + /* + * Matches with call to arm_smmu_rpm_get in + * arm_smmu_init_domain_contxt. + */ + if (smmu_domain->rpm_always_on) + arm_smmu_rpm_put(smmu); + /* * Disable the context bank and free the page tables before freeing * it. @@ -862,8 +1573,16 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) devm_free_irq(smmu->dev, irq, domain); } - free_io_pgtable_ops(smmu_domain->pgtbl_ops); - __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); + qcom_free_io_pgtable_ops(smmu_domain->pgtbl_ops); + + for (i = 0; i < smmu->num_mapping_groups; i++) + if ((cfg->cbndx == smmu->s2crs[i].cbndx) && + (smmu->s2crs[i].pinned)) { + pinned = true; + } + + if (!pinned) + __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); arm_smmu_rpm_put(smmu); } @@ -888,6 +1607,9 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) mutex_init(&smmu_domain->init_mutex); spin_lock_init(&smmu_domain->cb_lock); + spin_lock_init(&smmu_domain->sync_lock); + spin_lock_init(&smmu_domain->iotlb_gather_lock); + smmu_domain->secure_vmid = VMID_INVAL; return &smmu_domain->domain; } @@ -900,7 +1622,9 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) * Free the domain resources. We assume that all devices have * already been detached. */ + arm_smmu_put_dma_cookie(domain); arm_smmu_destroy_domain_context(domain); + iommu_logger_unregister(smmu_domain->logger); kfree(smmu_domain); } @@ -927,7 +1651,8 @@ static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) | FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) | - FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg); + FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg) | + FIELD_PREP(ARM_SMMU_S2CR_SHCFG, ARM_SMMU_S2CR_SHCFG_NSH); if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && smmu->smrs[idx].valid) @@ -1026,22 +1751,48 @@ static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) { + bool pinned = smmu->s2crs[idx].pinned; + u8 cbndx = smmu->s2crs[idx].cbndx; + if (--smmu->s2crs[idx].count) return false; smmu->s2crs[idx] = s2cr_init_val; - if (smmu->smrs) + if (pinned) { + smmu->s2crs[idx].pinned = true; + smmu->s2crs[idx].cbndx = cbndx; + } else if (smmu->smrs) { smmu->smrs[idx].valid = false; + } return true; } +static struct device_node *arm_smmu_get_of_node(struct device *dev) +{ + struct device_node *np; + + if (!dev->of_node) + return NULL; + + np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); + return np ? np : dev->of_node; +} + +static bool dev_defer_smr_configuration(struct device *dev) +{ + struct device_node *np = arm_smmu_get_of_node(dev); + + return of_property_read_bool(np, "qcom,iommu-defer-smr-config"); +} + static int arm_smmu_master_alloc_smes(struct device *dev) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); struct arm_smmu_device *smmu = cfg->smmu; struct arm_smmu_smr *smrs = smmu->smrs; + bool config_smrs = !dev_defer_smr_configuration(dev); int i, idx, ret; mutex_lock(&smmu->stream_map_mutex); @@ -1063,17 +1814,20 @@ static int arm_smmu_master_alloc_smes(struct device *dev) if (smrs && smmu->s2crs[idx].count == 0) { smrs[idx].id = sid; smrs[idx].mask = mask; - smrs[idx].valid = true; + smrs[idx].valid = config_smrs; + } else if (smrs && WARN_ON(smrs[idx].valid != config_smrs)) { + ret = -EINVAL; + goto out_err; } smmu->s2crs[idx].count++; cfg->smendx[i] = (s16)idx; } + mutex_unlock(&smmu->stream_map_mutex); /* It worked! Now, poke the actual hardware */ for_each_cfg_sme(cfg, fwspec, i, idx) arm_smmu_write_sme(smmu, idx); - mutex_unlock(&smmu->stream_map_mutex); return 0; out_err: @@ -1115,6 +1869,7 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, else type = S2CR_TYPE_TRANS; + mutex_lock(&smmu->stream_map_mutex); for_each_cfg_sme(cfg, fwspec, i, idx) { if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) continue; @@ -1124,9 +1879,146 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, s2cr[idx].cbndx = cbndx; arm_smmu_write_s2cr(smmu, idx); } + mutex_unlock(&smmu->stream_map_mutex); + return 0; } +static int arm_smmu_setup_default_domain(struct device *dev, + struct iommu_domain *domain) +{ + struct device_node *np; + int ret; + const char *str; + u32 val; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + np = arm_smmu_get_of_node(dev); + if (!np) + return 0; + + ret = of_property_read_string(np, "qcom,iommu-dma", &str); + if (ret) + str = "default"; + + if (!strcmp(str, "bypass")) { + smmu_domain->mapping_cfg.s1_bypass = 1; + } else if (!strcmp(str, "fastmap")) { + /* + * Fallback to the upstream dma-allocator if fastmap is not enabled. + * "fastmap" implies "atomic" due to it not calling arm_smmu_rpm_get() + * in its map/unmap functions. Its clients may or may not actually + * use iommu apis from atomic context. + */ + smmu_domain->mapping_cfg.atomic = 1; + if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_FAST)) + smmu_domain->mapping_cfg.fast = 1; + } else if (!strcmp(str, "atomic")) { + smmu_domain->mapping_cfg.atomic = 1; + } else if (!strcmp(str, "disabled")) { + /* DT properties only intended for use by default-domains */ + return 0; + } + + /* + * default value: + * Stall-on-fault + * faults trigger kernel panic + * return abort + */ + if (of_property_match_string(np, "qcom,iommu-faults", + "stall-disable") >= 0) + smmu_domain->fault_model.no_stall = 1; + + if (of_property_match_string(np, "qcom,iommu-faults", "no-CFRE") >= 0) + smmu_domain->fault_model.no_cfre = 1; + + if (of_property_match_string(np, "qcom,iommu-faults", "HUPCF") >= 0) + smmu_domain->fault_model.hupcf = 1; + + if (of_property_match_string(np, "qcom,iommu-faults", "non-fatal") >= 0) + smmu_domain->fault_model.non_fatal = 1; + + /* Default value: disabled */ + ret = of_property_read_u32(np, "qcom,iommu-vmid", &val); + if (!ret) + smmu_domain->secure_vmid = val; + + /* Default value: disabled */ + ret = of_property_read_string(np, "qcom,iommu-pagetable", &str); + if (ret) + str = "disabled"; + if (!strcmp(str, "coherent")) + smmu_domain->force_coherent_walk = true; + else if (!strcmp(str, "LLC")) + smmu_domain->pgtbl_quirks = IO_PGTABLE_QUIRK_ARM_OUTER_WBWA; + else if (!strcmp(str, "LLC_NWA")) + smmu_domain->pgtbl_quirks = IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA; + + /* Default value: disabled */ + if (of_property_read_bool(np, "qcom,iommu-earlymap")) + smmu_domain->delayed_s1_trans_enable = true; + + return 0; +} + +struct lookup_iommu_group_data { + struct device_node *np; + struct iommu_group *group; +}; + +/* This isn't a "fast lookup" since its N^2, but probably good enough */ +static int __bus_lookup_iommu_group(struct device *dev, void *priv) +{ + struct lookup_iommu_group_data *data = priv; + struct device_node *np; + struct iommu_group *group; + + group = iommu_group_get(dev); + if (!group) + return 0; + + np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); + if (np != data->np) { + iommu_group_put(group); + return 0; + } + + data->group = group; + return 1; +} + +static struct iommu_group *of_get_device_group(struct device *dev) +{ + struct lookup_iommu_group_data data = { + .np = NULL, + .group = NULL, + }; + struct iommu_group *group; + int ret; + + data.np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0); + if (!data.np) + return NULL; + + ret = bus_for_each_dev(&platform_bus_type, NULL, &data, + __bus_lookup_iommu_group); + if (ret > 0) + return data.group; + +#ifdef CONFIG_PCI + ret = bus_for_each_dev(&pci_bus_type, NULL, &data, + __bus_lookup_iommu_group); + if (ret > 0) + return data.group; +#endif + + group = generic_device_group(dev); + if (IS_ERR(group)) + return NULL; + return group; +} + static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); @@ -1135,7 +2027,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) struct arm_smmu_device *smmu; int ret; - if (!fwspec || fwspec->ops != &arm_smmu_ops) { + if (!fwspec || fwspec->ops != &arm_smmu_ops.iommu_ops) { dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); return -ENXIO; } @@ -1193,41 +2085,100 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) rpm_put: arm_smmu_rpm_put(smmu); + return ret; } +static gfp_t arm_smmu_domain_gfp_flags(struct arm_smmu_domain *smmu_domain) +{ + /* + * The dma layer always uses GFP_ATOMIC, which isn't indicative of + * the actual client needs. + */ + if (smmu_domain->mapping_cfg.atomic) + return GFP_ATOMIC; + + return GFP_KERNEL; +} + static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) { struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); int ret; if (!ops) return -ENODEV; - arm_smmu_rpm_get(smmu); + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + return ret; + + gfp = arm_smmu_domain_gfp_flags(smmu_domain); ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped); + arm_smmu_rpm_put(smmu); + if (!ret) + trace_map_pages(smmu_domain, iova, pgsize, pgcount); + + return ret; +} + +static int __maybe_unused arm_smmu_map_sg(struct iommu_domain *domain, unsigned long iova, + struct scatterlist *sg, unsigned int nents, int prot, + gfp_t gfp, size_t *mapped) +{ + struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; + struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int ret, i; + struct scatterlist *tmp; + + if (!ops) + return -ENODEV; + + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + return ret; + + gfp = arm_smmu_domain_gfp_flags(smmu_domain); + /* + * Use ops->map_sg() when it is available. While this function is unreachable, this + * has to be implemented as such to avoid the usage of ops->map_sg(), as it + * does not exist. + */ + for_each_sg(sg, tmp, nents, i) { + ret = ops->map(ops, iova + *mapped, sg_phys(tmp), tmp->length, prot, gfp); + if (ret) + break; + + *mapped += tmp->length; + } + + arm_smmu_rpm_put(smmu); + if (!ret) + trace_map_sg(smmu_domain, iova, sg, nents); return ret; } static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, size_t pgsize, size_t pgcount, - struct iommu_iotlb_gather *iotlb_gather) + struct iommu_iotlb_gather *gather) { - struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; - struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; size_t ret; if (!ops) return 0; - arm_smmu_rpm_get(smmu); - ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather); - arm_smmu_rpm_put(smmu); + ret = ops->unmap_pages(ops, iova, pgsize, pgcount, gather); + if (ret) + trace_unmap_pages(smmu_domain, iova, pgsize, pgcount); return ret; } @@ -1244,6 +2195,39 @@ static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) } } +/* Caller must call arm_smmu_rpm_get() beforehand. */ +static void __arm_smmu_qcom_tlb_sync(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct page *freelist, *page; + unsigned long flags; + + spin_lock_irqsave(&smmu_domain->iotlb_gather_lock, flags); + if (!smmu_domain->deferred_sync) { + spin_unlock_irqrestore(&smmu_domain->iotlb_gather_lock, flags); + return; + } + + if (smmu->version == ARM_SMMU_V2 || + smmu_domain->stage == ARM_SMMU_DOMAIN_S1) + arm_smmu_tlb_inv_context_s1(smmu_domain); + else + arm_smmu_tlb_sync_global(smmu); + + smmu_domain->deferred_sync = false; + + freelist = smmu_domain->freelist; + smmu_domain->freelist = NULL; + + while (freelist) { + page = freelist; + freelist = page->freelist; + qcom_io_pgtable_free_page(page); + } + spin_unlock_irqrestore(&smmu_domain->iotlb_gather_lock, flags); +} + static void arm_smmu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { @@ -1254,33 +2238,25 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain, return; arm_smmu_rpm_get(smmu); - if (smmu->version == ARM_SMMU_V2 || - smmu_domain->stage == ARM_SMMU_DOMAIN_S1) - arm_smmu_tlb_sync_context(smmu_domain); - else - arm_smmu_tlb_sync_global(smmu); + __arm_smmu_qcom_tlb_sync(domain); arm_smmu_rpm_put(smmu); } -static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, +static phys_addr_t __arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, dma_addr_t iova) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; struct device *dev = smmu->dev; void __iomem *reg; u32 tmp; u64 phys; unsigned long va, flags; - int ret, idx = cfg->cbndx; + int idx = cfg->cbndx; phys_addr_t addr = 0; - ret = arm_smmu_rpm_get(smmu); - if (ret < 0) - return 0; - spin_lock_irqsave(&smmu_domain->cb_lock, flags); va = iova & ~0xfffUL; if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) @@ -1295,7 +2271,6 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, dev_err(dev, "iova to phys timed out on %pad. Falling back to software table walk.\n", &iova); - arm_smmu_rpm_put(smmu); return ops->iova_to_phys(ops, iova); } @@ -1309,7 +2284,6 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); out: - arm_smmu_rpm_put(smmu); return addr; } @@ -1319,17 +2293,60 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + struct arm_smmu_device *smmu = smmu_domain->smmu; + phys_addr_t phys; if (!ops) return 0; if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && - smmu_domain->stage == ARM_SMMU_DOMAIN_S1) - return arm_smmu_iova_to_phys_hard(domain, iova); + smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + if (arm_smmu_rpm_get(smmu) < 0) + return 0; + + phys = __arm_smmu_iova_to_phys_hard(domain, iova); + + arm_smmu_rpm_put(smmu); + + return phys; + } return ops->iova_to_phys(ops, iova); } +/* + * This function can sleep, and cannot be called from atomic context. Will + * power on register block if required. This restriction does not apply to the + * original iova_to_phys() op. + */ +static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, + struct qcom_iommu_atos_txn *txn) +{ + phys_addr_t ret = 0; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + + if (smmu->options & ARM_SMMU_OPT_DISABLE_ATOS) + return 0; + + if (arm_smmu_rpm_get(smmu) < 0) + return 0; + + if (smmu->impl && smmu->impl->iova_to_phys_hard) { + ret = smmu->impl->iova_to_phys_hard(smmu_domain, txn); + goto out; + } + + if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && + smmu_domain->stage == ARM_SMMU_DOMAIN_S1) + ret = __arm_smmu_iova_to_phys_hard(domain, txn->addr); + +out: + arm_smmu_rpm_put(smmu); + + return ret; +} + static bool arm_smmu_capable(enum iommu_cap cap) { switch (cap) { @@ -1373,8 +2390,10 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) fwspec = dev_iommu_fwspec_get(dev); if (ret) goto out_free; - } else if (fwspec && fwspec->ops == &arm_smmu_ops) { + } else if (fwspec && fwspec->ops == &arm_smmu_ops.iommu_ops) { smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); + if (!smmu) + return ERR_PTR(-ENODEV); } else { return ERR_PTR(-ENODEV); } @@ -1436,7 +2455,7 @@ static void arm_smmu_release_device(struct device *dev) struct arm_smmu_device *smmu; int ret; - if (!fwspec || fwspec->ops != &arm_smmu_ops) + if (!fwspec || fwspec->ops != &arm_smmu_ops.iommu_ops) return; cfg = dev_iommu_priv_get(dev); @@ -1476,9 +2495,15 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev) int i, idx; mutex_lock(&smmu->stream_map_mutex); + group = of_get_device_group(dev); + if (group) + goto finish; + for_each_cfg_sme(cfg, fwspec, i, idx) { if (group && smmu->s2crs[idx].group && group != smmu->s2crs[idx].group) { + dev_err(dev, "ID:%x IDX:%x is already in a group!\n", + fwspec->ids[i], idx); mutex_unlock(&smmu->stream_map_mutex); return ERR_PTR(-EINVAL); } @@ -1487,16 +2512,27 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev) } if (group) { - mutex_unlock(&smmu->stream_map_mutex); - return iommu_group_ref_get(group); - } + iommu_group_ref_get(group); + } else { + if (dev_is_pci(dev)) + group = pci_device_group(dev); + else if (dev_is_fsl_mc(dev)) + group = fsl_mc_device_group(dev); + else + group = generic_device_group(dev); - if (dev_is_pci(dev)) - group = pci_device_group(dev); - else if (dev_is_fsl_mc(dev)) - group = fsl_mc_device_group(dev); - else - group = generic_device_group(dev); + if (IS_ERR(group)) { + mutex_unlock(&smmu->stream_map_mutex); + return NULL; + } + } +finish: + if (!IS_ERR(group) && smmu->impl && smmu->impl->device_group && + smmu->impl->device_group(dev, group)) { + iommu_group_put(group); + mutex_unlock(&smmu->stream_map_mutex); + return ERR_PTR(-EINVAL); + } /* Remember group for faster lookups */ if (!IS_ERR(group)) @@ -1567,41 +2603,202 @@ static void arm_smmu_get_resv_regions(struct device *dev, list_add_tail(®ion->list, head); iommu_dma_get_resv_regions(dev, head); + + qcom_iommu_generate_resv_regions(dev, head); } static int arm_smmu_def_domain_type(struct device *dev) { struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); - const struct arm_smmu_impl *impl = cfg->smmu->impl; + const struct arm_smmu_impl *impl; + if (!cfg) + return 0; + + impl = cfg->smmu->impl; if (impl && impl->def_domain_type) return impl->def_domain_type(dev); return 0; } -static struct iommu_ops arm_smmu_ops = { - .capable = arm_smmu_capable, - .domain_alloc = arm_smmu_domain_alloc, - .domain_free = arm_smmu_domain_free, - .attach_dev = arm_smmu_attach_dev, - .map_pages = arm_smmu_map_pages, - .unmap_pages = arm_smmu_unmap_pages, - .flush_iotlb_all = arm_smmu_flush_iotlb_all, - .iotlb_sync = arm_smmu_iotlb_sync, - .iova_to_phys = arm_smmu_iova_to_phys, - .probe_device = arm_smmu_probe_device, - .release_device = arm_smmu_release_device, - .probe_finalize = arm_smmu_probe_finalize, - .device_group = arm_smmu_device_group, - .enable_nesting = arm_smmu_enable_nesting, - .set_pgtable_quirks = arm_smmu_set_pgtable_quirks, - .of_xlate = arm_smmu_of_xlate, - .get_resv_regions = arm_smmu_get_resv_regions, - .put_resv_regions = generic_iommu_put_resv_regions, - .def_domain_type = arm_smmu_def_domain_type, - .pgsize_bitmap = -1UL, /* Restricted during device attach */ - .owner = THIS_MODULE, +static int __arm_smmu_sid_switch(struct device *dev, void *data) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); + struct arm_smmu_device *smmu; + enum sid_switch_direction dir = (typeof(dir))data; + int i, idx; + + if (!fwspec || !cfg) + return 0; + + smmu = cfg->smmu; + mutex_lock(&smmu->stream_map_mutex); + for_each_cfg_sme(cfg, fwspec, i, idx) { + smmu->smrs[idx].valid = dir == SID_ACQUIRE; + arm_smmu_write_sme(smmu, idx); + } + mutex_unlock(&smmu->stream_map_mutex); + return 0; +} + +/* + * Some devices support operation with different levels of security. In some + * modes, HLOS is no longer responsible for managing the S1 translations for + * a device. Unfortunately, the device may still use the same set of SIDS, so + * to prevent a potential stream-match conflict fault, HLOS needs to remove + * the SIDs fromits SMRs. Enforcement of this policy is implemented through + * virtualization of the SMR/S2CR regigisters. + */ +static int arm_smmu_sid_switch(struct device *dev, + enum sid_switch_direction dir) +{ + struct iommu_group *group; + int ret; + + group = iommu_group_get(dev); + ret = iommu_group_for_each_dev(group, (void *)dir, + __arm_smmu_sid_switch); + iommu_group_put(group); + + return ret; +} + +static int arm_smmu_get_context_bank_nr(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int ret; + + mutex_lock(&smmu_domain->init_mutex); + if (!smmu_domain->smmu) + ret = -EINVAL; + else + ret = smmu_domain->cfg.cbndx; + mutex_unlock(&smmu_domain->init_mutex); + return ret; +} + +static int arm_smmu_set_secure_vmid(struct iommu_domain *domain, enum vmid vmid) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int ret = 0; + + mutex_lock(&smmu_domain->init_mutex); + if (smmu_domain->smmu) + ret = -EPERM; + else if (WARN(smmu_domain->secure_vmid != VMID_INVAL, "secure vmid already set")) + ret = -EPERM; + else + smmu_domain->secure_vmid = vmid; + mutex_unlock(&smmu_domain->init_mutex); + return ret; +} + +static int arm_smmu_set_fault_model(struct iommu_domain *domain, int fault_model) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_fault_model *domain_model = &smmu_domain->fault_model; + int ret = 0; + + mutex_lock(&smmu_domain->init_mutex); + if (smmu_domain->smmu) { + ret = -EPERM; + } else { + domain_model->non_fatal = FIELD_GET(QCOM_IOMMU_FAULT_MODEL_NON_FATAL, fault_model); + domain_model->no_cfre = FIELD_GET(QCOM_IOMMU_FAULT_MODEL_NO_CFRE, fault_model); + domain_model->no_stall = FIELD_GET(QCOM_IOMMU_FAULT_MODEL_NO_STALL, fault_model); + domain_model->hupcf = FIELD_GET(QCOM_IOMMU_FAULT_MODEL_HUPCF, fault_model); + } + mutex_unlock(&smmu_domain->init_mutex); + return ret; +} + +static int arm_smmu_enable_s1_translation(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct arm_smmu_device *smmu = smmu_domain->smmu; + int idx; + struct arm_smmu_cb *cb; + int ret; + + mutex_lock(&smmu_domain->init_mutex); + if (!smmu_domain->smmu) { + ret = -EPERM; + goto out; + } else if (!smmu_domain->delayed_s1_trans_enable) { + ret = 0; + goto out; + } + + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + goto out; + + idx = cfg->cbndx; + cfg->sctlr.m = 1; + cb = &smmu->cbs[idx]; + cb->sctlr = arm_smmu_lpae_sctlr(cfg); + + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, cb->sctlr); + arm_smmu_rpm_put(smmu); + smmu_domain->delayed_s1_trans_enable = false; +out: + mutex_unlock(&smmu_domain->init_mutex); + return ret; +} + +static int arm_smmu_get_mappings_configuration(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + int ret = 0; + + mutex_lock(&smmu_domain->init_mutex); + if (!smmu_domain->smmu) { + ret = -EPERM; + } else { + ret |= smmu_domain->mapping_cfg.s1_bypass ? QCOM_IOMMU_MAPPING_CONF_S1_BYPASS : 0; + ret |= smmu_domain->mapping_cfg.atomic ? QCOM_IOMMU_MAPPING_CONF_ATOMIC : 0; + ret |= smmu_domain->mapping_cfg.fast ? QCOM_IOMMU_MAPPING_CONF_FAST : 0; + } + mutex_unlock(&smmu_domain->init_mutex); + return ret; +} + +static struct qcom_iommu_ops arm_smmu_ops = { + .iova_to_phys_hard = arm_smmu_iova_to_phys_hard, + .sid_switch = arm_smmu_sid_switch, + .get_fault_ids = arm_smmu_get_fault_ids, + .get_context_bank_nr = arm_smmu_get_context_bank_nr, + .set_secure_vmid = arm_smmu_set_secure_vmid, + .set_fault_model = arm_smmu_set_fault_model, + .enable_s1_translation = arm_smmu_enable_s1_translation, + .get_mappings_configuration = arm_smmu_get_mappings_configuration, + + .iommu_ops = { + .capable = arm_smmu_capable, + .domain_alloc = arm_smmu_domain_alloc, + .domain_free = arm_smmu_domain_free, + .attach_dev = arm_smmu_attach_dev, + .map_pages = arm_smmu_map_pages, + .unmap_pages = arm_smmu_unmap_pages, + .flush_iotlb_all = arm_smmu_flush_iotlb_all, + .iotlb_sync = arm_smmu_iotlb_sync, + .iova_to_phys = arm_smmu_iova_to_phys, + .probe_device = arm_smmu_probe_device, + .release_device = arm_smmu_release_device, + .probe_finalize = arm_smmu_probe_finalize, + .device_group = arm_smmu_device_group, + .enable_nesting = arm_smmu_enable_nesting, + .set_pgtable_quirks = arm_smmu_set_pgtable_quirks, + .of_xlate = arm_smmu_of_xlate, + .get_resv_regions = arm_smmu_get_resv_regions, + .put_resv_regions = generic_iommu_put_resv_regions, + .def_domain_type = arm_smmu_def_domain_type, + .pgsize_bitmap = -1UL, /* Restricted during device attach */ + .owner = THIS_MODULE, + } }; static void arm_smmu_device_reset(struct arm_smmu_device *smmu) @@ -1658,6 +2855,10 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) if (smmu->features & ARM_SMMU_FEAT_EXIDS) reg |= ARM_SMMU_sCR0_EXIDENABLE; + /* Force bypass transaction to be Non-Shareable & not io-coherent */ + reg &= ~ARM_SMMU_sCR0_SHCFG; + reg |= FIELD_PREP(ARM_SMMU_sCR0_SHCFG, ARM_SMMU_sCR0_SHCFG_NSH); + if (smmu->impl && smmu->impl->reset) smmu->impl->reset(smmu); @@ -1685,6 +2886,148 @@ static int arm_smmu_id_size_to_bits(int size) } } +static int arm_smmu_handoff_cbs(struct arm_smmu_device *smmu) +{ + u32 i, smr, s2cr; + u32 index; + struct arm_smmu_smr smrs; + struct arm_smmu_smr *handoff_smrs; + int num_handoff_smrs; + const __be32 *cell; + + cell = of_get_property(smmu->dev->of_node, "qcom,handoff-smrs", NULL); + if (!cell) + return 0; + + num_handoff_smrs = of_property_count_elems_of_size(smmu->dev->of_node, + "qcom,handoff-smrs", sizeof(u32) * 2); + if (num_handoff_smrs < 0) + return 0; + + handoff_smrs = kcalloc(num_handoff_smrs, sizeof(*handoff_smrs), + GFP_KERNEL); + if (!handoff_smrs) + return -ENOMEM; + + for (i = 0; i < num_handoff_smrs; i++) { + handoff_smrs[i].id = of_read_number(cell++, 1); + handoff_smrs[i].mask = of_read_number(cell++, 1); + handoff_smrs[i].valid = true; + } + + + for (i = 0; i < smmu->num_mapping_groups; ++i) { + + smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); + + if (smmu->features & ARM_SMMU_FEAT_EXIDS) { + s2cr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_S2CR(i)); + smrs.valid = FIELD_GET(ARM_SMMU_S2CR_EXIDVALID, s2cr); + if (!smrs.valid) + continue; + + smrs.id = FIELD_GET(ARM_SMMU_SMR_ID, smr); + smrs.mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); + + } else { + smrs.valid = FIELD_GET(ARM_SMMU_SMR_VALID, smr); + if (!smrs.valid) + continue; + + smrs.id = FIELD_GET(ARM_SMMU_SMR_ID, smr); + /* + * The SMR mask covers bits 30:16 when extended stream + * matching is not enabled. + */ + smrs.mask = FIELD_GET(ARM_SMMU_SMR_MASK, + smr & ~ARM_SMMU_SMR_VALID); + } + + for (index = 0; index < num_handoff_smrs; index++) { + + if (!handoff_smrs[index].valid) + continue; + + if ((handoff_smrs[index].mask & smrs.mask) == handoff_smrs[index].mask && + !((handoff_smrs[index].id ^ smrs.id) & ~smrs.mask)) { + + dev_dbg(smmu->dev, + "handoff-smrs match idx %d, id, 0x%x, mask 0x%x\n", + i, smrs.id, smrs.mask); + + s2cr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_S2CR(i)); + + smmu->smrs[i] = smrs; + + smmu->s2crs[i].group = NULL; + smmu->s2crs[i].count = 0; + smmu->s2crs[i].type = FIELD_GET(ARM_SMMU_S2CR_TYPE, s2cr); + smmu->s2crs[i].privcfg = FIELD_GET(ARM_SMMU_S2CR_PRIVCFG, s2cr); + smmu->s2crs[i].cbndx = FIELD_GET(ARM_SMMU_S2CR_CBNDX, s2cr); + + smmu->s2crs[i].pinned = true; + bitmap_set(smmu->context_map, smmu->s2crs[i].cbndx, 1); + handoff_smrs[index].valid = false; + + break; + + } else { + dev_dbg(smmu->dev, + "handoff-smrs no match idx %d, id, 0x%x, mask 0x%x\n", + i, smrs.id, smrs.mask); + } + } + } + + kfree(handoff_smrs); + + return 0; +} + +static int arm_smmu_parse_impl_def_registers(struct arm_smmu_device *smmu) +{ + struct device *dev = smmu->dev; + int i, ntuples, ret; + u32 *tuples; + struct arm_smmu_impl_def_reg *regs, *regit; + + if (!of_find_property(dev->of_node, "attach-impl-defs", &ntuples)) + return 0; + + ntuples /= sizeof(u32); + if (ntuples % 2) { + dev_err(dev, + "Invalid number of attach-impl-defs registers: %d\n", + ntuples); + return -EINVAL; + } + + regs = devm_kmalloc_array(dev, ntuples, sizeof(*regs), GFP_KERNEL); + if (!regs) + return -ENOMEM; + + tuples = kmalloc_array(ntuples * 2, sizeof(*tuples), GFP_KERNEL); + if (!tuples) + return -ENOMEM; + + ret = of_property_read_u32_array(dev->of_node, "attach-impl-defs", + tuples, ntuples); + if (ret) + goto out; + + for (i = 0, regit = regs; i < ntuples; i += 2, ++regit) { + regit->offset = tuples[i]; + regit->value = tuples[i + 1]; + } + + smmu->impl_def_attach_registers = regs; + smmu->num_impl_def_attach_registers = ntuples / 2; + +out: + kfree(tuples); + return ret; +} + static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) { unsigned int size; @@ -1870,10 +3213,10 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) smmu->pgsize_bitmap |= SZ_64K | SZ_512M; - if (arm_smmu_ops.pgsize_bitmap == -1UL) - arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; + if (arm_smmu_ops.iommu_ops.pgsize_bitmap == -1UL) + arm_smmu_ops.iommu_ops.pgsize_bitmap = smmu->pgsize_bitmap; else - arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; + arm_smmu_ops.iommu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", smmu->pgsize_bitmap); @@ -2077,6 +3420,10 @@ static int arm_smmu_device_probe(struct platform_device *pdev) int num_irqs, i, err; irqreturn_t (*global_fault)(int irq, void *dev); + /* We depend on this device for fastmap */ + if (!qcom_dma_iommu_is_ready()) + return -EPROBE_DEFER; + smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) { dev_err(dev, "failed to allocate arm_smmu_device\n"); @@ -2130,32 +3477,40 @@ static int arm_smmu_device_probe(struct platform_device *pdev) for (i = 0; i < num_irqs; ++i) { int irq = platform_get_irq(pdev, i); - if (irq < 0) + if (irq < 0) { + dev_err(dev, "failed to get irq index %d\n", i); return -ENODEV; + } smmu->irqs[i] = irq; } - err = devm_clk_bulk_get_all(dev, &smmu->clks); - if (err < 0) { - dev_err(dev, "failed to get clocks %d\n", err); - return err; - } - smmu->num_clks = err; + smmu->pwr = arm_smmu_init_power_resources(dev); + if (IS_ERR(smmu->pwr)) + return PTR_ERR(smmu->pwr); - err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks); + /* + * We can't use arm_smmu_rpm_get() because pm-runtime isn't + * enabled yet. + */ + err = arm_smmu_power_on(smmu->pwr); if (err) return err; err = arm_smmu_device_cfg_probe(smmu); if (err) - return err; + goto out_power_off; + + err = arm_smmu_parse_impl_def_registers(smmu); + if (err) + goto out_power_off; if (smmu->version == ARM_SMMU_V2) { if (smmu->num_context_banks > smmu->num_context_irqs) { dev_err(dev, "found only %d context irq(s) but %d required\n", smmu->num_context_irqs, smmu->num_context_banks); - return -ENODEV; + err = -ENODEV; + goto out_power_off; } /* Ignore superfluous interrupts */ @@ -2168,26 +3523,35 @@ static int arm_smmu_device_probe(struct platform_device *pdev) global_fault = arm_smmu_global_fault; for (i = 0; i < smmu->num_global_irqs; ++i) { - err = devm_request_irq(smmu->dev, smmu->irqs[i], + err = devm_request_threaded_irq(smmu->dev, smmu->irqs[i], + NULL, global_fault, - IRQF_SHARED, + IRQF_ONESHOT | IRQF_SHARED, "arm-smmu global fault", smmu); if (err) { dev_err(dev, "failed to request global IRQ %d (%u)\n", i, smmu->irqs[i]); - return err; + goto out_power_off; } } + /* QCOM Additions */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + smmu->phys_addr = res->start; + parse_driver_options(smmu); + err = arm_smmu_handoff_cbs(smmu); + if (err) + goto out_power_off; + err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL, "smmu.%pa", &ioaddr); if (err) { dev_err(dev, "Failed to register iommu in sysfs\n"); - return err; + goto out_power_off; } - err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); + err = iommu_device_register(&smmu->iommu, &arm_smmu_ops.iommu_ops, dev); if (err) { dev_err(dev, "Failed to register iommu\n"); goto err_sysfs_remove; @@ -2196,6 +3560,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) platform_set_drvdata(pdev, smmu); arm_smmu_device_reset(smmu); arm_smmu_test_smr_masks(smmu); + arm_smmu_interrupt_selftest(smmu); /* * We want to avoid touching dev->power.lock in fastpaths unless @@ -2203,7 +3568,11 @@ static int arm_smmu_device_probe(struct platform_device *pdev) * can serve as an ideal proxy for that decision. So, conditionally * enable pm_runtime. */ - if (dev->pm_domain) { + /* + * QCOM's nonupstream gdsc driver doesn't support pm_domains. + * So check for presence of gdsc instead. + */ + if (smmu->pwr->num_gdscs) { pm_runtime_set_active(dev); pm_runtime_enable(dev); } @@ -2214,7 +3583,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) * ready to handle default domain setup as soon as any SMMU exists. */ if (!using_legacy_binding) { - err = arm_smmu_bus_init(&arm_smmu_ops); + err = arm_smmu_bus_init(&arm_smmu_ops.iommu_ops); if (err) goto err_unregister_device; } @@ -2225,6 +3594,8 @@ static int arm_smmu_device_probe(struct platform_device *pdev) iommu_device_unregister(&smmu->iommu); err_sysfs_remove: iommu_device_sysfs_remove(&smmu->iommu); +out_power_off: + arm_smmu_power_off(smmu, smmu->pwr); return err; } @@ -2242,6 +3613,9 @@ static int arm_smmu_device_remove(struct platform_device *pdev) iommu_device_unregister(&smmu->iommu); iommu_device_sysfs_remove(&smmu->iommu); + if (smmu->impl && smmu->impl->device_remove) + smmu->impl->device_remove(smmu); + arm_smmu_rpm_get(smmu); /* Turn the thing off */ arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD); @@ -2250,9 +3624,8 @@ static int arm_smmu_device_remove(struct platform_device *pdev) if (pm_runtime_enabled(smmu->dev)) pm_runtime_force_suspend(smmu->dev); else - clk_bulk_disable(smmu->num_clks, smmu->clks); + arm_smmu_power_off(smmu, smmu->pwr); - clk_bulk_unprepare(smmu->num_clks, smmu->clks); return 0; } @@ -2266,12 +3639,10 @@ static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) struct arm_smmu_device *smmu = dev_get_drvdata(dev); int ret; - ret = clk_bulk_enable(smmu->num_clks, smmu->clks); + ret = arm_smmu_power_on(smmu->pwr); if (ret) return ret; - arm_smmu_device_reset(smmu); - return 0; } @@ -2279,7 +3650,7 @@ static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev) { struct arm_smmu_device *smmu = dev_get_drvdata(dev); - clk_bulk_disable(smmu->num_clks, smmu->clks); + arm_smmu_power_off(smmu, smmu->pwr); return 0; } @@ -2297,9 +3668,17 @@ static int __maybe_unused arm_smmu_pm_resume(struct device *dev) return 0; ret = arm_smmu_runtime_resume(dev); - if (ret) + if (ret) { clk_bulk_unprepare(smmu->num_clks, smmu->clks); + return ret; + } + /* + * QCOM HW supports register retention. So we really only need to + * re-program the registers for hibernation. Don't do this during + * runtime_resume to avoid latency. + */ + arm_smmu_device_reset(smmu); return ret; } @@ -2337,7 +3716,34 @@ static struct platform_driver arm_smmu_driver = { .remove = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; -module_platform_driver(arm_smmu_driver); + +static int __init arm_smmu_init(void) +{ + int ret; + ktime_t cur; + + cur = ktime_get(); + ret = platform_driver_register(&qsmmuv500_tbu_driver); + if (ret) + return ret; + + ret = platform_driver_register(&arm_smmu_driver); + if (ret) { + platform_driver_unregister(&qsmmuv500_tbu_driver); + return ret; + } + + trace_smmu_init(ktime_us_delta(ktime_get(), cur)); + return ret; +} +subsys_initcall(arm_smmu_init); + +static void __exit arm_smmu_exit(void) +{ + platform_driver_unregister(&arm_smmu_driver); + platform_driver_unregister(&qsmmuv500_tbu_driver); +} +module_exit(arm_smmu_exit); MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon "); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.h b/drivers/iommu/arm/arm-smmu/arm-smmu.h index 0d210edb28b5..f75f380d5ec8 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.h +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.h @@ -5,6 +5,8 @@ * Copyright (C) 2013 ARM Limited * * Author: Will Deacon + * + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef _ARM_SMMU_H @@ -22,10 +24,15 @@ #include #include #include +#include + +#include "../../qcom-io-pgtable.h" /* Configuration registers */ #define ARM_SMMU_GR0_sCR0 0x0 #define ARM_SMMU_sCR0_VMID16EN BIT(31) +#define ARM_SMMU_sCR0_SHCFG GENMASK(23, 22) +#define ARM_SMMU_sCR0_SHCFG_NSH 0x3 #define ARM_SMMU_sCR0_BSU GENMASK(15, 14) #define ARM_SMMU_sCR0_FB BIT(13) #define ARM_SMMU_sCR0_PTM BIT(12) @@ -117,6 +124,8 @@ enum arm_smmu_s2cr_type { S2CR_TYPE_FAULT, }; #define ARM_SMMU_S2CR_EXIDVALID BIT(10) +#define ARM_SMMU_S2CR_SHCFG GENMASK(9, 8) +#define ARM_SMMU_S2CR_SHCFG_NSH 0x3 #define ARM_SMMU_S2CR_CBNDX GENMASK(7, 0) /* Context bank attribute registers */ @@ -136,12 +145,23 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CBAR_VMID GENMASK(7, 0) #define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2)) +#define CBFRSYNRA_SID_MASK (0xffff) #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) #define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16) #define ARM_SMMU_CBA2R_VA64 BIT(0) #define ARM_SMMU_CB_SCTLR 0x0 +#define ARM_SMMU_SCTLR_WACFG GENMASK(27, 26) +#define ARM_SMMU_SCTLR_WACFG_WA 0x2 +#define ARM_SMMU_SCTLR_RACFG GENMASK(25, 24) +#define ARM_SMMU_SCTLR_RACFG_RA 0x2 +#define ARM_SMMU_SCTLR_SHCFG GENMASK(23, 22) +#define ARM_SMMU_SCTLR_SHCFG_OSH 0x1 +#define ARM_SMMU_SCTLR_SHCFG_NSH 0x3 +#define ARM_SMMU_SCTLR_MTCFG BIT(20) +#define ARM_SMMU_SCTLR_MEM_ATTR GENMASK(19, 16) +#define ARM_SMMU_SCTLR_MEM_ATTR_OISH_WB_CACHE 0xf #define ARM_SMMU_SCTLR_S1_ASIDPNE BIT(12) #define ARM_SMMU_SCTLR_CFCFG BIT(7) #define ARM_SMMU_SCTLR_HUPCF BIT(8) @@ -156,6 +176,7 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CB_RESUME 0x8 #define ARM_SMMU_RESUME_TERMINATE BIT(0) +#define ARM_SMMU_RESUME_RESUME 0 #define ARM_SMMU_CB_TCR2 0x10 #define ARM_SMMU_TCR2_SEP GENMASK(17, 15) @@ -219,10 +240,19 @@ enum arm_smmu_cbar_type { ARM_SMMU_FSR_TF | \ ARM_SMMU_FSR_IGN) +#define ARM_SMMU_CB_FSRRESTORE 0x5c #define ARM_SMMU_CB_FAR 0x60 #define ARM_SMMU_CB_FSYNR0 0x68 #define ARM_SMMU_FSYNR0_WNR BIT(4) +#define ARM_SMMU_FSYNR0_PNU BIT(5) +#define ARM_SMMU_FSYNR0_IND BIT(6) +#define ARM_SMMU_FSYNR0_NSATTR BIT(8) + +#define ARM_SMMU_CB_FSYNR1 0x6c +#define ARM_SMMU_FSYNR1_BID GENMASK(15, 13) +#define ARM_SMMU_FSYNR1_PID GENMASK(12, 8) +#define ARM_SMMU_FSYNR1_MID GENMASK(7, 0) #define ARM_SMMU_CB_FSYNR1 0x6c @@ -235,6 +265,24 @@ enum arm_smmu_cbar_type { #define ARM_SMMU_CB_TLBSTATUS 0x7f4 #define ARM_SMMU_CB_ATS1PR 0x800 +/* Implementation Defined Register Space 5 registers*/ +/* Relative to IMPL_DEF5 page */ +#define ARM_SMMU_STATS_SYNC_INV_TBU_ACK 0x5dc +#define TBU_SYNC_ACK GENMASK(31, 17) +#define TBU_SYNC_REQ BIT(16) +#define TBU_INV_ACK GENMASK(15, 1) +#define TBU_INV_REQ BIT(0) +#define APPS_SMMU_TBU_REG_ACCESS_REQ_NS 0x5f8 +#define APPS_SMMU_TBU_REG_ACCESS_ACK_NS 0x5fc + +/* Relative to SMMU_BASE */ +#define ARM_SMMU_TBU_PWR_STATUS 0x2204 + +/* Relative SMMU_BASE */ +#define ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR 0x2670 +#define TCU_SYNC_IN_PRGSS BIT(20) +#define TCU_INV_IN_PRGSS BIT(16) + #define ARM_SMMU_CB_ATSR 0x8f0 #define ARM_SMMU_ATSR_ACTIVE BIT(0) @@ -242,8 +290,9 @@ enum arm_smmu_cbar_type { /* Maximum number of context banks per SMMU */ #define ARM_SMMU_MAX_CBS 128 -#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */ +#define TLB_LOOP_TIMEOUT 500000 /* 500ms */ #define TLB_SPIN_COUNT 10 +#define TLB_LOOP_INC_MAX 1000 /*1ms*/ /* Shared driver definitions */ enum arm_smmu_arch_version { @@ -260,12 +309,42 @@ enum arm_smmu_implementation { QCOM_SMMUV500, }; +struct arm_smmu_impl_def_reg { + u32 offset; + u32 value; +}; + +/* + * Describes resources required for on/off power operation. + * Separate reference count is provided for atomic/nonatomic + * operations. + */ +struct arm_smmu_power_resources { + struct device *dev; + + struct clk **clocks; + int num_clocks; + + struct regulator_bulk_data *gdscs; + int num_gdscs; + + struct icc_path *icc_path; + + /* Protects power_count */ + struct mutex power_lock; + int power_count; + + int (*resume)(struct arm_smmu_power_resources *pwr); + void (*suspend)(struct arm_smmu_power_resources *pwr); +}; + struct arm_smmu_s2cr { struct iommu_group *group; int count; enum arm_smmu_s2cr_type type; enum arm_smmu_s2cr_privcfg privcfg; u8 cbndx; + bool pinned; }; struct arm_smmu_smr { @@ -297,6 +376,12 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_EXIDS (1 << 12) u32 features; +#define ARM_SMMU_OPT_FATAL_ASF (1 << 0) +#define ARM_SMMU_OPT_3LVL_TABLES (1 << 2) +#define ARM_SMMU_OPT_NO_ASID_RETENTION (1 << 3) +#define ARM_SMMU_OPT_DISABLE_ATOS (1 << 4) +#define ARM_SMMU_OPT_CONTEXT_FAULT_RETRY (1 << 5) + u32 options; enum arm_smmu_arch_version version; enum arm_smmu_implementation model; const struct arm_smmu_impl *impl; @@ -329,6 +414,17 @@ struct arm_smmu_device { /* IOMMU core code handle */ struct iommu_device iommu; + + /* Specific to QCOM */ + struct arm_smmu_impl_def_reg *impl_def_attach_registers; + unsigned int num_impl_def_attach_registers; + + struct arm_smmu_power_resources *pwr; + + /* used for qsmmuv500 scm_io_readl */ + phys_addr_t phys_addr; + + unsigned long sync_timed_out; }; enum arm_smmu_context_fmt { @@ -345,6 +441,19 @@ struct arm_smmu_cfg { u16 asid; u16 vmid; }; + u32 procid; + struct { + u32 wacfg:2; + u32 racfg:2; + u32 shcfg:2; + u32 mtcfg:1; + u32 memattr:4; + u32 hupcf:1; + u32 cfcfg:1; + u32 cfre:1; + u32 m:1; + } sctlr; + enum arm_smmu_cbar_type cbar; enum arm_smmu_context_fmt fmt; bool flush_walk_prefer_tlbiasid; @@ -355,6 +464,7 @@ struct arm_smmu_cb { u64 ttbr[2]; u32 tcr[2]; u32 mair[2]; + u32 sctlr; struct arm_smmu_cfg *cfg; }; @@ -365,16 +475,53 @@ enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_BYPASS, }; +struct arm_smmu_fault_model { + char non_fatal : 1; + char no_cfre : 1; + char no_stall : 1; + char hupcf : 1; +}; + +struct arm_smmu_mapping_cfg { + char s1_bypass : 1; + char atomic : 1; + char fast : 1; +}; + struct arm_smmu_domain { struct arm_smmu_device *smmu; + struct device *dev; struct io_pgtable_ops *pgtbl_ops; unsigned long pgtbl_quirks; + bool force_coherent_walk; const struct iommu_flush_ops *flush_ops; struct arm_smmu_cfg cfg; enum arm_smmu_domain_stage stage; struct mutex init_mutex; /* Protects smmu pointer */ - spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */ + spinlock_t cb_lock; /* Serialises ATS1* ops */ + spinlock_t sync_lock; /* Serialises TLB syncs */ + struct arm_smmu_fault_model fault_model; + struct arm_smmu_mapping_cfg mapping_cfg; + bool delayed_s1_trans_enable; + u32 secure_vmid; + + /* + * Track PMDs which require tlb invalidate prior to being + * freed, or before their iovas can be reused by iommu_map(). + */ + spinlock_t iotlb_gather_lock; + struct page *freelist; + bool deferred_sync; + + struct iommu_debug_attachment *logger; struct iommu_domain domain; + /* mapping_cfg.atomic indicates that runtime power management should be disabled. */ + bool rpm_always_on; + +#ifdef CONFIG_ARM_SMMU_CONTEXT_FAULT_RETRY + u64 prev_fault_address; + u32 fault_retry_counter; +#endif }; struct arm_smmu_master_cfg { @@ -421,7 +568,28 @@ static inline u32 arm_smmu_lpae_vtcr(const struct io_pgtable_cfg *cfg) FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz); } +static inline u32 arm_smmu_lpae_sctlr(struct arm_smmu_cfg *cfg) +{ + bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; + + return FIELD_PREP(ARM_SMMU_SCTLR_WACFG, cfg->sctlr.wacfg) | + FIELD_PREP(ARM_SMMU_SCTLR_RACFG, cfg->sctlr.racfg) | + FIELD_PREP(ARM_SMMU_SCTLR_SHCFG, cfg->sctlr.shcfg) | + FIELD_PREP(ARM_SMMU_SCTLR_MTCFG, cfg->sctlr.mtcfg) | + FIELD_PREP(ARM_SMMU_SCTLR_MEM_ATTR, cfg->sctlr.memattr) | + FIELD_PREP(ARM_SMMU_SCTLR_S1_ASIDPNE, stage1) | + FIELD_PREP(ARM_SMMU_SCTLR_HUPCF, cfg->sctlr.hupcf) | + FIELD_PREP(ARM_SMMU_SCTLR_CFCFG, cfg->sctlr.cfcfg) | + ARM_SMMU_SCTLR_CFIE | + FIELD_PREP(ARM_SMMU_SCTLR_CFRE, cfg->sctlr.cfre) | + FIELD_PREP(ARM_SMMU_SCTLR_E, IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) | + ARM_SMMU_SCTLR_AFE | + ARM_SMMU_SCTLR_TRE | + FIELD_PREP(ARM_SMMU_SCTLR_M, cfg->sctlr.m); +} + /* Implementation details, yay! */ + struct arm_smmu_impl { u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset); void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset, @@ -433,6 +601,13 @@ struct arm_smmu_impl { int (*reset)(struct arm_smmu_device *smmu); int (*init_context)(struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg *cfg, struct device *dev); + void (*init_context_bank)(struct arm_smmu_domain *smmu_domain, + struct device *dev); + phys_addr_t (*iova_to_phys_hard)(struct arm_smmu_domain *smmu_domain, + struct qcom_iommu_atos_txn *txn); + void (*tlb_sync_timeout)(struct arm_smmu_device *smmu); + void (*device_remove)(struct arm_smmu_device *smmu); + int (*device_group)(struct device *dev, struct iommu_group *group); void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync, int status); int (*def_domain_type)(struct device *dev); @@ -504,6 +679,13 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, #define ARM_SMMU_GR0 0 #define ARM_SMMU_GR1 1 + +/* + * Implementation defined space starts after SMMU GR space, so IMPL_DEF page n + * is page n + 2 in the SMMU register space. + */ +#define ARM_SMMU_IMPL_DEF5 7 + #define ARM_SMMU_CB(s, n) ((s)->numpage + (n)) #define arm_smmu_gr0_read(s, o) \ @@ -528,8 +710,21 @@ static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu); struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); +struct arm_smmu_device *qsmmuv500_impl_init(struct arm_smmu_device *smmu); +struct arm_smmu_device *qcom_adreno_smmu_impl_init(struct arm_smmu_device *smmu); void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx); int arm_mmu500_reset(struct arm_smmu_device *smmu); +int arm_smmu_power_on(struct arm_smmu_power_resources *pwr); +void arm_smmu_power_off(struct arm_smmu_device *smmu, + struct arm_smmu_power_resources *pwr); +struct arm_smmu_power_resources *arm_smmu_init_power_resources( + struct device *dev); + +extern struct platform_driver qsmmuv500_tbu_driver; + +/* Misc. constants */ +#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) + #endif /* _ARM_SMMU_H */