github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

iommu/arm-smmu-v3: Handle ARM erratum for CONT under invalidation with SVA

Browse Source 
The erratum (MMU-700: #3777127, S3: #3673557) deals with under
invalidation of a CONT PTE grouping in the SMMU. The recommended work
around is to use a range invalidation that spans the entire CONT. The only
user of CONT in the kernel right now is through SVA sharing a CPU page
table that contains a CONT created by the mm.

Previously it was thought that this errata was dealt with because the
driver always uses range invalidation. However, there is a subtle detail
in the errata that the invalidation range must fully enclose the entire
CONT for it to work.

It seems that two sequential range invalidations, with a split point
falling inside a CONT grouping, will not prevent the errata.

The SMMU's range invalidation generation algorithm does not produce a
single invalidation for a single SVA invalidation request, nor does the mm
carefully align the SVA invalidation ranges to accommodate the splitting
of the invalidation into several ranges.

Thus, when processing a SVA invalidation, the range invalidation splitting
routine can generate a range invalidation that is split in the middle of
the CONT and risk under invalidation from this errata. This condition
could be triggered by a malicious userspace manipulating the TLB gathers
via mmap/mprotect/munmap.

Update the errata list to the include the S3 variation, detect the IOMMUs
that have it, and then have SVA invalidations use a simplified version of
the over invalidation algorithm from the tlbi rework series. This ensures
that a single range invalidation is issued for a single MMU notifier
callback and now the range invalidation is guarenteed to cover any posible
CONT.

Future work to add CONT to iommu_domain page tables should either use this
one-invalidate/one-range invalidation algorithm or disable CONT support in
the iommu_domain.

Cc: stable@vger.kernel.org
Cc: Vijayanand Jitta <vijayanand.jitta@oss.qualcomm.com>
Fixes: 3f1ce8e85e ("iommu/arm-smmu-v3: Share process page tables")
Reviewed-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Pranjal Shrivastava <peaan@google.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Tested-by: Pranjal Shrivastava <praan@google.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Will Deacon <will@kernel.org>
This commit is contained in:
Jason Gunthorpe 2026-10-02 22:23:20 -03:00 committed by Will Deacon
parent b370a5e979
commit 84c6089a36
4 changed files with 88 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Documentation/arch/arm64/silicon-errata.rst
View File

@ -271,7 +271,8 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| ARM | MMU L1 | #3878312 | N/A |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | MMU S3 | #3995052 | N/A |
| ARM | MMU S3 | #3995052, | N/A |
| | | #3673557 | |
+----------------+-----------------+-----------------+-----------------------------+
| ARM | GIC-700 | #2941627 | ARM64_ERRATUM_2941627 |
+----------------+-----------------+-----------------+-----------------------------+

7
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
View File

@ -215,6 +215,13 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
if (system_supports_haft())
feat_mask |= ARM_SMMU_FEAT_HAFT;
/*
* The workaround for ARM_SMMU_OPT_FULL_CONT_RANGE_INV requires range
* invalidation support.
*/
if (smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV)
feat_mask |= ARM_SMMU_FEAT_RANGE_INV;
if ((smmu->features & feat_mask) != feat_mask)
return false;

91
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
View File

@ -2560,6 +2560,37 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu,
}
}
/*
* Generate a range invalidation for ARM_SMMU_OPT_FULL_CONT_RANGE_INV by
* ensuring the entire SVA requested range is covered with a single range
* invalidation command. The scale is adjusted so that the range invalidation
* may extend past the end of the requested range. This ensures that any CONT
* the MM is invalidating is covered by a single range invalidation. TTL and
* LEAF are always 0 because this is only used by SVA.
*/
static bool arm_smmu_cmdq_batch_add_range_inv(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd,
unsigned long iova, size_t size,
u8 tgsz_lg2)
{
u64 cur_tg = iova >> tgsz_lg2;
u64 num_tg = ((iova + size - 1) >> tgsz_lg2) - cur_tg + 1;
unsigned int scale = fls64((num_tg - 1) / 32);
if (scale > 31)
return false;
cmd->data[0] |=
FIELD_PREP(CMDQ_TLBI_0_NUM,
DIV_ROUND_UP_ULL(num_tg, 1ULL << scale) - 1) |
FIELD_PREP(CMDQ_TLBI_0_SCALE, scale);
cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_TG, (tgsz_lg2 - 10) / 2) |
(cur_tg << tgsz_lg2);
arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, cmd);
return true;
}
static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size,
size_t granule)
{
@ -2587,21 +2618,30 @@ static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size,
static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd,
bool leaf,
bool single_range_inv, bool leaf,
unsigned long iova, size_t size,
unsigned int granule)
{
if (arm_smmu_inv_size_too_big(inv->smmu, size, granule)) {
struct arm_smmu_cmd nsize_cmd = *cmd;
struct arm_smmu_cmd nsize_cmd;
u64p_replace_bits(&nsize_cmd.data[0], inv->nsize_opcode,
CMDQ_0_OP);
arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, &nsize_cmd);
if (arm_smmu_inv_size_too_big(inv->smmu, size, granule))
goto full_inv;
if (single_range_inv && size > granule) {
if (!arm_smmu_cmdq_batch_add_range_inv(inv->smmu, cmds, cmd,
iova, size, inv->pgsize))
goto full_inv;
return;
}
arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, leaf,
iova, size, granule, inv->pgsize);
arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, leaf, iova, size,
granule, inv->pgsize);
return;
full_inv:
nsize_cmd = *cmd;
u64p_replace_bits(&nsize_cmd.data[0], inv->nsize_opcode, CMDQ_0_OP);
arm_smmu_cmdq_batch_add_cmd_p(inv->smmu, cmds, &nsize_cmd);
}
static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur,
@ -2622,7 +2662,8 @@ static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur,
static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
unsigned long iova, size_t size,
unsigned int granule, bool leaf)
unsigned int granule,
bool single_range_inv, bool leaf)
{
struct arm_smmu_cmdq_batch cmds = {};
struct arm_smmu_inv *cur;
@ -2652,14 +2693,16 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs,
case INV_TYPE_S1_ASID:
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode,
cur->id, 0);
arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf,
iova, size, granule);
arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd,
single_range_inv, leaf, iova,
size, granule);
break;
case INV_TYPE_S2_VMID:
cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode,
0, cur->id);
arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, leaf,
iova, size, granule);
arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd,
single_range_inv, leaf, iova,
size, granule);
break;
case INV_TYPE_S2_VMID_S1_CLEAR:
/* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */
@ -2706,6 +2749,9 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
unsigned int granule, bool leaf)
{
struct arm_smmu_invs *invs;
bool single_range_inv =
smmu_domain->stage == ARM_SMMU_DOMAIN_SVA &&
(smmu_domain->smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV);
/*
* An invalidation request must follow some IOPTE change and then load
@ -2745,10 +2791,12 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain,
unsigned long flags;
read_lock_irqsave(&invs->rwlock, flags);
__arm_smmu_domain_inv_range(invs, iova, size, granule, leaf);
__arm_smmu_domain_inv_range(invs, iova, size, granule,
single_range_inv, leaf);
read_unlock_irqrestore(&invs->rwlock, flags);
} else {
__arm_smmu_domain_inv_range(invs, iova, size, granule, leaf);
__arm_smmu_domain_inv_range(invs, iova, size, granule,
single_range_inv, leaf);
}
rcu_read_unlock();
@ -5081,12 +5129,21 @@ static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)
/* Arm errata 2268618, 2812531 */
smmu->features &= ~ARM_SMMU_FEAT_NESTING;
}
/* Arm errata 3777127 */
smmu->options |= ARM_SMMU_OPT_FULL_CONT_RANGE_INV;
break;
case IIDR_PRODUCTID_ARM_MMU_L1:
case IIDR_PRODUCTID_ARM_MMU_S3:
/* Arm errata 3878312/3995052 */
/* Arm errata 3878312 */
smmu->features &= ~ARM_SMMU_FEAT_BTM;
break;
case IIDR_PRODUCTID_ARM_MMU_S3:
/* Arm errata 3995052 */
smmu->features &= ~ARM_SMMU_FEAT_BTM;
/* Arm errata 3673557 */
if (variant < 1 || (variant == 1 && revision < 1))
smmu->options |=
ARM_SMMU_OPT_FULL_CONT_RANGE_INV;
break;
}
break;
}

5
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
View File

@ -940,6 +940,11 @@ struct arm_smmu_device {
#define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3)
#define ARM_SMMU_OPT_TEGRA241_CMDQV (1 << 4)
#define ARM_SMMU_OPT_OVR_INSTCFG_DATA (1 << 5)
/*
* Range invalidation is mandatory and one range invalidation must fully span an
* invalidated CONT
*/
#define ARM_SMMU_OPT_FULL_CONT_RANGE_INV (1 << 6)
u32 options;
struct arm_smmu_cmdq cmdq;