github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-22 04:34:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

iommu/arm-smmu-v3: Directly encode simple commands

Browse Source 
Add make functions to build commands for

 CMDQ_OP_TLBI_EL2_ALL
 CMDQ_OP_TLBI_NSNH_ALL
 CMDQ_OP_CFGI_ALL
 CMDQ_OP_PREFETCH_CFG
 CMDQ_OP_CFGI_STE
 CMDQ_OP_CFGI_CD
 CMDQ_OP_RESUME
 CMDQ_OP_PRI_RESP

Convert all of these call sites to use the make function instead of
going through arm_smmu_cmdq_build_cmd(). Use a #define so the general
pattern is always:

   arm_smmu_cmdq_issue_cmd(smmu, arm_smmu_make_cmd_XX(..));

Add arm_smmu_cmdq_batch_add_cmd() which takes struct arm_smmu_cmd
directly to match the new flow.

Reviewed-by: Pranjal Shrivastava <praan@google.com>
Reviewed-by: Mostafa Saleh <smostafa@google.com>
Tested-by: Pranjal Shrivastava <praan@google.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Nicolin Chen <nicolinc@nvidia.com>
Tested-by: Samiullah Khawaja <skhawaja@google.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Will Deacon <will@kernel.org>
This commit is contained in:
Jason Gunthorpe 2026-05-13 20:57:45 -03:00 committed by Will Deacon
parent c5758947cb
commit 6e771be45e
2 changed files with 151 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

213
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
View File

@ -277,23 +277,6 @@ static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out,
cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
switch (ent->opcode) {
case CMDQ_OP_TLBI_EL2_ALL:
case CMDQ_OP_TLBI_NSNH_ALL:
break;
case CMDQ_OP_PREFETCH_CFG:
cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
break;
case CMDQ_OP_CFGI_CD:
cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
fallthrough;
case CMDQ_OP_CFGI_STE:
cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
break;
case CMDQ_OP_CFGI_ALL:
/* Cover the entire SID range */
cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
break;
case CMDQ_OP_TLBI_NH_VA:
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
fallthrough;
@ -333,26 +316,6 @@ static int arm_smmu_cmdq_build_cmd(struct arm_smmu_cmd *cmd_out,
cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
break;
case CMDQ_OP_PRI_RESP:
cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
switch (ent->pri.resp) {
case PRI_RESP_DENY:
case PRI_RESP_FAIL:
case PRI_RESP_SUCC:
break;
default:
return -EINVAL;
}
cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
break;
case CMDQ_OP_RESUME:
cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
break;
case CMDQ_OP_CMD_SYNC:
if (ent->sync.msiaddr) {
cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
@ -924,25 +887,24 @@ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
return ret;
}
static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
struct arm_smmu_cmd *cmd,
bool sync)
static int arm_smmu_cmdq_issue_cmd_p(struct arm_smmu_device *smmu,
struct arm_smmu_cmd *cmd, bool sync)
{
return arm_smmu_cmdq_issue_cmdlist(
smmu, arm_smmu_get_cmdq(smmu, cmd), cmd, 1, sync);
}
static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
struct arm_smmu_cmd *cmd)
{
return __arm_smmu_cmdq_issue_cmd(smmu, cmd, false);
}
#define arm_smmu_cmdq_issue_cmd(smmu, cmd) \
({ \
struct arm_smmu_cmd __cmd = cmd; \
arm_smmu_cmdq_issue_cmd_p(smmu, &__cmd, false); \
})
static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
struct arm_smmu_cmd *cmd)
{
return __arm_smmu_cmdq_issue_cmd(smmu, cmd, true);
}
#define arm_smmu_cmdq_issue_cmd_with_sync(smmu, cmd) \
({ \
struct arm_smmu_cmd __cmd = cmd; \
arm_smmu_cmdq_issue_cmd_p(smmu, &__cmd, true); \
})
static void arm_smmu_cmdq_batch_init_cmd(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
@ -962,14 +924,41 @@ static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu,
arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd);
}
static void arm_smmu_cmdq_batch_add_cmd_p(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmd *cmd)
{
bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) &&
(smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC);
bool unsupported_cmd;
unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd);
if (force_sync || unsupported_cmd) {
arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
cmds->num, true);
arm_smmu_cmdq_batch_init_cmd(smmu, cmds, cmd);
}
if (cmds->num == CMDQ_BATCH_ENTRIES) {
arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
cmds->num, false);
arm_smmu_cmdq_batch_init_cmd(smmu, cmds, cmd);
}
cmds->cmds[cmds->num++] = *cmd;
}
#define arm_smmu_cmdq_batch_add_cmd(smmu, cmds, cmd) \
({ \
struct arm_smmu_cmd __cmd = cmd; \
arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, &__cmd); \
})
static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmdq_ent *ent)
{
bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) &&
(smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC);
struct arm_smmu_cmd cmd;
bool unsupported_cmd;
if (unlikely(arm_smmu_cmdq_build_cmd(&cmd, ent))) {
dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
@ -977,20 +966,7 @@ static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
return;
}
unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, &cmd);
if (force_sync || unsupported_cmd) {
arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
cmds->num, true);
arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd);
}
if (cmds->num == CMDQ_BATCH_ENTRIES) {
arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
cmds->num, false);
arm_smmu_cmdq_batch_init_cmd(smmu, cmds, &cmd);
}
cmds->cmds[cmds->num++] = cmd;
arm_smmu_cmdq_batch_add_cmd_p(smmu, cmds, &cmd);
}
static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
@ -1003,32 +979,29 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused,
struct iommu_page_response *resp)
{
struct arm_smmu_cmdq_ent cmd = {0};
struct arm_smmu_master *master = dev_iommu_priv_get(dev);
int sid = master->streams[0].id;
struct arm_smmu_cmd hw_cmd;
u8 resume_resp;
if (WARN_ON(!master->stall_enabled))
return;
cmd.opcode = CMDQ_OP_RESUME;
cmd.resume.sid = sid;
cmd.resume.stag = resp->grpid;
switch (resp->code) {
case IOMMU_PAGE_RESP_INVALID:
case IOMMU_PAGE_RESP_FAILURE:
cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
resume_resp = CMDQ_RESUME_0_RESP_ABORT;
break;
case IOMMU_PAGE_RESP_SUCCESS:
cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
resume_resp = CMDQ_RESUME_0_RESP_RETRY;
break;
default:
resume_resp = CMDQ_RESUME_0_RESP_TERM;
break;
}
arm_smmu_cmdq_build_cmd(&hw_cmd, &cmd);
arm_smmu_cmdq_issue_cmd(master->smmu, &hw_cmd);
arm_smmu_cmdq_issue_cmd(master->smmu,
arm_smmu_make_cmd_resume(master->streams[0].id,
resp->grpid,
resume_resp));
/*
* Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
* RESUME consumption guarantees that the stalled transaction will be
@ -1552,19 +1525,14 @@ static void arm_smmu_sync_cd(struct arm_smmu_master *master,
size_t i;
struct arm_smmu_cmdq_batch cmds;
struct arm_smmu_device *smmu = master->smmu;
struct arm_smmu_cmdq_ent cmd = {
.opcode = CMDQ_OP_CFGI_CD,
.cfgi = {
.ssid = ssid,
.leaf = leaf,
},
};
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_cfgi_cd(0, ssid, leaf);
arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd);
for (i = 0; i < master->num_streams; i++) {
cmd.cfgi.sid = master->streams[i].id;
arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
}
arm_smmu_cmdq_batch_init_cmd(smmu, &cmds, &cmd);
for (i = 0; i < master->num_streams; i++)
arm_smmu_cmdq_batch_add_cmd(
smmu, &cmds,
arm_smmu_make_cmd_cfgi_cd(master->streams[i].id, ssid,
leaf));
arm_smmu_cmdq_batch_submit(smmu, &cmds);
}
@ -1857,17 +1825,10 @@ static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer)
{
struct arm_smmu_ste_writer *ste_writer =
container_of(writer, struct arm_smmu_ste_writer, writer);
struct arm_smmu_cmdq_ent ent = {
.opcode = CMDQ_OP_CFGI_STE,
.cfgi = {
.sid = ste_writer->sid,
.leaf = true,
},
};
struct arm_smmu_cmd cmd;
arm_smmu_cmdq_build_cmd(&cmd, &ent);
arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd);
arm_smmu_cmdq_issue_cmd_with_sync(
writer->master->smmu,
arm_smmu_make_cmd_cfgi_ste(ste_writer->sid, true));
}
static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = {
@ -1892,17 +1853,9 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data);
/* It's likely that we'll want to use the new STE soon */
if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) {
struct arm_smmu_cmdq_ent
prefetch_ent = { .opcode = CMDQ_OP_PREFETCH_CFG,
.prefetch = {
.sid = sid,
} };
struct arm_smmu_cmd prefetch_cmd;
arm_smmu_cmdq_build_cmd(&prefetch_cmd, &prefetch_ent);
arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
}
if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
arm_smmu_cmdq_issue_cmd(smmu,
arm_smmu_make_cmd_prefetch_cfg(sid));
}
void arm_smmu_make_abort_ste(struct arm_smmu_ste *target)
@ -2327,22 +2280,10 @@ static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
evt[1] & PRIQ_1_ADDR_MASK);
if (last) {
struct arm_smmu_cmdq_ent ent = {
.opcode = CMDQ_OP_PRI_RESP,
.substream_valid = ssv,
.pri = {
.sid = sid,
.ssid = ssid,
.grpid = grpid,
.resp = PRI_RESP_DENY,
},
};
struct arm_smmu_cmd cmd;
arm_smmu_cmdq_build_cmd(&cmd, &ent);
arm_smmu_cmdq_issue_cmd(smmu, &cmd);
}
if (last)
arm_smmu_cmdq_issue_cmd(
smmu, arm_smmu_make_cmd_pri_resp(sid, ssid, ssv, grpid,
PRI_RESP_DENY));
}
static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
@ -3464,7 +3405,7 @@ static void arm_smmu_inv_flush_iotlb_tag(struct arm_smmu_inv *inv)
cmd.opcode = inv->nsize_opcode;
arm_smmu_cmdq_build_cmd(&hw_cmd, &cmd);
arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, &hw_cmd);
arm_smmu_cmdq_issue_cmd_with_sync(inv->smmu, hw_cmd);
}
/* Should be installed after arm_smmu_install_ste_for_dev() */
@ -4827,8 +4768,6 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
{
int ret;
u32 reg, enables;
struct arm_smmu_cmdq_ent ent;
struct arm_smmu_cmd cmd;
/* Clear CR0 and sync (disables SMMU and queue processing) */
reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
@ -4875,20 +4814,16 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
}
/* Invalidate any cached configuration */
ent.opcode = CMDQ_OP_CFGI_ALL;
arm_smmu_cmdq_build_cmd(&cmd, &ent);
arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
arm_smmu_cmdq_issue_cmd_with_sync(smmu, arm_smmu_make_cmd_cfgi_all());
/* Invalidate any stale TLB entries */
if (smmu->features & ARM_SMMU_FEAT_HYP) {
ent.opcode = CMDQ_OP_TLBI_EL2_ALL;
arm_smmu_cmdq_build_cmd(&cmd, &ent);
arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
arm_smmu_cmdq_issue_cmd_with_sync(
smmu, arm_smmu_make_cmd_op(CMDQ_OP_TLBI_EL2_ALL));
}
ent.opcode = CMDQ_OP_TLBI_NSNH_ALL;
arm_smmu_cmdq_build_cmd(&cmd, &ent);
arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
arm_smmu_cmdq_issue_cmd_with_sync(
smmu, arm_smmu_make_cmd_op(CMDQ_OP_TLBI_NSNH_ALL));
/* Event queue */
writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);

109
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
View File

@ -437,6 +437,12 @@ struct arm_smmu_cmd {
#define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
#define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
enum pri_resp {
PRI_RESP_DENY = 0,
PRI_RESP_FAIL = 1,
PRI_RESP_SUCC = 2,
};
#define CMDQ_RESUME_0_RESP_TERM 0UL
#define CMDQ_RESUME_0_RESP_RETRY 1UL
#define CMDQ_RESUME_0_RESP_ABORT 2UL
@ -475,6 +481,77 @@ enum arm_smmu_cmdq_opcode {
CMDQ_OP_CMD_SYNC = 0x46,
};
static inline struct arm_smmu_cmd
arm_smmu_make_cmd_op(enum arm_smmu_cmdq_opcode op)
{
struct arm_smmu_cmd cmd = {};
cmd.data[0] = FIELD_PREP(CMDQ_0_OP, op);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_cfgi_all(void)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CFGI_ALL);
cmd.data[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_prefetch_cfg(u32 sid)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_PREFETCH_CFG);
cmd.data[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, sid);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_cfgi_ste(u32 sid, bool leaf)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CFGI_STE);
cmd.data[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid);
cmd.data[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, leaf);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_cfgi_cd(u32 sid, u32 ssid,
bool leaf)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_CFGI_CD);
cmd.data[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid) |
FIELD_PREP(CMDQ_CFGI_0_SSID, ssid);
cmd.data[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, leaf);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_resume(u32 sid, u16 stag,
u8 resp)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_RESUME);
cmd.data[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, sid) |
FIELD_PREP(CMDQ_RESUME_0_RESP, resp);
cmd.data[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, stag);
return cmd;
}
static inline struct arm_smmu_cmd arm_smmu_make_cmd_pri_resp(u32 sid, u32 ssid,
bool ssv,
u16 grpid,
enum pri_resp resp)
{
struct arm_smmu_cmd cmd = arm_smmu_make_cmd_op(CMDQ_OP_PRI_RESP);
cmd.data[0] |= FIELD_PREP(CMDQ_0_SSV, ssv) |
FIELD_PREP(CMDQ_PRI_0_SID, sid) |
FIELD_PREP(CMDQ_PRI_0_SSID, ssid);
cmd.data[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, grpid) |
FIELD_PREP(CMDQ_PRI_1_RESP, resp);
return cmd;
}
/* Event queue */
#define EVTQ_ENT_SZ_SHIFT 5
#define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
@ -535,12 +612,6 @@ enum arm_smmu_cmdq_opcode {
#define MSI_IOVA_BASE 0x8000000
#define MSI_IOVA_LENGTH 0x100000
enum pri_resp {
PRI_RESP_DENY = 0,
PRI_RESP_FAIL = 1,
PRI_RESP_SUCC = 2,
};
struct arm_smmu_cmdq_ent {
/* Common fields */
u8 opcode;
@ -548,19 +619,6 @@ struct arm_smmu_cmdq_ent {
/* Command-specific fields */
union {
struct {
u32 sid;
} prefetch;
struct {
u32 sid;
u32 ssid;
union {
bool leaf;
u8 span;
};
} cfgi;
struct {
u8 num;
u8 scale;
@ -580,19 +638,6 @@ struct arm_smmu_cmdq_ent {
bool global;
} atc;
struct {
u32 sid;
u32 ssid;
u16 grpid;
enum pri_resp resp;
} pri;
struct {
u32 sid;
u16 stag;
u8 resp;
} resume;
struct {
u64 msiaddr;
} sync;