diff --git a/drivers/net/ethernet/marvell/octeontx2/af/Makefile b/drivers/net/ethernet/marvell/octeontx2/af/Makefile index 244de500963e..91b7d6e96a61 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/Makefile +++ b/drivers/net/ethernet/marvell/octeontx2/af/Makefile @@ -13,4 +13,4 @@ rvu_af-y := cgx.o rvu.o rvu_cgx.o rvu_npa.o rvu_nix.o \ rvu_cpt.o rvu_devlink.o rpm.o rvu_cn10k.o rvu_switch.o \ rvu_sdp.o rvu_npc_hash.o mcs.o mcs_rvu_if.o mcs_cnf10kb.o \ rvu_rep.o cn20k/mbox_init.o cn20k/nix.o cn20k/debugfs.o \ - cn20k/npa.o + cn20k/npa.o cn20k/npc.o diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c index 498968bf4cf5..3debf2fae1a4 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c @@ -11,7 +11,280 @@ #include #include "struct.h" +#include "rvu.h" #include "debugfs.h" +#include "cn20k/npc.h" + +static int npc_mcam_layout_show(struct seq_file *s, void *unused) +{ + int i, j, sbd, idx0, idx1, vidx0, vidx1; + struct npc_priv_t *npc_priv; + char buf0[32], buf1[32]; + struct npc_subbank *sb; + unsigned int bw0, bw1; + bool v0, v1; + int pf1, pf2; + bool e0, e1; + void *map; + + npc_priv = s->private; + + sbd = npc_priv->subbank_depth; + + for (i = npc_priv->num_subbanks - 1; i >= 0; i--) { + sb = &npc_priv->sb[i]; + mutex_lock(&sb->lock); + + if (sb->flags & NPC_SUBBANK_FLAG_FREE) + goto next; + + bw0 = bitmap_weight(sb->b0map, npc_priv->subbank_depth); + if (sb->key_type == NPC_MCAM_KEY_X4) { + seq_printf(s, "\n\nsubbank:%u, x4, free=%u, used=%u\n", + sb->idx, sb->free_cnt, bw0); + + for (j = sbd - 1; j >= 0; j--) { + if (!test_bit(j, sb->b0map)) + continue; + + idx0 = sb->b0b + j; + map = xa_load(&npc_priv->xa_idx2pf_map, idx0); + pf1 = xa_to_value(map); + + map = xa_load(&npc_priv->xa_idx2vidx_map, idx0); + if (map) { + vidx0 = xa_to_value(map); + snprintf(buf0, sizeof(buf0), + "v:%u", vidx0); + } + + seq_printf(s, "\t%u(%#x) %s\n", idx0, pf1, + map ? buf0 : " "); + } + goto next; + } + + bw1 = bitmap_weight(sb->b1map, npc_priv->subbank_depth); + seq_printf(s, "\n\nsubbank:%u, x2, free=%u, used=%u\n", + sb->idx, sb->free_cnt, bw0 + bw1); + seq_printf(s, "bank1(%u)\t\tbank0(%u)\n", bw1, bw0); + + for (j = sbd - 1; j >= 0; j--) { + e0 = test_bit(j, sb->b0map); + e1 = test_bit(j, sb->b1map); + + if (!e1 && !e0) + continue; + + if (e1 && e0) { + idx0 = sb->b0b + j; + map = xa_load(&npc_priv->xa_idx2pf_map, idx0); + pf1 = xa_to_value(map); + + map = xa_load(&npc_priv->xa_idx2vidx_map, idx0); + v0 = !!map; + if (v0) { + vidx0 = xa_to_value(map); + snprintf(buf0, sizeof(buf0), "v:%05u", + vidx0); + } + + idx1 = sb->b1b + j; + map = xa_load(&npc_priv->xa_idx2pf_map, idx1); + pf2 = xa_to_value(map); + + map = xa_load(&npc_priv->xa_idx2vidx_map, idx1); + v1 = !!map; + if (v1) { + vidx1 = xa_to_value(map); + snprintf(buf1, sizeof(buf1), "v:%05u", + vidx1); + } + + seq_printf(s, "%05u(%#x) %s\t\t%05u(%#x) %s\n", + idx1, pf2, v1 ? buf1 : " ", + idx0, pf1, v0 ? buf0 : " "); + + continue; + } + + if (e0) { + idx0 = sb->b0b + j; + map = xa_load(&npc_priv->xa_idx2pf_map, idx0); + pf1 = xa_to_value(map); + + map = xa_load(&npc_priv->xa_idx2vidx_map, idx0); + if (map) { + vidx0 = xa_to_value(map); + snprintf(buf0, sizeof(buf0), "v:%05u", + vidx0); + } + + seq_printf(s, "\t\t \t\t%05u(%#x) %s\n", idx0, + pf1, map ? buf0 : " "); + continue; + } + + idx1 = sb->b1b + j; + map = xa_load(&npc_priv->xa_idx2pf_map, idx1); + pf1 = xa_to_value(map); + map = xa_load(&npc_priv->xa_idx2vidx_map, idx1); + if (map) { + vidx1 = xa_to_value(map); + snprintf(buf1, sizeof(buf1), "v:%05u", vidx1); + } + + seq_printf(s, "%05u(%#x) %s\n", idx1, pf1, + map ? buf1 : " "); + } +next: + mutex_unlock(&sb->lock); + } + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(npc_mcam_layout); + +static int npc_mcam_default_show(struct seq_file *s, void *unused) +{ + struct npc_priv_t *npc_priv; + unsigned long index; + u16 ptr[4], pcifunc; + struct rvu *rvu; + int rc, i; + void *map; + + npc_priv = npc_priv_get(); + rvu = s->private; + + seq_puts(s, "\npcifunc\tBcast\tmcast\tpromisc\tucast\n"); + + xa_for_each(&npc_priv->xa_pf_map, index, map) { + pcifunc = index; + + for (i = 0; i < ARRAY_SIZE(ptr); i++) + ptr[i] = USHRT_MAX; + + rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], + &ptr[1], &ptr[2], &ptr[3]); + if (rc) + continue; + + seq_printf(s, "%#x\t", pcifunc); + for (i = 0; i < ARRAY_SIZE(ptr); i++) { + if (ptr[i] != USHRT_MAX) + seq_printf(s, "%u\t", ptr[i]); + else + seq_puts(s, "\t"); + } + seq_puts(s, "\n"); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(npc_mcam_default); + +static int npc_vidx2idx_map_show(struct seq_file *s, void *unused) +{ + struct npc_priv_t *npc_priv; + unsigned long index, start; + struct xarray *xa; + void *map; + + npc_priv = s->private; + start = npc_priv->bank_depth * 2; + xa = &npc_priv->xa_vidx2idx_map; + + seq_puts(s, "\nvidx\tmcam_idx\n"); + + xa_for_each_start(xa, index, map, start) + seq_printf(s, "%lu\t%lu\n", index, xa_to_value(map)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(npc_vidx2idx_map); + +static int npc_idx2vidx_map_show(struct seq_file *s, void *unused) +{ + struct npc_priv_t *npc_priv; + unsigned long index; + struct xarray *xa; + void *map; + + npc_priv = s->private; + xa = &npc_priv->xa_idx2vidx_map; + + seq_puts(s, "\nmidx\tvidx\n"); + + xa_for_each(xa, index, map) + seq_printf(s, "%lu\t%lu\n", index, xa_to_value(map)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(npc_idx2vidx_map); + +static int npc_defrag_show(struct seq_file *s, void *unused) +{ + struct npc_defrag_show_node *node; + struct npc_priv_t *npc_priv; + u16 sbd, bdm; + + npc_priv = s->private; + bdm = npc_priv->bank_depth - 1; + sbd = npc_priv->subbank_depth; + + seq_puts(s, "\nold(sb) -> new(sb)\t\tvidx\n"); + + mutex_lock(&npc_priv->lock); + list_for_each_entry(node, &npc_priv->defrag_lh, list) + seq_printf(s, "%u(%u)\t%u(%u)\t%u\n", node->old_midx, + (node->old_midx & bdm) / sbd, + node->new_midx, + (node->new_midx & bdm) / sbd, + node->vidx); + mutex_unlock(&npc_priv->lock); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(npc_defrag); + +int npc_cn20k_debugfs_init(struct rvu *rvu) +{ + struct npc_priv_t *npc_priv = npc_priv_get(); + struct dentry *npc_dentry; + + npc_dentry = debugfs_create_file("mcam_layout", 0444, rvu->rvu_dbg.npc, + npc_priv, &npc_mcam_layout_fops); + + if (!npc_dentry) + return -EFAULT; + + npc_dentry = debugfs_create_file("mcam_default", 0444, rvu->rvu_dbg.npc, + rvu, &npc_mcam_default_fops); + + if (!npc_dentry) + return -EFAULT; + + npc_dentry = debugfs_create_file("vidx2idx", 0444, rvu->rvu_dbg.npc, + npc_priv, &npc_vidx2idx_map_fops); + if (!npc_dentry) + return -EFAULT; + + npc_dentry = debugfs_create_file("idx2vidx", 0444, rvu->rvu_dbg.npc, + npc_priv, &npc_idx2vidx_map_fops); + if (!npc_dentry) + return -EFAULT; + + npc_dentry = debugfs_create_file("defrag", 0444, rvu->rvu_dbg.npc, + npc_priv, &npc_defrag_fops); + if (!npc_dentry) + return -EFAULT; + + return 0; +} + +void npc_cn20k_debugfs_deinit(struct rvu *rvu) +{ + debugfs_remove_recursive(rvu->rvu_dbg.npc); +} void print_nix_cn20k_sq_ctx(struct seq_file *m, struct nix_cn20k_sq_ctx_s *sq_ctx) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h index a2e3a2cd6edb..0c5f05883666 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h @@ -16,6 +16,9 @@ #include "struct.h" #include "../mbox.h" +int npc_cn20k_debugfs_init(struct rvu *rvu); +void npc_cn20k_debugfs_deinit(struct rvu *rvu); + void print_nix_cn20k_sq_ctx(struct seq_file *m, struct nix_cn20k_sq_ctx_s *sq_ctx); void print_nix_cn20k_cq_ctx(struct seq_file *m, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c index bd3aab7770dd..71401dec0d77 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c @@ -397,6 +397,12 @@ int rvu_alloc_cint_qint_mem(struct rvu *rvu, struct rvu_pfvf *pfvf, if (is_rvu_otx2(rvu) || is_cn20k(rvu->pdev)) return 0; + /* sanity check */ + cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_NIXX_CFG(0) | + (RVU_AFPF << 16)); + if (!cfg) + return 0; + ctx_cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST3); /* Alloc memory for CQINT's HW contexts */ cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); @@ -420,5 +426,16 @@ int rvu_alloc_cint_qint_mem(struct rvu *rvu, struct rvu_pfvf *pfvf, rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_BASE(nixlf), (u64)pfvf->nix_qints_ctx->iova); + rvu_write64(rvu, BLKADDR_NIX0, RVU_AF_BAR2_SEL, RVU_AF_BAR2_PFID); + rvu_write64(rvu, BLKADDR_NIX0, + AF_BAR2_ALIASX(0, NIX_GINT_INT_W1S), ALTAF_RDY); + /* wait for ack */ + err = rvu_poll_reg(rvu, BLKADDR_NIX0, + AF_BAR2_ALIASX(0, NIX_GINT_INT), ALTAF_RDY, true); + if (err) + rvu->altaf_ready = false; + else + rvu->altaf_ready = true; + return 0; } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c new file mode 100644 index 000000000000..7291fdb89b03 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c @@ -0,0 +1,4522 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell RVU Admin Function driver + * + * Copyright (C) 2026 Marvell. + * + */ +#include +#include + +#include "rvu.h" +#include "npc.h" +#include "npc_profile.h" +#include "rvu_npc_hash.h" +#include "rvu_npc.h" +#include "cn20k/npc.h" +#include "cn20k/reg.h" +#include "rvu_npc_fs.h" + +static struct npc_priv_t npc_priv = { + .num_banks = MAX_NUM_BANKS, +}; + +static const char *npc_kw_name[NPC_MCAM_KEY_MAX] = { + [NPC_MCAM_KEY_DYN] = "DYNAMIC", + [NPC_MCAM_KEY_X2] = "X2", + [NPC_MCAM_KEY_X4] = "X4", +}; + +static const char *npc_dft_rule_name[NPC_DFT_RULE_MAX_ID] = { + [NPC_DFT_RULE_PROMISC_ID] = "Promisc", + [NPC_DFT_RULE_MCAST_ID] = "Mcast", + [NPC_DFT_RULE_BCAST_ID] = "Bcast", + [NPC_DFT_RULE_UCAST_ID] = "Ucast", +}; + +#define KEX_EXTR_CFG(bytesm1, hdr_ofs, ena, key_ofs) \ + (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \ + ((key_ofs) & 0x3F)) + +#define NPC_DFT_RULE_ID_MK(pcifunc, id) \ + ((pcifunc) | FIELD_PREP(GENMASK_ULL(31, 16), id)) + +#define NPC_DFT_RULE_ID_2_PCIFUNC(rid) \ + FIELD_GET(GENMASK_ULL(15, 0), rid) + +#define NPC_DFT_RULE_ID_2_ID(rid) \ + FIELD_GET(GENMASK_ULL(31, 16), rid) + +#define NPC_DFT_RULE_PRIO 127 + +static const char cn20k_def_pfl_name[] = "default"; + +static struct npc_mcam_kex_extr npc_mkex_extr_default = { + .mkex_sign = MKEX_CN20K_SIGN, + .name = "default", + .kpu_version = NPC_KPU_PROFILE_VER, + .keyx_cfg = { + /* nibble: LA..LE (ltype only) + Error code + Channel */ + [NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_DYN << 32) | + NPC_PARSE_NIBBLE_INTF_RX | + NPC_CN20K_PARSE_NIBBLE_ERRCODE, + + /* nibble: LA..LE (ltype only) */ + [NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) | + NPC_CN20K_PARSE_NIBBLE_INTF_TX, + }, + .intf_extr_lid = { + /* Default RX MCAM KEX profile */ + [NIX_INTF_RX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, + NPC_LID_LC, NPC_LID_LC, NPC_LID_LD }, + [NIX_INTF_TX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, + NPC_LID_LC, NPC_LID_LD }, + }, + .intf_extr_lt = { + /* Default RX MCAM KEX profile */ + [NIX_INTF_RX] = { + [0] = { + /* Layer A: Ethernet: */ + [NPC_LT_LA_ETHER] = + /* DMAC: 6 bytes, KW1[63:15] */ + KEX_EXTR_CFG(0x05, 0x0, 0x1, + NPC_KEXOF_DMAC + 1), + [NPC_LT_LA_CPT_HDR] = + /* DMAC: 6 bytes, KW1[63:15] */ + KEX_EXTR_CFG(0x05, 0x0, 0x1, + NPC_KEXOF_DMAC + 1), + }, + [1] = { + /* Layer A: Ethernet: */ + [NPC_LT_LA_ETHER] = + /* Ethertype: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), + [NPC_LT_LA_CPT_HDR] = + /* Ethertype: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), + }, + [2] = { + /* Layer B: Single VLAN (CTAG) */ + [NPC_LT_LB_CTAG] = + /* CTAG VLAN: 2 bytes, KW1[15:0] */ + KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), + /* Layer B: Stacked VLAN (STAG|QinQ) */ + [NPC_LT_LB_STAG_QINQ] = + /* Outer VLAN: 2 bytes, KW1[15:0] */ + KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), + [NPC_LT_LB_FDSA] = + /* SWITCH PORT: 1 byte, KW1[7:0] */ + KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x8), + }, + [3] = { + [NPC_LT_LB_CTAG] = + /* Ethertype: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), + [NPC_LT_LB_STAG_QINQ] = + /* Ethertype: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x6), + [NPC_LT_LB_FDSA] = + /* Ethertype: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), + }, + [4] = { + /* Layer C: IPv4 */ + [NPC_LT_LC_IP] = + /* SIP+DIP: 8 bytes, KW3[7:0], KW2[63:8] */ + KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x11), + /* Layer C: IPv6 */ + [NPC_LT_LC_IP6] = + /* Everything up to SADDR: 8 bytes, KW3[7:0], + * KW2[63:8] + */ + KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x11), + }, + [5] = { + [NPC_LT_LC_IP] = + /* TOS: 1 byte, KW2[7:0] */ + KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x10), + }, + [6] = { + /* Layer D:UDP */ + [NPC_LT_LD_UDP] = + /* SPORT+DPORT: 4 bytes, KW3[39:8] */ + KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), + /* Layer D:TCP */ + [NPC_LT_LD_TCP] = + /* SPORT+DPORT: 4 bytes, KW3[39:8] */ + KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), + }, + }, + /* Default TX MCAM KEX profile */ + [NIX_INTF_TX] = { + [0] = { + /* Layer A: NIX_INST_HDR_S + Ethernet */ + /* NIX appends 8 bytes of NIX_INST_HDR_S at the + * start of each TX packet supplied to NPC. + */ + [NPC_LT_LA_IH_NIX_ETHER] = + /* PF_FUNC: 2B , KW0 [47:32] */ + KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), + /* Layer A: HiGig2: */ + [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = + /* PF_FUNC: 2B , KW0 [47:32] */ + KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), + }, + [1] = { + [NPC_LT_LA_IH_NIX_ETHER] = + /* SQ_ID 3 bytes, KW1[63:16] */ + KEX_EXTR_CFG(0x02, 0x02, 0x1, 0xa), + [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = + /* VID: 2 bytes, KW1[31:16] */ + KEX_EXTR_CFG(0x01, 0x10, 0x1, 0xa), + }, + [2] = { + /* Layer B: Single VLAN (CTAG) */ + [NPC_LT_LB_CTAG] = + /* CTAG VLAN[2..3] KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), + /* Layer B: Stacked VLAN (STAG|QinQ) */ + [NPC_LT_LB_STAG_QINQ] = + /* Outer VLAN: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), + }, + [3] = { + [NPC_LT_LB_CTAG] = + /* CTAG VLAN[2..3] KW1[15:0] */ + KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x8), + [NPC_LT_LB_STAG_QINQ] = + /* Outer VLAN: 2 Bytes, KW1[15:0] */ + KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x8), + }, + [4] = { + /* Layer C: IPv4 */ + [NPC_LT_LC_IP] = + /* SIP+DIP: 8 bytes, KW2[63:0] */ + KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x10), + /* Layer C: IPv6 */ + [NPC_LT_LC_IP6] = + /* Everything up to SADDR: 8 bytes, KW2[63:0] */ + KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x10), + }, + [5] = { + /* Layer D:UDP */ + [NPC_LT_LD_UDP] = + /* SPORT+DPORT: 4 bytes, KW3[31:0] */ + KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), + /* Layer D:TCP */ + [NPC_LT_LD_TCP] = + /* SPORT+DPORT: 4 bytes, KW3[31:0] */ + KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), + }, + }, + }, +}; + +struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void) +{ + return &npc_mkex_extr_default; +} + +static u16 npc_idx2vidx(u16 idx) +{ + unsigned long index; + void *map; + u16 vidx; + int val; + + vidx = idx; + index = idx; + + map = xa_load(&npc_priv.xa_idx2vidx_map, index); + if (!map) + goto done; + + val = xa_to_value(map); + if (val == -1) + goto done; + + vidx = val; + +done: + return vidx; +} + +static bool npc_is_vidx(u16 vidx) +{ + return vidx >= npc_priv.bank_depth * 2; +} + +static u16 npc_vidx2idx(u16 vidx) +{ + unsigned long sentinel = (unsigned long)-1; + unsigned long index; + void *map; + int val; + u16 idx; + + idx = vidx; + index = vidx; + + map = xa_load(&npc_priv.xa_vidx2idx_map, index); + if (!map) + goto done; + + val = xa_to_value(map); + if (val == sentinel) + goto done; + + idx = val; + +done: + return idx; +} + +u16 npc_cn20k_vidx2idx(u16 idx) +{ + if (!npc_priv.init_done) + return idx; + + if (!npc_is_vidx(idx)) + return idx; + + return npc_vidx2idx(idx); +} + +u16 npc_cn20k_idx2vidx(u16 idx) +{ + if (!npc_priv.init_done) + return idx; + + if (npc_is_vidx(idx)) + return idx; + + return npc_idx2vidx(idx); +} + +static int npc_vidx_maps_del_entry(struct rvu *rvu, u16 vidx, u16 *old_midx) +{ + u16 mcam_idx; + void *map; + + if (!npc_is_vidx(vidx)) { + dev_err(rvu->dev, + "%s: vidx(%u) does not map to proper mcam idx\n", + __func__, vidx); + return -ESRCH; + } + + mcam_idx = npc_vidx2idx(vidx); + + map = xa_erase(&npc_priv.xa_vidx2idx_map, vidx); + if (!map) { + dev_err(rvu->dev, + "%s: vidx(%u) does not map to proper mcam idx\n", + __func__, vidx); + return -ESRCH; + } + + map = xa_erase(&npc_priv.xa_idx2vidx_map, mcam_idx); + if (!map) { + dev_err(rvu->dev, + "%s: vidx(%u) is not valid\n", + __func__, vidx); + return -ESRCH; + } + + if (old_midx) + *old_midx = mcam_idx; + + return 0; +} + +static int npc_vidx_maps_modify(struct rvu *rvu, u16 vidx, u16 new_midx) +{ + u16 old_midx; + void *map; + int rc; + + if (!npc_is_vidx(vidx)) { + dev_err(rvu->dev, + "%s: vidx(%u) does not map to proper mcam idx\n", + __func__, vidx); + return -ESRCH; + } + + map = xa_erase(&npc_priv.xa_vidx2idx_map, vidx); + if (!map) { + dev_err(rvu->dev, + "%s: vidx(%u) could not be deleted from vidx2idx map\n", + __func__, vidx); + return -ESRCH; + } + + old_midx = xa_to_value(map); + + rc = xa_insert(&npc_priv.xa_vidx2idx_map, vidx, + xa_mk_value(new_midx), GFP_KERNEL); + if (rc) { + dev_err(rvu->dev, + "%s: vidx(%u) cannot be added to vidx2idx map\n", + __func__, vidx); + goto fail1; + } + + map = xa_erase(&npc_priv.xa_idx2vidx_map, old_midx); + if (!map) { + dev_err(rvu->dev, + "%s: old_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", + __func__, old_midx, vidx); + rc = -ESRCH; + goto fail2; + } + + rc = xa_insert(&npc_priv.xa_idx2vidx_map, new_midx, + xa_mk_value(vidx), GFP_KERNEL); + if (rc) { + dev_err(rvu->dev, + "%s: new_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", + __func__, new_midx, vidx); + goto fail3; + } + + return 0; + +fail3: + /* Restore vidx at old_midx location */ + if (xa_insert(&npc_priv.xa_idx2vidx_map, old_midx, + xa_mk_value(vidx), GFP_KERNEL)) + dev_err(rvu->dev, + "%s: Error to roll back idx2vidx old_midx=%u vidx=%u\n", + __func__, old_midx, vidx); +fail2: + /* Erase new_midx inserted at vidx */ + if (!xa_erase(&npc_priv.xa_vidx2idx_map, vidx)) + dev_err(rvu->dev, + "%s: Failed to roll back vidx2idx vidx=%u\n", + __func__, vidx); + +fail1: + /* Restore old_midx at vidx location */ + if (xa_insert(&npc_priv.xa_vidx2idx_map, vidx, + xa_mk_value(old_midx), GFP_KERNEL)) + dev_err(rvu->dev, + "%s: Failed to roll back vidx2idx to old_midx=%u, vidx=%u\n", + __func__, old_midx, vidx); + + return rc; +} + +static int npc_vidx_maps_add_entry(struct rvu *rvu, u16 mcam_idx, int pcifunc, + u16 *vidx) +{ + int rc, max, min; + u32 id; + + /* Virtual index start from maximum mcam index + 1 */ + max = npc_priv.bank_depth * 2 * 2 - 1; + min = npc_priv.bank_depth * 2; + + rc = xa_alloc(&npc_priv.xa_vidx2idx_map, &id, + xa_mk_value(mcam_idx), + XA_LIMIT(min, max), GFP_KERNEL); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to add to vidx2idx map (%u)\n", + __func__, mcam_idx); + goto fail1; + } + + rc = xa_insert(&npc_priv.xa_idx2vidx_map, mcam_idx, + xa_mk_value(id), GFP_KERNEL); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to add to idx2vidx map (%u)\n", + __func__, mcam_idx); + goto fail2; + } + + if (vidx) + *vidx = id; + + return 0; + +fail2: + xa_erase(&npc_priv.xa_vidx2idx_map, id); +fail1: + return rc; +} + +static void npc_config_kpmcam(struct rvu *rvu, int blkaddr, + const struct npc_kpu_profile_cam *kpucam, + int kpm, int entry) +{ + struct npc_kpu_cam cam0 = {0}; + struct npc_kpu_cam cam1 = {0}; + + cam1.state = kpucam->state & kpucam->state_mask; + cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; + cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; + cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; + + cam0.state = ~kpucam->state & kpucam->state_mask; + cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; + cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; + cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; + + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 0), *(u64 *)&cam0); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 1), *(u64 *)&cam1); +} + +static void +npc_config_kpmaction(struct rvu *rvu, int blkaddr, + const struct npc_kpu_profile_action *kpuaction, + int kpm, int entry, bool pkind) +{ + struct npc_kpm_action0 action0 = {0}; + struct npc_kpu_action1 action1 = {0}; + u64 reg; + + action1.errlev = kpuaction->errlev; + action1.errcode = kpuaction->errcode; + action1.dp0_offset = kpuaction->dp0_offset; + action1.dp1_offset = kpuaction->dp1_offset; + action1.dp2_offset = kpuaction->dp2_offset; + + if (pkind) + reg = NPC_AF_PKINDX_ACTION1(entry); + else + reg = NPC_AF_KPMX_ENTRYX_ACTION1(kpm, entry); + + rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); + + action0.byp_count = kpuaction->bypass_count & 0x7; + action0.capture_ena = kpuaction->cap_ena & 1; + action0.parse_done = kpuaction->parse_done & 1; + action0.next_state = kpuaction->next_state & 0xf; + action0.capture_lid = kpuaction->lid & 0x7; + + /* Parser functionality will work correctly even though + * upper flag bits are silently discarded + */ + action0.capture_ltype = kpuaction->ltype & 0xf; + action0.capture_flags = kpuaction->flags & 0xf; + action0.ptr_advance = kpuaction->ptr_advance; + + action0.var_len_offset = kpuaction->offset; + action0.var_len_mask = kpuaction->mask; + action0.var_len_right = kpuaction->right & 1; + action0.var_len_shift = kpuaction->shift & 1; + + if (pkind) + reg = NPC_AF_PKINDX_ACTION0(entry); + else + reg = NPC_AF_KPMX_ENTRYX_ACTION0(kpm, entry); + + rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); +} + +static void +npc_program_single_kpm_profile(struct rvu *rvu, int blkaddr, + int kpm, int start_entry, + const struct npc_kpu_profile *profile) +{ + int entry, num_entries, max_entries; + u64 idx; + + if (profile->cam_entries != profile->action_entries) { + dev_err(rvu->dev, + "kpm%d: CAM and action entries [%d != %d] not equal\n", + kpm, profile->cam_entries, profile->action_entries); + + WARN(1, "Fatal error\n"); + return; + } + + max_entries = rvu->hw->npc_kpu_entries / 2; + entry = start_entry; + /* Program CAM match entries for previous kpm extracted data */ + num_entries = min_t(int, profile->cam_entries, max_entries); + for (idx = 0; entry < num_entries + start_entry; entry++, idx++) + npc_config_kpmcam(rvu, blkaddr, &profile->cam[idx], + kpm, entry); + + entry = start_entry; + /* Program this kpm's actions */ + num_entries = min_t(int, profile->action_entries, max_entries); + for (idx = 0; entry < num_entries + start_entry; entry++, idx++) + npc_config_kpmaction(rvu, blkaddr, &profile->action[idx], + kpm, entry, false); +} + +static void +npc_enable_kpm_entry(struct rvu *rvu, int blkaddr, int kpm, int num_entries) +{ + u64 entry_mask; + + entry_mask = npc_enable_mask(num_entries); + /* Disable first KPU_CN20K_MAX_CST_ENT entries for built-in profile */ + if (!rvu->kpu.custom) + entry_mask |= GENMASK_ULL(KPU_CN20K_MAX_CST_ENT - 1, 0); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 0), entry_mask); + if (num_entries <= 64) { + /* Disable all the entries in W1, W2 and W3 */ + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 1), + npc_enable_mask(0)); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 2), + npc_enable_mask(0)); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 3), + npc_enable_mask(0)); + return; + } + + num_entries = num_entries - 64; + entry_mask = npc_enable_mask(num_entries); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 1), entry_mask); + if (num_entries <= 64) { + /* Disable all the entries in W2 and W3 */ + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 2), + npc_enable_mask(0)); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 3), + npc_enable_mask(0)); + return; + } + + num_entries = num_entries - 64; + entry_mask = npc_enable_mask(num_entries); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 2), entry_mask); + if (num_entries <= 64) { + /* Disable all the entries in W3 */ + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 3), + npc_enable_mask(0)); + return; + } + + num_entries = num_entries - 64; + entry_mask = npc_enable_mask(num_entries); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 3), entry_mask); +} + +#define KPU_OFFSET 8 +static void npc_program_kpm_profile(struct rvu *rvu, int blkaddr, int num_kpms) +{ + const struct npc_kpu_profile *profile1, *profile2; + int idx, total_cam_entries; + + for (idx = 0; idx < num_kpms; idx++) { + profile1 = &rvu->kpu.kpu[idx]; + npc_program_single_kpm_profile(rvu, blkaddr, idx, 0, profile1); + profile2 = &rvu->kpu.kpu[idx + KPU_OFFSET]; + npc_program_single_kpm_profile(rvu, blkaddr, idx, + profile1->cam_entries, + profile2); + total_cam_entries = profile1->cam_entries + + profile2->cam_entries; + npc_enable_kpm_entry(rvu, blkaddr, idx, total_cam_entries); + rvu_write64(rvu, blkaddr, NPC_AF_KPMX_PASS2_OFFSET(idx), + profile1->cam_entries); + /* Enable the KPUs associated with this KPM */ + rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x01); + rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx + KPU_OFFSET), + 0x01); + } +} + +void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr) +{ + struct rvu_hwinfo *hw = rvu->hw; + int num_pkinds, idx; + + /* Disable all KPMs and their entries */ + for (idx = 0; idx < hw->npc_kpms; idx++) { + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(idx, 0), ~0ULL); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(idx, 1), ~0ULL); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(idx, 2), ~0ULL); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(idx, 3), ~0ULL); + } + + for (idx = 0; idx < hw->npc_kpus; idx++) + rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); + + /* Load and customize KPU profile. */ + npc_load_kpu_profile(rvu); + + /* Configure KPU and KPM mapping for second pass */ + rvu_write64(rvu, blkaddr, NPC_AF_KPM_PASS2_CFG, 0x76543210); + + /* First program IKPU profile i.e PKIND configs. + * Check HW max count to avoid configuring junk or + * writing to unsupported CSR addresses. + */ + num_pkinds = rvu->kpu.pkinds; + num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); + + for (idx = 0; idx < num_pkinds; idx++) + npc_config_kpmaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], + 0, idx, true); + + /* Program KPM CAM and Action profiles */ + npc_program_kpm_profile(rvu, blkaddr, hw->npc_kpms); +} + +struct npc_priv_t *npc_priv_get(void) +{ + return &npc_priv; +} + +static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, + struct npc_mcam_kex_extr *mkex_extr, + u8 intf) +{ + u8 num_extr = rvu->hw->npc_kex_extr; + int extr, lt; + u64 val; + + if (is_npc_intf_tx(intf)) + return; + + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), + mkex_extr->keyx_cfg[NIX_INTF_RX]); + + /* Program EXTRACTOR */ + for (extr = 0; extr < num_extr; extr++) + rvu_write64(rvu, blkaddr, + NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), + mkex_extr->intf_extr_lid[intf][extr]); + + /* Program EXTRACTOR_LTYPE */ + for (extr = 0; extr < num_extr; extr++) { + for (lt = 0; lt < NPC_MAX_LT; lt++) { + val = mkex_extr->intf_extr_lt[intf][extr][lt]; + CN20K_SET_EXTR_LT(intf, extr, lt, val); + } + } +} + +static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, + struct npc_mcam_kex_extr *mkex_extr, + u8 intf) +{ + u8 num_extr = rvu->hw->npc_kex_extr; + int extr, lt; + u64 val; + + if (is_npc_intf_rx(intf)) + return; + + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), + mkex_extr->keyx_cfg[NIX_INTF_TX]); + + /* Program EXTRACTOR */ + for (extr = 0; extr < num_extr; extr++) + rvu_write64(rvu, blkaddr, + NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), + mkex_extr->intf_extr_lid[intf][extr]); + + /* Program EXTRACTOR_LTYPE */ + for (extr = 0; extr < num_extr; extr++) { + for (lt = 0; lt < NPC_MAX_LT; lt++) { + val = mkex_extr->intf_extr_lt[intf][extr][lt]; + CN20K_SET_EXTR_LT(intf, extr, lt, val); + } + } +} + +static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, + struct npc_mcam_kex_extr *mkex_extr) +{ + struct rvu_hwinfo *hw = rvu->hw; + u8 intf; + + for (intf = 0; intf < hw->npc_intfs; intf++) { + npc_program_mkex_rx(rvu, blkaddr, mkex_extr, intf); + npc_program_mkex_tx(rvu, blkaddr, mkex_extr, intf); + } + + /* Programme mkex hash profile */ + npc_program_mkex_hash(rvu, blkaddr); +} + +void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr, + const char *mkex_profile) +{ + struct npc_mcam_kex_extr *mcam_kex_extr; + struct device *dev = &rvu->pdev->dev; + void __iomem *mkex_prfl_addr = NULL; + u64 prfl_sz; + int ret; + + /* If user not selected mkex profile */ + if (rvu->kpu_fwdata_sz || + !strncmp(mkex_profile, cn20k_def_pfl_name, MKEX_NAME_LEN)) + goto program_mkex_extr; + + /* Setting up the mapping for mkex profile image */ + ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); + if (ret < 0) + goto program_mkex_extr; + + mcam_kex_extr = (struct npc_mcam_kex_extr __force *)mkex_prfl_addr; + + while (((s64)prfl_sz > 0) && + (mcam_kex_extr->mkex_sign != MKEX_END_SIGN)) { + /* Compare with mkex mod_param name string */ + if (mcam_kex_extr->mkex_sign == MKEX_CN20K_SIGN && + !strncmp(mcam_kex_extr->name, mkex_profile, + MKEX_NAME_LEN)) { + rvu->kpu.mcam_kex_prfl.mkex_extr = mcam_kex_extr; + goto program_mkex_extr; + } + + mcam_kex_extr++; + prfl_sz -= sizeof(struct npc_mcam_kex_extr); + } + dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); + rvu->kpu.mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get(); + +program_mkex_extr: + dev_info(rvu->dev, "Using %s mkex profile\n", + rvu->kpu.mcam_kex_prfl.mkex_extr->name); + /* Program selected mkex profile */ + npc_program_mkex_profile(rvu, blkaddr, + rvu->kpu.mcam_kex_prfl.mkex_extr); + if (mkex_prfl_addr) + iounmap(mkex_prfl_addr); +} + +void +npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr, + int index, bool enable) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + int mcam_idx = index % mcam->banksize; + int bank = index / mcam->banksize; + u64 cfg, hw_prio; + u8 kw_type; + + npc_mcam_idx_2_key_type(rvu, index, &kw_type); + if (kw_type == NPC_MCAM_KEY_X2) { + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, + bank)); + hw_prio = cfg & GENMASK_ULL(30, 24); + cfg = enable ? 1 : 0; + cfg |= hw_prio; + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), + cfg); + return; + } + + /* For NPC_CN20K_MCAM_KEY_X4 keys, both the banks + * need to be programmed with the same value. + */ + for (bank = 0; bank < mcam->banks_per_entry; bank++) { + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, + bank)); + hw_prio = cfg & GENMASK_ULL(30, 24); + cfg = enable ? 1 : 0; + cfg |= hw_prio; + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), + cfg); + } +} + +void +npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int bank, int index) +{ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), + 0); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), + 0); + + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 0); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 0); + + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 0); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 0); + + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 0); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 0); + + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 0); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 0); + + /* Clear corresponding stats register */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0); +} + +static void npc_cn20k_get_keyword(struct cn20k_mcam_entry *entry, int idx, + u64 *cam0, u64 *cam1) +{ + u64 kw_mask; + + /* The two banks of every MCAM entry are used as a single double-wide + * entry that is compared with the search key as follows: + * + * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW0] + * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW1] + * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW2] + * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW3] + * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW4] + * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW5] + * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW6] + * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW7] + */ + *cam1 = entry->kw[idx]; + kw_mask = entry->kw_mask[idx]; + *cam1 &= kw_mask; + *cam0 = ~*cam1 & kw_mask; +} + +/*------------------------------------------------------------------------- + *Kex type| mcam | cam1 |cam0 | req_kwtype||<----- output > | + * in | | | | || | | + * profile| len | | | ||len | type | + *------------------------------------------------------------------------- + *X2 | 256 (X2) | 001b |110b | 0 ||X2 | X2 | + *------------------------------------------------------------------------| + *X4 | 256 (X2) | 000b |000b | 0 ||X2 | DYN | + *------------------------------------------------------------------------| + *X4 | 512 (X4) | 010b |101b | 0 ||X4 | X4 | + *------------------------------------------------------------------------| + *DYN | 256 (X2) | 000b |000b | 0 ||X2 | DYN | + *------------------------------------------------------------------------| + *DYN | 512 (X4) | 010b |101b | 0 ||X4 | X4 | + *------------------------------------------------------------------------| + *X4 | 256 (X2) | 000b |000b | X2 ||DYN | DYN | + *------------------------------------------------------------------------| + *DYNC | 256 (X2) | 000b |000b | X2 ||DYN | DYN | + *------------------------------------------------------------------------| + * X2 | 512 (X4) | xxxb |xxxb | X4 ||INVAL | INVAL | + *------------------------------------------------------------------------| + */ +static void npc_cn20k_config_kw_x2(struct rvu *rvu, struct npc_mcam *mcam, + int blkaddr, int index, u8 intf, + struct cn20k_mcam_entry *entry, + int bank, u8 kw_type, int kw, + u8 req_kw_type) +{ + u64 intf_ext = 0, intf_ext_mask = 0; + u8 tx_intf_mask = ~intf & 0x3; + u8 tx_intf = intf, kex_type; + u8 kw_type_mask = ~kw_type; + u64 cam0, cam1, kex_cfg; + + if (is_npc_intf_tx(intf)) { + /* Last bit must be set and rest don't care + * for TX interfaces + */ + tx_intf_mask = 0x1; + tx_intf = intf & tx_intf_mask; + tx_intf_mask = ~tx_intf & tx_intf_mask; + } + + kex_cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); + kex_type = (kex_cfg & GENMASK_ULL(34, 32)) >> 32; + if ((kex_type == NPC_MCAM_KEY_DYN || kex_type == NPC_MCAM_KEY_X4) && + kw_type == NPC_MCAM_KEY_X2) { + kw_type = 0; + kw_type_mask = 0; + } + + /* Say, we need to write x2 keyword in an x4 subbank. + * req_kw_type will be x2, and kw_type will be x4. + * So in the case ignore kw bits in mcam. + */ + if (kw_type == NPC_MCAM_KEY_X4 && req_kw_type == NPC_MCAM_KEY_X2) { + kw_type = 0; + kw_type_mask = 0; + } + + intf_ext = ((u64)kw_type << 16) | tx_intf; + intf_ext_mask = (((u64)kw_type_mask << 16) & GENMASK_ULL(18, 16)) | + tx_intf_mask; + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), + intf_ext); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), + intf_ext_mask); + + /* Set the match key */ + npc_cn20k_get_keyword(entry, kw, &cam0, &cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), + cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), + cam0); + + npc_cn20k_get_keyword(entry, kw + 1, &cam0, &cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), + cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), + cam0); + + npc_cn20k_get_keyword(entry, kw + 2, &cam0, &cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), + cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), + cam0); + + npc_cn20k_get_keyword(entry, kw + 3, &cam0, &cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), + cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), + cam0); +} + +static void npc_cn20k_config_kw_x4(struct rvu *rvu, struct npc_mcam *mcam, + int blkaddr, int index, u8 intf, + struct cn20k_mcam_entry *entry, + u8 kw_type, u8 req_kw_type) +{ + int kw = 0, bank; + + for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) + npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, + index, intf, + entry, bank, kw_type, + kw, req_kw_type); +} + +static void +npc_cn20k_set_mcam_bank_cfg(struct rvu *rvu, int blkaddr, int mcam_idx, + int bank, u8 kw_type, bool enable, u8 hw_prio) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + u64 bank_cfg; + + bank_cfg = (u64)hw_prio << 24; + if (enable) + bank_cfg |= 0x1; + + if (kw_type == NPC_MCAM_KEY_X2) { + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), + bank_cfg); + return; + } + + /* For NPC_MCAM_KEY_X4 keys, both the banks + * need to be programmed with the same value. + */ + for (bank = 0; bank < mcam->banks_per_entry; bank++) { + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), + bank_cfg); + } +} + +void npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index, + u8 intf, struct cn20k_mcam_entry *entry, + bool enable, u8 hw_prio, u8 req_kw_type) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + int mcam_idx = index % mcam->banksize; + int bank = index / mcam->banksize; + int kw = 0; + u8 kw_type; + + /* Disable before mcam entry update */ + npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, false); + + npc_mcam_idx_2_key_type(rvu, index, &kw_type); + /* CAM1 takes the comparison value and + * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. + * CAM1 = 0 & CAM0 = 1 => match if key = 0 + * CAM1 = 1 & CAM0 = 0 => match if key = 1 + * CAM1 = 0 & CAM0 = 0 => always match i.e dontcare. + */ + if (kw_type == NPC_MCAM_KEY_X2) { + /* Clear mcam entry to avoid writes being suppressed by NPC */ + npc_cn20k_clear_mcam_entry(rvu, blkaddr, bank, mcam_idx); + npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, + mcam_idx, intf, entry, + bank, kw_type, kw, req_kw_type); + /* Set 'action' */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 0), + entry->action); + + /* Set 'action2' for inline receive */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 2), + entry->action2); + + /* Set TAG 'action' */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 1), + entry->vtag_action); + goto set_cfg; + } + + /* Clear mcam entry to avoid writes being suppressed by NPC */ + npc_cn20k_clear_mcam_entry(rvu, blkaddr, 0, mcam_idx); + npc_cn20k_clear_mcam_entry(rvu, blkaddr, 1, mcam_idx); + + npc_cn20k_config_kw_x4(rvu, mcam, blkaddr, + mcam_idx, intf, entry, + kw_type, req_kw_type); + for (bank = 0; bank < mcam->banks_per_entry; bank++) { + /* Set 'action' */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 0), + entry->action); + + /* Set TAG 'action' */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 1), + entry->vtag_action); + + /* Set 'action2' for inline receive */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 2), + entry->action2); + } + +set_cfg: + /* TODO: */ + /* PF installing VF rule */ + npc_cn20k_set_mcam_bank_cfg(rvu, blkaddr, mcam_idx, bank, + kw_type, enable, hw_prio); +} + +void npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + u64 cfg, sreg, dreg, soff, doff; + u8 src_kwtype, dest_kwtype; + int bank, i, sb, db; + int dbank, sbank; + + dbank = npc_get_bank(mcam, dest); + sbank = npc_get_bank(mcam, src); + npc_mcam_idx_2_key_type(rvu, src, &src_kwtype); + npc_mcam_idx_2_key_type(rvu, dest, &dest_kwtype); + if (src_kwtype != dest_kwtype) + return; + + src &= (mcam->banksize - 1); + dest &= (mcam->banksize - 1); + + /* Copy INTF's, W0's, W1's, W2's, W3s CAM0 and CAM1 configuration */ + for (bank = 0; bank < mcam->banks_per_entry; bank++) { + sb = sbank + bank; + sreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(src, sb, 0); + db = dbank + bank; + dreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(dest, db, 0); + for (i = 0; i < 10; i++) { + cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); + rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); + } + + /* Copy action */ + for (i = 0; i < 3; i++) { + soff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(src, + sb, i); + cfg = rvu_read64(rvu, blkaddr, soff); + + doff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(dest, db, + i); + rvu_write64(rvu, blkaddr, doff, cfg); + } + + /* Copy bank configuration */ + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(src, sb)); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(dest, db), cfg); + if (src_kwtype == NPC_MCAM_KEY_X2) + break; + } +} + +static void npc_cn20k_fill_entryword(struct cn20k_mcam_entry *entry, int idx, + u64 cam0, u64 cam1) +{ + entry->kw[idx] = cam1; + entry->kw_mask[idx] = cam1 ^ cam0; +} + +void npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index, + struct cn20k_mcam_entry *entry, + u8 *intf, u8 *ena, u8 *hw_prio) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + u64 cam0, cam1, bank_cfg, cfg; + int kw = 0, bank; + u8 kw_type; + + npc_mcam_idx_2_key_type(rvu, index, &kw_type); + + bank = npc_get_bank(mcam, index); + index &= (mcam->banksize - 1); + + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, + bank, 1)) & 3; + *intf = cfg; + + bank_cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, bank)); + *ena = bank_cfg & 0x1; + *hw_prio = (bank_cfg & GENMASK_ULL(30, 24)) >> 24; + if (kw_type == NPC_MCAM_KEY_X2) { + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); + goto read_action; + } + + for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) { + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); + } + +read_action: + /* 'action' is set to same value for both bank '0' and '1'. + * Hence, reading bank '0' should be enough. + */ + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, 0, 0)); + entry->action = cfg; + + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, 0, 1)); + entry->vtag_action = cfg; +} + +int rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu *rvu, + struct npc_cn20k_mcam_write_entry_req *req, + struct msg_rsp *rsp) +{ + struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); + struct npc_mcam *mcam = &rvu->hw->mcam; + u16 pcifunc = req->hdr.pcifunc; + int blkaddr, rc; + u8 nix_intf; + + req->entry = npc_cn20k_vidx2idx(req->entry); + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) + return NPC_MCAM_INVALID_REQ; + + mutex_lock(&mcam->lock); + rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); + if (rc) + goto exit; + + if (!is_npc_interface_valid(rvu, req->intf)) { + rc = NPC_MCAM_INVALID_REQ; + goto exit; + } + + if (is_npc_intf_tx(req->intf)) + nix_intf = pfvf->nix_tx_intf; + else + nix_intf = pfvf->nix_rx_intf; + + /* For AF installed rules, the nix_intf should be set to target NIX */ + if (is_pffunc_af(req->hdr.pcifunc)) + nix_intf = req->intf; + + npc_cn20k_config_mcam_entry(rvu, blkaddr, req->entry, nix_intf, + &req->entry_data, req->enable_entry, + req->hw_prio, req->req_kw_type); + + rc = 0; +exit: + mutex_unlock(&mcam->lock); + return rc; +} + +int rvu_mbox_handler_npc_cn20k_mcam_read_entry(struct rvu *rvu, + struct npc_mcam_read_entry_req *req, + struct npc_cn20k_mcam_read_entry_rsp *rsp) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + u16 pcifunc = req->hdr.pcifunc; + int blkaddr, rc; + + req->entry = npc_cn20k_vidx2idx(req->entry); + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) + return NPC_MCAM_INVALID_REQ; + + mutex_lock(&mcam->lock); + rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); + if (!rc) + npc_cn20k_read_mcam_entry(rvu, blkaddr, req->entry, + &rsp->entry_data, &rsp->intf, + &rsp->enable, &rsp->hw_prio); + + mutex_unlock(&mcam->lock); + return rc; +} + +int rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu *rvu, + struct npc_cn20k_mcam_alloc_and_write_entry_req *req, + struct npc_mcam_alloc_and_write_entry_rsp *rsp) +{ + struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); + struct npc_mcam_alloc_entry_req entry_req; + struct npc_mcam_alloc_entry_rsp entry_rsp; + struct npc_mcam *mcam = &rvu->hw->mcam; + u16 entry = NPC_MCAM_ENTRY_INVALID; + int blkaddr, rc; + u8 nix_intf; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) + return NPC_MCAM_INVALID_REQ; + + if (!is_npc_interface_valid(rvu, req->intf)) + return NPC_MCAM_INVALID_REQ; + + /* Try to allocate a MCAM entry */ + entry_req.hdr.pcifunc = req->hdr.pcifunc; + entry_req.contig = true; + entry_req.ref_prio = req->ref_prio; + entry_req.ref_entry = req->ref_entry; + entry_req.count = 1; + entry_req.virt = req->virt; + + rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, + &entry_req, &entry_rsp); + if (rc) + return rc; + + if (!entry_rsp.count) + return NPC_MCAM_ALLOC_FAILED; + + /* entry_req.count is 1, so single entry is allocated */ + entry = npc_cn20k_vidx2idx(entry_rsp.entry); + + mutex_lock(&mcam->lock); + + if (is_npc_intf_tx(req->intf)) + nix_intf = pfvf->nix_tx_intf; + else + nix_intf = pfvf->nix_rx_intf; + + npc_cn20k_config_mcam_entry(rvu, blkaddr, entry, nix_intf, + &req->entry_data, req->enable_entry, + req->hw_prio, req->req_kw_type); + + mutex_unlock(&mcam->lock); + + rsp->entry = entry_rsp.entry; + return 0; +} + +static int rvu_npc_get_base_steer_rule_type(struct rvu *rvu, u16 pcifunc) +{ + if (is_lbk_vf(rvu, pcifunc)) + return NIXLF_PROMISC_ENTRY; + + return NIXLF_UCAST_ENTRY; +} + +int rvu_mbox_handler_npc_cn20k_read_base_steer_rule(struct rvu *rvu, + struct msg_req *req, + struct npc_cn20k_mcam_read_base_rule_rsp *rsp) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + int index, blkaddr, nixlf, rc = 0; + u16 pcifunc = req->hdr.pcifunc; + u8 intf, enable, hw_prio; + struct rvu_pfvf *pfvf; + int rl_type; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) + return NPC_MCAM_INVALID_REQ; + + /* Return the channel number in case of PF */ + if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { + pfvf = rvu_get_pfvf(rvu, pcifunc); + rsp->entry.kw[0] = pfvf->rx_chan_base; + rsp->entry.kw_mask[0] = 0xFFFULL; + goto out; + } + + /* Find the pkt steering rule installed by PF to this VF */ + mutex_lock(&mcam->lock); + for (index = 0; index < mcam->bmap_entries; index++) { + if (mcam->entry2target_pffunc[index] == pcifunc) + goto read_entry; + } + + rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); + if (rc < 0) { + mutex_unlock(&mcam->lock); + goto out; + } + + rl_type = rvu_npc_get_base_steer_rule_type(rvu, pcifunc); + + /* Read the default ucast entry if there is no pkt steering rule */ + index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, rl_type); + if (index < 0) { + mutex_unlock(&mcam->lock); + goto out; + } + +read_entry: + /* Read the mcam entry */ + npc_cn20k_read_mcam_entry(rvu, blkaddr, index, + &rsp->entry, &intf, + &enable, &hw_prio); + mutex_unlock(&mcam->lock); +out: + return rc; +} + +static u8 npc_map2cn20k_flag(u8 flag) +{ + switch (flag) { + case NPC_F_LC_U_IP_FRAG: + return NPC_CN20K_F_LC_L_IP_FRAG; + + case NPC_F_LC_U_IP6_FRAG: + return NPC_CN20K_F_LC_L_IP6_FRAG; + + case NPC_F_LC_L_6TO4: + return NPC_CN20K_F_LC_L_6TO4; + + case NPC_F_LC_L_MPLS_IN_IP: + return NPC_CN20K_F_LC_U_MPLS_IN_IP; + + case NPC_F_LC_L_IP6_TUN_IP6: + return NPC_CN20K_F_LC_U_IP6_TUN_IP6; + + case NPC_F_LC_L_IP6_MPLS_IN_IP: + return NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP; + + default: + break; + } + + WARN(1, "%s: Invalid flag=%u\n", __func__, flag); + return 0xff; +} + +void +npc_cn20k_update_action_entries_n_flags(struct rvu *rvu, + struct npc_kpu_profile_adapter *pfl) +{ + struct npc_kpu_profile_action *action; + int entries, ltype; + u8 flags, val; + + for (int i = 0; i < pfl->kpus; i++) { + action = pfl->kpu[i].action; + entries = pfl->kpu[i].action_entries; + + for (int j = 0; j < entries; j++) { + if (action[j].lid != NPC_LID_LC) + continue; + + ltype = action[j].ltype; + + if (ltype != NPC_LT_LC_IP && + ltype != NPC_LT_LC_IP6 && + ltype != NPC_LT_LC_IP_OPT && + ltype != NPC_LT_LC_IP6_EXT) + continue; + + flags = action[j].flags; + + switch (flags) { + case NPC_F_LC_U_IP_FRAG: + case NPC_F_LC_U_IP6_FRAG: + case NPC_F_LC_L_6TO4: + case NPC_F_LC_L_MPLS_IN_IP: + case NPC_F_LC_L_IP6_TUN_IP6: + case NPC_F_LC_L_IP6_MPLS_IN_IP: + val = npc_map2cn20k_flag(flags); + if (val == 0xFF) { + dev_err(rvu->dev, + "%s: Error to get flag value\n", + __func__); + return; + } + + action[j].flags = val; + break; + default: + break; + } + } + } +} + +int npc_cn20k_apply_custom_kpu(struct rvu *rvu, + struct npc_kpu_profile_adapter *profile) +{ + size_t hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); + struct npc_cn20k_kpu_profile_fwdata *fw = rvu->kpu_fwdata; + struct npc_kpu_profile_action *action; + struct npc_kpu_profile_cam *cam; + struct npc_kpu_fwdata *fw_kpu; + size_t offset = 0; + u16 kpu, entry; + int entries; + + hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); + + if (rvu->kpu_fwdata_sz < hdr_sz) { + dev_warn(rvu->dev, "Invalid KPU profile size\n"); + return -EINVAL; + } + + if (le64_to_cpu(fw->signature) != KPU_SIGN) { + dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", + fw->signature); + return -EINVAL; + } + + /* Verify if the using known profile structure */ + if (NPC_KPU_VER_MAJ(profile->version) > + NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { + dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", + NPC_KPU_VER_MAJ(profile->version), + NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); + return -EINVAL; + } + + /* Verify if profile is aligned with the required kernel changes */ + if (NPC_KPU_VER_MIN(profile->version) < + NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { + dev_warn(rvu->dev, + "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", + NPC_KPU_VER_MAJ(profile->version), + NPC_KPU_VER_MIN(profile->version), + NPC_KPU_VER_PATCH(profile->version), + NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), + NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), + NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); + return -EINVAL; + } + + /* Verify if profile fits the HW */ + if (fw->kpus > profile->kpus) { + dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus, + profile->kpus); + return -EINVAL; + } + + profile->mcam_kex_prfl.mkex_extr = &fw->mkex; + if (profile->mcam_kex_prfl.mkex_extr->mkex_sign != MKEX_CN20K_SIGN) { + dev_warn(rvu->dev, "Invalid MKEX profile signature:%llx\n", + profile->mcam_kex_prfl.mkex_extr->mkex_sign); + return -EINVAL; + } + + profile->custom = 1; + profile->name = fw->name; + profile->version = le64_to_cpu(fw->version); + profile->lt_def = &fw->lt_def; + + for (kpu = 0; kpu < fw->kpus; kpu++) { + fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); + if (fw_kpu->entries > KPU_CN20K_MAX_CST_ENT) + dev_warn(rvu->dev, + "Too many custom entries on KPU%d: %d > %d\n", + kpu, fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); + entries = min(fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); + cam = (struct npc_kpu_profile_cam *)fw_kpu->data; + offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); + action = (struct npc_kpu_profile_action *)(fw->data + offset); + offset += fw_kpu->entries * sizeof(*action); + if (rvu->kpu_fwdata_sz < hdr_sz + offset) { + dev_warn(rvu->dev, + "Profile size mismatch on KPU%i parsing.\n", + kpu + 1); + return -EINVAL; + } + + for (entry = 0; entry < entries; entry++) { + profile->kpu[kpu].cam[entry] = cam[entry]; + profile->kpu[kpu].action[entry] = action[entry]; + } + } + npc_cn20k_update_action_entries_n_flags(rvu, profile); + + return 0; +} + +int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type) +{ + struct npc_subbank *sb; + int bank_off, sb_id; + + /* mcam_idx should be less than (2 * bank depth) */ + if (mcam_idx >= npc_priv.bank_depth * 2) { + dev_err(rvu->dev, "%s: bad params\n", + __func__); + return -EINVAL; + } + + /* find mcam offset per bank */ + bank_off = mcam_idx & (npc_priv.bank_depth - 1); + + /* Find subbank id */ + sb_id = bank_off / npc_priv.subbank_depth; + + /* Check if subbank id is more than maximum + * number of subbanks available + */ + if (sb_id >= npc_priv.num_subbanks) { + dev_err(rvu->dev, "%s: invalid subbank %d\n", + __func__, sb_id); + return -EINVAL; + } + + sb = &npc_priv.sb[sb_id]; + + *key_type = sb->key_type; + + return 0; +} + +static int npc_subbank_idx_2_mcam_idx(struct rvu *rvu, struct npc_subbank *sb, + u16 sub_off, u16 *mcam_idx) +{ + int off, bot; + + /* for x4 section, maximum allowed subbank index = + * subsection depth - 1 + */ + if (sb->key_type == NPC_MCAM_KEY_X4 && + sub_off >= npc_priv.subbank_depth) { + dev_err(rvu->dev, + "%s: Failed to get mcam idx (x4) sb->idx=%u sub_off=%u", + __func__, sb->idx, sub_off); + return -EINVAL; + } + + /* for x2 section, maximum allowed subbank index = + * 2 * subsection depth - 1 + */ + if (sb->key_type == NPC_MCAM_KEY_X2 && + sub_off >= npc_priv.subbank_depth * 2) { + dev_err(rvu->dev, + "%s: Failed to get mcam idx (x2) sb->idx=%u sub_off=%u", + __func__, sb->idx, sub_off); + return -EINVAL; + } + + /* Find subbank offset from respective subbank (w.r.t bank) */ + off = sub_off & (npc_priv.subbank_depth - 1); + + /* if subsection idx is in bank1, add bank depth, + * which is part of sb->b1b + */ + bot = sub_off >= npc_priv.subbank_depth ? sb->b1b : sb->b0b; + + *mcam_idx = bot + off; + return 0; +} + +static int npc_mcam_idx_2_subbank_idx(struct rvu *rvu, u16 mcam_idx, + struct npc_subbank **sb, + int *sb_off) +{ + int bank_off, sb_id; + + /* mcam_idx should be less than (2 * bank depth) */ + if (mcam_idx >= npc_priv.bank_depth * 2) { + dev_err(rvu->dev, "%s: Invalid mcam idx %u\n", + __func__, mcam_idx); + return -EINVAL; + } + + /* find mcam offset per bank */ + bank_off = mcam_idx & (npc_priv.bank_depth - 1); + + /* Find subbank id */ + sb_id = bank_off / npc_priv.subbank_depth; + + /* Check if subbank id is more than maximum + * number of subbanks available + */ + if (sb_id >= npc_priv.num_subbanks) { + dev_err(rvu->dev, "%s: invalid subbank %d\n", + __func__, sb_id); + return -EINVAL; + } + + *sb = &npc_priv.sb[sb_id]; + + /* Subbank offset per bank */ + *sb_off = bank_off % npc_priv.subbank_depth; + + /* Index in a subbank should add subbank depth + * if it is in bank1 + */ + if (mcam_idx >= npc_priv.bank_depth) + *sb_off += npc_priv.subbank_depth; + + return 0; +} + +static int __npc_subbank_contig_alloc(struct rvu *rvu, + struct npc_subbank *sb, + int key_type, int sidx, + int eidx, int prio, + int count, int t, int b, + unsigned long *bmap, + u16 *save) +{ + int k, offset, delta = 0; + int cnt = 0, sbd; + + sbd = npc_priv.subbank_depth; + + if (sidx >= npc_priv.bank_depth) + delta = sbd; + + switch (prio) { + case NPC_MCAM_LOWER_PRIO: + case NPC_MCAM_ANY_PRIO: + /* Find an area of size 'count' from sidx to eidx */ + offset = bitmap_find_next_zero_area(bmap, sbd, sidx - b, + count, 0); + + if (offset >= sbd) { + dev_err(rvu->dev, + "%s: Could not find contiguous(%d) entries\n", + __func__, count); + return -EFAULT; + } + + dev_dbg(rvu->dev, + "%s: sidx=%d eidx=%d t=%d b=%d offset=%d count=%d delta=%d\n", + __func__, sidx, eidx, t, b, offset, + count, delta); + + for (cnt = 0; cnt < count; cnt++) + save[cnt] = offset + cnt + delta; + + break; + + case NPC_MCAM_HIGHER_PRIO: + /* Find an area of 'count' from eidx to sidx */ + for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { + /* If an intermediate slot is not free, + * reset the counter (cnt) to zero as + * request is for contiguous. + */ + if (test_bit(k, bmap)) { + cnt = 0; + continue; + } + + save[cnt++] = k + delta; + } + break; + } + + /* Found 'count' number of free slots */ + if (cnt == count) + return 0; + + dev_dbg(rvu->dev, + "%s: Could not find contiguous(%d) entries in subbank=%u\n", + __func__, count, sb->idx); + return -EFAULT; +} + +static int __npc_subbank_non_contig_alloc(struct rvu *rvu, + struct npc_subbank *sb, + int key_type, int sidx, + int eidx, int prio, + int t, int b, + unsigned long *bmap, + int count, u16 *save, + bool max_alloc, int *alloc_cnt) +{ + unsigned long index; + int cnt = 0, delta; + int k, sbd; + + sbd = npc_priv.subbank_depth; + delta = sidx >= npc_priv.bank_depth ? sbd : 0; + + switch (prio) { + /* Find an area of size 'count' from sidx to eidx */ + case NPC_MCAM_LOWER_PRIO: + case NPC_MCAM_ANY_PRIO: + index = find_next_zero_bit(bmap, sbd, sidx - b); + if (index >= sbd) { + dev_err(rvu->dev, + "%s: Error happened to alloc %u, bitmap_weight=%u, sb->idx=%u\n", + __func__, count, + bitmap_weight(bmap, sbd), + sb->idx); + break; + } + + for (k = index; cnt < count && k <= (eidx - b); k++) { + /* Skip used slots */ + if (test_bit(k, bmap)) + continue; + + save[cnt++] = k + delta; + } + break; + + /* Find an area of 'count' from eidx to sidx */ + case NPC_MCAM_HIGHER_PRIO: + for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { + /* Skip used slots */ + if (test_bit(k, bmap)) + continue; + + save[cnt++] = k + delta; + } + break; + } + + /* Update allocated 'cnt' to alloc_cnt */ + *alloc_cnt = cnt; + + /* Successfully allocated requested count slots */ + if (cnt == count) + return 0; + + /* Allocation successful for cnt < count */ + if (max_alloc && cnt > 0) + return 0; + + dev_dbg(rvu->dev, + "%s: Could not find non contiguous entries(%u) in subbank(%u) cnt=%d max_alloc=%d\n", + __func__, count, sb->idx, cnt, max_alloc); + + return -EFAULT; +} + +static void __npc_subbank_sboff_2_off(struct rvu *rvu, struct npc_subbank *sb, + int sb_off, unsigned long **bmap, + int *off) +{ + int sbd; + + sbd = npc_priv.subbank_depth; + + *off = sb_off & (sbd - 1); + *bmap = (sb_off >= sbd) ? sb->b1map : sb->b0map; +} + +/* set/clear bitmap */ +static bool __npc_subbank_mark_slot(struct rvu *rvu, + struct npc_subbank *sb, + int sb_off, bool set) +{ + unsigned long *bmap; + int off; + + /* if sb_off >= subbank.depth, then slots are in + * bank1 + */ + __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); + + dev_dbg(rvu->dev, + "%s: Marking set=%d sb_off=%d sb->idx=%d off=%d\n", + __func__, set, sb_off, sb->idx, off); + + if (set) { + /* Slot is already used */ + if (test_bit(off, bmap)) + return false; + + sb->free_cnt--; + set_bit(off, bmap); + return true; + } + + /* Slot is already free */ + if (!test_bit(off, bmap)) + return false; + + sb->free_cnt++; + clear_bit(off, bmap); + return true; +} + +static int __npc_subbank_mark_free(struct rvu *rvu, struct npc_subbank *sb) +{ + int rc, blkaddr; + + sb->flags = NPC_SUBBANK_FLAG_FREE; + sb->key_type = 0; + + bitmap_clear(sb->b0map, 0, npc_priv.subbank_depth); + bitmap_clear(sb->b1map, 0, npc_priv.subbank_depth); + + if (!xa_erase(&npc_priv.xa_sb_used, sb->arr_idx)) { + dev_err(rvu->dev, + "%s: Error to delete from xa_sb_used array\n", + __func__); + return -EFAULT; + } + + rc = xa_insert(&npc_priv.xa_sb_free, sb->arr_idx, + xa_mk_value(sb->idx), GFP_KERNEL); + if (rc) { + rc = xa_insert(&npc_priv.xa_sb_used, sb->arr_idx, + xa_mk_value(sb->idx), GFP_KERNEL); + if (rc) + dev_err(rvu->dev, + "%s: Failed to roll back sb(%u) arr_idx=%d\n", + __func__, sb->idx, sb->arr_idx); + + dev_err(rvu->dev, + "%s: Error to add sb(%u) to xa_sb_free array at arr_idx=%d\n", + __func__, sb->idx, sb->arr_idx); + return rc; + } + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + rvu_write64(rvu, blkaddr, + NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), + NPC_MCAM_KEY_X2); + + return rc; +} + +static int __npc_subbank_mark_used(struct rvu *rvu, struct npc_subbank *sb, + int key_type) +{ + int rc; + + sb->flags = NPC_SUBBANK_FLAG_USED; + sb->key_type = key_type; + if (key_type == NPC_MCAM_KEY_X4) + sb->free_cnt = npc_priv.subbank_depth; + else + sb->free_cnt = 2 * npc_priv.subbank_depth; + + bitmap_clear(sb->b0map, 0, npc_priv.subbank_depth); + bitmap_clear(sb->b1map, 0, npc_priv.subbank_depth); + + if (!xa_erase(&npc_priv.xa_sb_free, sb->arr_idx)) { + dev_err(rvu->dev, + "%s: Error to delete from xa_sb_free array\n", + __func__); + return -EFAULT; + } + + rc = xa_insert(&npc_priv.xa_sb_used, sb->arr_idx, + xa_mk_value(sb->idx), GFP_KERNEL); + if (rc) + dev_err(rvu->dev, + "%s: Error to add to xa_sb_used array\n", __func__); + + return rc; +} + +static bool __npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, + u16 sb_off) +{ + bool deleted = false; + unsigned long *bmap; + int rc, off; + + deleted = __npc_subbank_mark_slot(rvu, sb, sb_off, false); + if (!deleted) + goto done; + + __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); + + /* Check whether we can mark whole subbank as free */ + if (sb->key_type == NPC_MCAM_KEY_X4) { + if (sb->free_cnt < npc_priv.subbank_depth) + goto done; + } else { + if (sb->free_cnt < 2 * npc_priv.subbank_depth) + goto done; + } + + /* All slots in subbank are unused. Mark the subbank as free + * and add to free pool + */ + rc = __npc_subbank_mark_free(rvu, sb); + if (rc) + dev_err(rvu->dev, "%s: Error to free subbank\n", __func__); + +done: + return deleted; +} + +static int +npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, u16 sb_off) +{ + bool deleted; + + mutex_lock(&sb->lock); + deleted = __npc_subbank_free(rvu, sb, sb_off); + mutex_unlock(&sb->lock); + + return deleted ? 0 : -EFAULT; +} + +static int __npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, + int key_type, int ref, int limit, int prio, + bool contig, int count, u16 *mcam_idx, + int idx_sz, bool max_alloc, int *alloc_cnt) +{ + int cnt, t, b, i, blkaddr; + bool new_sub_bank = false; + unsigned long *bmap; + u16 *save = NULL; + int sidx, eidx; + bool diffbank; + int bw, bfree; + int rc = 0; + bool ret; + + /* Check if enough space is there to return requested number of + * mcam indexes in case of contiguous allocation + */ + if (!max_alloc && count > idx_sz) { + dev_err(rvu->dev, + "%s: Less space, count=%d idx_sz=%d sb_id=%d\n", + __func__, count, idx_sz, sb->idx); + return -ENOSPC; + } + + /* Allocation on multiple subbank is not supported by this function. + * it means that ref and limit should be on same subbank. + * + * ref and limit values should be validated w.r.t prio as below. + * say ref = 100, limit = 200, + * if NPC_MCAM_LOWER_PRIO, allocate index 100 + * if NPC_MCAM_HIGHER_PRIO, below sanity test returns error. + * if NPC_MCAM_ANY_PRIO, allocate index 100 + * + * say ref = 200, limit = 100 + * if NPC_MCAM_LOWER_PRIO, below sanity test returns error. + * if NPC_MCAM_HIGHER_PRIO, allocate index 200 + * if NPC_MCAM_ANY_PRIO, allocate index 100 + * + * Please note that NPC_MCAM_ANY_PRIO does not have any restriction + * on "ref" and "limit" values. ie, ref > limit and limit > ref + * are valid cases. + */ + if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || + (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { + dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", + __func__, ref, limit); + return -EINVAL; + } + + /* x4 indexes are from 0 to bank size as it combines two x2 banks */ + if (key_type == NPC_MCAM_KEY_X4 && + (ref >= npc_priv.bank_depth || limit >= npc_priv.bank_depth)) { + dev_err(rvu->dev, + "%s: Wrong ref_enty(%d) or limit(%d) for x4\n", + __func__, ref, limit); + return -EINVAL; + } + + /* This function is called either bank0 or bank1 portion of a subbank. + * so ref and limit should be on same bank. + */ + diffbank = !!((ref & npc_priv.bank_depth) ^ + (limit & npc_priv.bank_depth)); + if (diffbank) { + dev_err(rvu->dev, + "%s: request ref and limit should be from same bank\n", + __func__); + return -EINVAL; + } + + sidx = min_t(int, limit, ref); + eidx = max_t(int, limit, ref); + + /* Find total number of slots available; both used and free */ + cnt = eidx - sidx + 1; + if (contig && cnt < count) { + dev_err(rvu->dev, + "%s: Wrong ref_enty(%d) or limit(%d) for count(%d)\n", + __func__, ref, limit, count); + return -EINVAL; + } + + /* If subbank is free, check if requested number of indexes is less than + * or equal to mcam entries available in the subbank if contig. + */ + if (sb->flags & NPC_SUBBANK_FLAG_FREE) { + if (contig && count > npc_priv.subbank_depth) { + dev_err(rvu->dev, "%s: Less number of entries\n", + __func__); + return -ENOSPC; + } + + new_sub_bank = true; + goto process; + } + + /* Flag should be set for all used subbanks */ + WARN_ONCE(!(sb->flags & NPC_SUBBANK_FLAG_USED), + "Used flag is not set(%#x)\n", sb->flags); + + /* If subbank key type does not match with requested key_type, + * return error + */ + if (sb->key_type != key_type) { + dev_dbg(rvu->dev, "%s: subbank key_type mismatch\n", __func__); + return -EINVAL; + } + +process: + /* if ref or limit >= npc_priv.bank_depth, index are in bank1. + * else bank0. + */ + if (ref >= npc_priv.bank_depth) { + bmap = sb->b1map; + t = sb->b1t; + b = sb->b1b; + } else { + bmap = sb->b0map; + t = sb->b0t; + b = sb->b0b; + } + + /* Calculate free slots */ + bw = bitmap_weight(bmap, npc_priv.subbank_depth); + bfree = npc_priv.subbank_depth - bw; + + if (!bfree) { + dev_dbg(rvu->dev, "%s: subbank is full\n", __func__); + return -ENOSPC; + } + + /* If request is for contiguous , then max we can allocate is + * equal to subbank_depth + */ + if (contig && bfree < count) { + dev_dbg(rvu->dev, "%s: no space for entry\n", __func__); + return -ENOSPC; + } + + /* 'save' array stores available indexes temporarily before + * marking it as allocated + */ + save = kcalloc(count, sizeof(u16), GFP_KERNEL); + if (!save) { + rc = -ENOMEM; + goto err1; + } + + if (contig) { + rc = __npc_subbank_contig_alloc(rvu, sb, key_type, + sidx, eidx, prio, + count, t, b, + bmap, save); + /* contiguous allocation success means that + * requested number of free slots got + * allocated + */ + if (!rc) + *alloc_cnt = count; + + } else { + rc = __npc_subbank_non_contig_alloc(rvu, sb, key_type, + sidx, eidx, prio, + t, b, bmap, + count, save, + max_alloc, alloc_cnt); + } + + if (rc) + goto err1; + + /* Mark new subbank bank as used */ + if (new_sub_bank) { + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) { + dev_err(rvu->dev, + "%s: NPC block not implemented\n", __func__); + rc = -EFAULT; + goto err1; + } + + rc = __npc_subbank_mark_used(rvu, sb, key_type); + if (rc) { + dev_err(rvu->dev, + "%s: Error to mark subbank as used\n", + __func__); + goto err2; + } + + /* Configure section type to key_type */ + rvu_write64(rvu, blkaddr, + NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), + key_type); + } + + for (i = 0; i < *alloc_cnt; i++) { + rc = npc_subbank_idx_2_mcam_idx(rvu, sb, save[i], + &mcam_idx[i]); + if (rc) { + dev_err(rvu->dev, + "%s: Error to find mcam idx for %u\n", + __func__, save[i]); + /* TODO: handle err case gracefully */ + goto err3; + } + + /* Mark all slots as used */ + ret = __npc_subbank_mark_slot(rvu, sb, save[i], true); + if (!ret) { + dev_err(rvu->dev, "%s: Error to mark mcam_idx %u\n", + __func__, mcam_idx[i]); + rc = -EFAULT; + goto err3; + } + } + kfree(save); + return 0; + +err3: + for (int j = 0; j < i; j++) + __npc_subbank_mark_slot(rvu, sb, save[j], false); +err2: + if (new_sub_bank) + __npc_subbank_mark_free(rvu, sb); +err1: + kfree(save); + return rc; +} + +static int +npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, + int key_type, int ref, int limit, int prio, + bool contig, int count, u16 *mcam_idx, + int idx_sz, bool max_alloc, int *alloc_cnt) +{ + int rc; + + mutex_lock(&sb->lock); + rc = __npc_subbank_alloc(rvu, sb, key_type, ref, limit, prio, + contig, count, mcam_idx, idx_sz, + max_alloc, alloc_cnt); + mutex_unlock(&sb->lock); + + return rc; +} + +static int +npc_del_from_pf_maps(struct rvu *rvu, u16 mcam_idx) +{ + int pcifunc, idx; + void *map; + + map = xa_erase(&npc_priv.xa_idx2pf_map, mcam_idx); + if (!map) { + dev_err(rvu->dev, + "%s: failed to erase mcam_idx(%u) from xa_idx2pf map\n", + __func__, mcam_idx); + return -EFAULT; + } + + pcifunc = xa_to_value(map); + map = xa_load(&npc_priv.xa_pf_map, pcifunc); + if (!map) { + dev_err(rvu->dev, + "%s: failed to find entry for (%u) from xa_pf_map, mcam=%u\n", + __func__, pcifunc, mcam_idx); + return -ESRCH; + } + + idx = xa_to_value(map); + + map = xa_erase(&npc_priv.xa_pf2idx_map[idx], mcam_idx); + if (!map) { + dev_err(rvu->dev, + "%s: failed to erase mcam_idx(%u) from xa_pf2idx_map map\n", + __func__, mcam_idx); + return -EFAULT; + } + return 0; +} + +static int +npc_add_to_pf_maps(struct rvu *rvu, u16 mcam_idx, int pcifunc) +{ + int rc, idx; + void *map; + + dev_dbg(rvu->dev, + "%s: add2maps mcam_idx(%u) to xa_idx2pf map pcifunc=%#x\n", + __func__, mcam_idx, pcifunc); + + rc = xa_insert(&npc_priv.xa_idx2pf_map, mcam_idx, + xa_mk_value(pcifunc), GFP_KERNEL); + + if (rc) { + map = xa_load(&npc_priv.xa_idx2pf_map, mcam_idx); + dev_err(rvu->dev, + "%s: failed to insert mcam_idx(%u) to xa_idx2pf map, existing value=%lu\n", + __func__, mcam_idx, xa_to_value(map)); + return -EFAULT; + } + + map = xa_load(&npc_priv.xa_pf_map, pcifunc); + if (!map) { + dev_err(rvu->dev, + "%s: failed to find pf map entry for pcifunc=%#x, mcam=%u\n", + __func__, pcifunc, mcam_idx); + return -ESRCH; + } + + idx = xa_to_value(map); + + rc = xa_insert(&npc_priv.xa_pf2idx_map[idx], mcam_idx, + xa_mk_value(pcifunc), GFP_KERNEL); + + if (rc) { + map = xa_load(&npc_priv.xa_pf2idx_map[idx], mcam_idx); + xa_erase(&npc_priv.xa_idx2pf_map, mcam_idx); + dev_err(rvu->dev, + "%s: failed to insert mcam_idx(%u) to xa_pf2idx_map map, earlier value=%lu idx=%u\n", + __func__, mcam_idx, xa_to_value(map), idx); + + return -EFAULT; + } + + return 0; +} + +static bool +npc_subbank_suits(struct npc_subbank *sb, int key_type) +{ + mutex_lock(&sb->lock); + + if (!sb->key_type) { + mutex_unlock(&sb->lock); + return true; + } + + if (sb->key_type == key_type) { + mutex_unlock(&sb->lock); + return true; + } + + mutex_unlock(&sb->lock); + return false; +} + +#define SB_ALIGN_UP(val) (((val) + npc_priv.subbank_depth) & \ + ~((npc_priv.subbank_depth) - 1)) +#define SB_ALIGN_DOWN(val) ALIGN_DOWN((val), npc_priv.subbank_depth) + +static void npc_subbank_iter_down(struct rvu *rvu, + int ref, int limit, + int *cur_ref, int *cur_limit, + bool *start, bool *stop) +{ + int align; + + *stop = false; + + /* ALIGN_DOWN the limit to current subbank boundary bottom index */ + if (*start) { + *start = false; + *cur_ref = ref; + align = SB_ALIGN_DOWN(ref); + if (align < limit) { + *stop = true; + *cur_limit = limit; + return; + } + *cur_limit = align; + return; + } + + *cur_ref = *cur_limit - 1; + align = *cur_ref - npc_priv.subbank_depth + 1; + if (align <= limit) { + *stop = true; + *cur_limit = limit; + return; + } + + *cur_limit = align; +} + +static void npc_subbank_iter_up(struct rvu *rvu, + int ref, int limit, + int *cur_ref, int *cur_limit, + bool *start, bool *stop) +{ + int align; + + *stop = false; + + /* ALIGN_UP the limit to current subbank boundary top index */ + if (*start) { + *start = false; + *cur_ref = ref; + + /* Find next lower prio subbank's bottom index */ + align = SB_ALIGN_UP(ref); + + /* Crosses limit ? */ + if (align - 1 > limit) { + *stop = true; + *cur_limit = limit; + return; + } + + /* Current subbank's top index */ + *cur_limit = align - 1; + return; + } + + *cur_ref = *cur_limit + 1; + align = *cur_ref + npc_priv.subbank_depth - 1; + + if (align >= limit) { + *stop = true; + *cur_limit = limit; + return; + } + + *cur_limit = align; +} + +static int +npc_subbank_iter(struct rvu *rvu, int key_type, + int ref, int limit, int prio, + int *cur_ref, int *cur_limit, + bool *start, bool *stop) +{ + if (prio != NPC_MCAM_HIGHER_PRIO) + npc_subbank_iter_up(rvu, ref, limit, + cur_ref, cur_limit, + start, stop); + else + npc_subbank_iter_down(rvu, ref, limit, + cur_ref, cur_limit, + start, stop); + + /* limit and ref should < bank_depth for x4 */ + if (key_type == NPC_MCAM_KEY_X4) { + if (*cur_ref >= npc_priv.bank_depth) + return -EINVAL; + + if (*cur_limit >= npc_priv.bank_depth) + return -EINVAL; + } + /* limit and ref should < 2 * bank_depth, for x2 */ + if (*cur_ref >= 2 * npc_priv.bank_depth) + return -EINVAL; + + if (*cur_limit >= 2 * npc_priv.bank_depth) + return -EINVAL; + + return 0; +} + +static int npc_idx_free(struct rvu *rvu, u16 *mcam_idx, int count, + bool maps_del) +{ + struct npc_subbank *sb; + u16 vidx, midx; + int sb_off, i; + bool ret; + int rc; + + /* Check if we can dealloc indexes properly ? */ + for (i = 0; i < count; i++) { + rc = npc_mcam_idx_2_subbank_idx(rvu, npc_vidx2idx(mcam_idx[i]), + &sb, &sb_off); + if (rc) { + dev_err(rvu->dev, + "Failed to free mcam idx=%u\n", mcam_idx[i]); + return rc; + } + } + + for (i = 0; i < count; i++) { + if (npc_is_vidx(mcam_idx[i])) { + vidx = mcam_idx[i]; + midx = npc_vidx2idx(vidx); + } else { + midx = mcam_idx[i]; + vidx = npc_idx2vidx(midx); + } + + if (midx >= npc_priv.bank_depth * npc_priv.num_banks) { + dev_err(rvu->dev, + "%s: Invalid mcam_idx=%u cannot be deleted\n", + __func__, mcam_idx[i]); + return -EINVAL; + } + + rc = npc_mcam_idx_2_subbank_idx(rvu, midx, + &sb, &sb_off); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to find subbank info for vidx=%u\n", + __func__, vidx); + return rc; + } + + ret = npc_subbank_free(rvu, sb, sb_off); + if (ret) { + dev_err(rvu->dev, + "%s: Failed to find subbank info for vidx=%u\n", + __func__, vidx); + return -EINVAL; + } + + if (!maps_del) + continue; + + rc = npc_del_from_pf_maps(rvu, midx); + if (rc) + return rc; + + /* If there is no vidx mapping; continue */ + if (vidx == midx) + continue; + + rc = npc_vidx_maps_del_entry(rvu, vidx, NULL); + if (rc) + return rc; + } + + return 0; +} + +static int npc_multi_subbank_ref_alloc(struct rvu *rvu, int key_type, + int ref, int limit, int prio, + bool contig, int count, + u16 *mcam_idx) +{ + struct npc_subbank *sb; + unsigned long *bmap; + int sb_off, off, rc; + int cnt = 0; + bool bitset; + + if (prio != NPC_MCAM_HIGHER_PRIO) { + while (ref <= limit) { + /* Calculate subbank and subbank index */ + rc = npc_mcam_idx_2_subbank_idx(rvu, ref, + &sb, &sb_off); + if (rc) + goto err; + + /* If subbank is not suitable for requested key type + * restart search from next subbank + */ + if (!npc_subbank_suits(sb, key_type)) { + ref = SB_ALIGN_UP(ref); + if (contig) { + rc = npc_idx_free(rvu, mcam_idx, + cnt, false); + if (rc) + return rc; + cnt = 0; + } + continue; + } + + mutex_lock(&sb->lock); + + /* If subbank is free; mark it as used */ + if (sb->flags & NPC_SUBBANK_FLAG_FREE) { + rc = __npc_subbank_mark_used(rvu, sb, + key_type); + if (rc) { + mutex_unlock(&sb->lock); + dev_err(rvu->dev, + "%s:Error to add to use array\n", + __func__); + goto err; + } + } + + /* Find correct bmap */ + __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); + + /* if bit is already set, reset 'cnt' */ + bitset = test_bit(off, bmap); + if (bitset) { + mutex_unlock(&sb->lock); + if (contig) { + rc = npc_idx_free(rvu, mcam_idx, + cnt, false); + if (rc) + return rc; + cnt = 0; + } + + ref++; + continue; + } + + set_bit(off, bmap); + sb->free_cnt--; + mcam_idx[cnt++] = ref; + mutex_unlock(&sb->lock); + + if (cnt == count) + return 0; + ref++; + } + + /* Could not allocate request count slots */ + goto err; + } + while (ref >= limit) { + rc = npc_mcam_idx_2_subbank_idx(rvu, ref, + &sb, &sb_off); + if (rc) + goto err; + + if (!npc_subbank_suits(sb, key_type)) { + ref = SB_ALIGN_DOWN(ref) - 1; + if (contig) { + rc = npc_idx_free(rvu, mcam_idx, cnt, false); + if (rc) + return rc; + + cnt = 0; + } + continue; + } + + mutex_lock(&sb->lock); + + if (sb->flags & NPC_SUBBANK_FLAG_FREE) { + rc = __npc_subbank_mark_used(rvu, sb, key_type); + if (rc) { + mutex_unlock(&sb->lock); + dev_err(rvu->dev, + "%s:Error to add to use array\n", + __func__); + goto err; + } + } + + __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); + bitset = test_bit(off, bmap); + if (bitset) { + mutex_unlock(&sb->lock); + if (contig) { + rc = npc_idx_free(rvu, mcam_idx, cnt, false); + if (rc) + return rc; + + cnt = 0; + } + ref--; + continue; + } + + mcam_idx[cnt++] = ref; + sb->free_cnt--; + set_bit(off, bmap); + mutex_unlock(&sb->lock); + + if (cnt == count) + return 0; + ref--; + } + +err: + rc = npc_idx_free(rvu, mcam_idx, cnt, false); + if (rc) + dev_err(rvu->dev, + "%s: Error happened while freeing cnt=%u indexes\n", + __func__, cnt); + + return -ENOSPC; +} + +static int npc_subbank_free_cnt(struct rvu *rvu, struct npc_subbank *sb, + int key_type) +{ + int cnt, spd; + + spd = npc_priv.subbank_depth; + mutex_lock(&sb->lock); + + if (sb->flags & NPC_SUBBANK_FLAG_FREE) + cnt = key_type == NPC_MCAM_KEY_X4 ? spd : 2 * spd; + else + cnt = sb->free_cnt; + + mutex_unlock(&sb->lock); + return cnt; +} + +static int npc_subbank_ref_alloc(struct rvu *rvu, int key_type, + int ref, int limit, int prio, + bool contig, int count, + u16 *mcam_idx) +{ + struct npc_subbank *sb1, *sb2; + bool max_alloc, start, stop; + int r, l, sb_idx1, sb_idx2; + int tot = 0, rc; + int alloc_cnt; + + max_alloc = !contig; + + start = true; + stop = false; + + /* Loop until we cross the ref/limit boundary */ + while (!stop) { + rc = npc_subbank_iter(rvu, key_type, ref, limit, prio, + &r, &l, &start, &stop); + + dev_dbg(rvu->dev, + "%s: ref=%d limit=%d r=%d l=%d start=%d stop=%d tot=%d count=%d rc=%d\n", + __func__, ref, limit, r, l, + start, stop, tot, count, rc); + + if (rc) + goto err; + + /* Find subbank and subbank index for ref */ + rc = npc_mcam_idx_2_subbank_idx(rvu, r, &sb1, + &sb_idx1); + if (rc) + goto err; + + dev_dbg(rvu->dev, + "%s: ref subbank=%d off=%d\n", + __func__, sb1->idx, sb_idx1); + + /* Skip subbank if it is not available for the keytype */ + if (!npc_subbank_suits(sb1, key_type)) { + dev_dbg(rvu->dev, + "%s: not suitable sb=%d key_type=%d\n", + __func__, sb1->idx, key_type); + continue; + } + + /* Find subbank and subbank index for limit */ + rc = npc_mcam_idx_2_subbank_idx(rvu, l, &sb2, + &sb_idx2); + if (rc) + goto err; + + dev_dbg(rvu->dev, + "%s: limit subbank=%d off=%d\n", + __func__, sb_idx1, sb_idx2); + + /* subbank of ref and limit should be same */ + if (sb1 != sb2) { + dev_err(rvu->dev, + "%s: l(%d) and r(%d) are not in same subbank\n", + __func__, r, l); + goto err; + } + + if (contig && + npc_subbank_free_cnt(rvu, sb1, key_type) < count) { + dev_dbg(rvu->dev, "%s: less count =%d\n", + __func__, + npc_subbank_free_cnt(rvu, sb1, key_type)); + continue; + } + + /* Try in one bank of a subbank */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb1, key_type, + r, l, prio, contig, + count - tot, mcam_idx + tot, + count - tot, max_alloc, + &alloc_cnt); + + tot += alloc_cnt; + + dev_dbg(rvu->dev, "%s: Allocated tot=%d alloc_cnt=%d\n", + __func__, tot, alloc_cnt); + + if (!rc && count == tot) + return 0; + } +err: + dev_dbg(rvu->dev, "%s: Error to allocate\n", + __func__); + + /* non contiguous allocation fails. We need to do clean up */ + if (max_alloc) { + rc = npc_idx_free(rvu, mcam_idx, tot, false); + if (rc) + dev_err(rvu->dev, + "%s: failed to free %u indexes\n", + __func__, tot); + } + + return -EFAULT; +} + +/* Minimize allocation from bottom and top subbanks for noref allocations. + * Default allocations are ref based, and will be allocated from top + * subbanks (least priority subbanks). Since default allocation is at very + * early stage of kernel netdev probes, this subbanks will be moved to + * used subbanks list. This will pave a way for noref allocation from these + * used subbanks. Skip allocation for these top and bottom, and try free + * bank next. If none slot is available, come back and search in these + * subbanks. + */ + +static int npc_subbank_restricted_idxs[2]; +static bool restrict_valid = true; + +static bool npc_subbank_restrict_usage(struct rvu *rvu, int index) +{ + int i; + + if (!restrict_valid) + return false; + + for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) { + if (index == npc_subbank_restricted_idxs[i]) + return true; + } + + return false; +} + +static int npc_subbank_noref_alloc(struct rvu *rvu, int key_type, bool contig, + int count, u16 *mcam_idx) +{ + struct npc_subbank *sb; + unsigned long index; + int tot = 0, rc; + bool max_alloc; + int alloc_cnt; + int idx, i; + void *val; + + max_alloc = !contig; + + /* Check used subbanks for free slots */ + xa_for_each(&npc_priv.xa_sb_used, index, val) { + idx = xa_to_value(val); + + /* Minimize allocation from restricted subbanks + * in noref allocations. + */ + if (npc_subbank_restrict_usage(rvu, idx)) + continue; + + sb = &npc_priv.sb[idx]; + + /* Skip if not suitable subbank */ + if (!npc_subbank_suits(sb, key_type)) + continue; + + if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) + continue; + + /* try in bank 0. Try passing ref and limit equal to + * subbank boundaries + */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, key_type, + sb->b0b, sb->b0t, 0, + contig, count - tot, + mcam_idx + tot, + count - tot, + max_alloc, &alloc_cnt); + + /* Non contiguous allocation may allocate less than + * requested 'count'. + */ + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + /* Successfully allocated */ + if (!rc && count == tot) + return 0; + + /* x4 entries can be allocated from bank 0 only */ + if (key_type == NPC_MCAM_KEY_X4) + continue; + + /* try in bank 1 for x2 */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, key_type, + sb->b1b, sb->b1t, 0, + contig, count - tot, + mcam_idx + tot, + count - tot, max_alloc, + &alloc_cnt); + + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + if (!rc && count == tot) + return 0; + } + + /* Allocate in free subbanks */ + xa_for_each(&npc_priv.xa_sb_free, index, val) { + idx = xa_to_value(val); + sb = &npc_priv.sb[idx]; + + /* Minimize allocation from restricted subbanks + * in noref allocations. + */ + if (npc_subbank_restrict_usage(rvu, idx)) + continue; + + if (!npc_subbank_suits(sb, key_type)) + continue; + + /* try in bank 0 */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, key_type, + sb->b0b, sb->b0t, 0, + contig, count - tot, + mcam_idx + tot, + count - tot, + max_alloc, &alloc_cnt); + + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + /* Successfully allocated */ + if (!rc && count == tot) + return 0; + + /* x4 entries can be allocated from bank 0 only */ + if (key_type == NPC_MCAM_KEY_X4) + continue; + + /* try in bank 1 for x2 */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, + key_type, sb->b1b, sb->b1t, 0, + contig, count - tot, + mcam_idx + tot, count - tot, + max_alloc, &alloc_cnt); + + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + if (!rc && count == tot) + return 0; + } + + /* Allocate from restricted subbanks */ + for (i = 0; restrict_valid && + (i < ARRAY_SIZE(npc_subbank_restricted_idxs)); i++) { + idx = npc_subbank_restricted_idxs[i]; + sb = &npc_priv.sb[idx]; + + /* Skip if not suitable subbank */ + if (!npc_subbank_suits(sb, key_type)) + continue; + + if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) + continue; + + /* try in bank 0. Try passing ref and limit equal to + * subbank boundaries + */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, key_type, + sb->b0b, sb->b0t, 0, + contig, count - tot, + mcam_idx + tot, + count - tot, + max_alloc, &alloc_cnt); + + /* Non contiguous allocation may allocate less than + * requested 'count'. + */ + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + /* Successfully allocated */ + if (!rc && count == tot) + return 0; + + /* x4 entries can be allocated from bank 0 only */ + if (key_type == NPC_MCAM_KEY_X4) + continue; + + /* try in bank 1 for x2 */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, key_type, + sb->b1b, sb->b1t, 0, + contig, count - tot, + mcam_idx + tot, + count - tot, max_alloc, + &alloc_cnt); + + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + if (!rc && count == tot) + return 0; + } + + /* non contiguous allocation fails. We need to do clean up */ + if (max_alloc) + npc_idx_free(rvu, mcam_idx, tot, false); + + dev_dbg(rvu->dev, "%s: non-contig allocation fails\n", + __func__); + + return -EFAULT; +} + +int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count) +{ + return npc_idx_free(rvu, mcam_idx, count, true); +} + +int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type, + int prio, u16 *mcam_idx, int ref, int limit, + bool contig, int count, bool virt) +{ + bool defrag_candidate = false; + int i, eidx, rc, bd; + bool ref_valid; + u16 vidx; + + bd = npc_priv.bank_depth; + + /* Special case: ref == 0 && limit= 0 && prio == HIGH && count == 1 + * Here user wants to allocate 0th entry + */ + if (!ref && !limit && prio == NPC_MCAM_HIGHER_PRIO && + count == 1) { + rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, + prio, contig, count, mcam_idx); + + if (rc) + return rc; + goto add2map; + } + + ref_valid = !!(limit || ref); + defrag_candidate = !ref_valid && !contig && virt; + if (!ref_valid) { + if (contig && count > npc_priv.subbank_depth) + goto try_noref_multi_subbank; + + rc = npc_subbank_noref_alloc(rvu, key_type, contig, + count, mcam_idx); + if (!rc) + goto add2map; + +try_noref_multi_subbank: + eidx = (key_type == NPC_MCAM_KEY_X4) ? bd - 1 : 2 * bd - 1; + + if (prio == NPC_MCAM_HIGHER_PRIO) + rc = npc_multi_subbank_ref_alloc(rvu, key_type, + eidx, 0, + NPC_MCAM_HIGHER_PRIO, + contig, count, + mcam_idx); + else + rc = npc_multi_subbank_ref_alloc(rvu, key_type, + 0, eidx, + NPC_MCAM_LOWER_PRIO, + contig, count, + mcam_idx); + + if (!rc) + goto add2map; + + return rc; + } + + if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || + (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { + dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", + __func__, ref, limit); + return -EINVAL; + } + + if ((key_type == NPC_MCAM_KEY_X4 && (ref >= bd || limit >= bd)) || + (key_type == NPC_MCAM_KEY_X2 && + (ref >= 2 * bd || limit >= 2 * bd))) { + dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", + __func__, ref, limit); + return -EINVAL; + } + + if (contig && count > npc_priv.subbank_depth) + goto try_ref_multi_subbank; + + rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, + prio, contig, count, mcam_idx); + if (!rc) + goto add2map; + +try_ref_multi_subbank: + rc = npc_multi_subbank_ref_alloc(rvu, key_type, + ref, limit, prio, + contig, count, mcam_idx); + if (!rc) + goto add2map; + + return rc; + +add2map: + for (i = 0; i < count; i++) { + rc = npc_add_to_pf_maps(rvu, mcam_idx[i], pcifunc); + if (rc) + goto err; + + if (!defrag_candidate) + continue; + + rc = npc_vidx_maps_add_entry(rvu, mcam_idx[i], pcifunc, &vidx); + if (rc) { + npc_del_from_pf_maps(rvu, mcam_idx[i]); + goto err; + } + + /* Return vidx to caller */ + mcam_idx[i] = vidx; + } + + return 0; +err: + for (int j = 0; j < i; j++) { + npc_del_from_pf_maps(rvu, npc_vidx2idx(mcam_idx[j])); + + if (!defrag_candidate) + continue; + + npc_vidx_maps_del_entry(rvu, mcam_idx[j], NULL); + } + + return rc; + +} + +void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free, + int *x4_free, int *sb_free) +{ + struct npc_subbank *sb; + int i; + + /* Reset all stats to zero */ + *x2_free = 0; + *x4_free = 0; + *sb_free = 0; + + for (i = 0; i < npc_priv.num_subbanks; i++) { + sb = &npc_priv.sb[i]; + mutex_lock(&sb->lock); + + /* Count number of free subbanks */ + if (sb->flags & NPC_SUBBANK_FLAG_FREE) { + (*sb_free)++; + goto next; + } + + /* Sumup x4 free count */ + if (sb->key_type == NPC_MCAM_KEY_X4) { + (*x4_free) += sb->free_cnt; + goto next; + } + + /* Sumup x2 free counts */ + (*x2_free) += sb->free_cnt; +next: + mutex_unlock(&sb->lock); + } +} + +int +rvu_mbox_handler_npc_cn20k_get_fcnt(struct rvu *rvu, + struct msg_req *req, + struct npc_cn20k_get_fcnt_rsp *rsp) +{ + npc_cn20k_subbank_calc_free(rvu, &rsp->free_x2, + &rsp->free_x4, &rsp->free_subbanks); + return 0; +} + +int +rvu_mbox_handler_npc_cn20k_get_kex_cfg(struct rvu *rvu, + struct msg_req *req, + struct npc_cn20k_get_kex_cfg_rsp *rsp) +{ + int extr, lt; + + rsp->rx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_RX); + rsp->tx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_TX); + + /* Get EXTRACTOR LID */ + for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { + rsp->intf_extr_lid[NIX_INTF_RX][extr] = + CN20K_GET_EXTR_LID(NIX_INTF_RX, extr); + rsp->intf_extr_lid[NIX_INTF_TX][extr] = + CN20K_GET_EXTR_LID(NIX_INTF_TX, extr); + } + + /* Get EXTRACTOR LTYPE */ + for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { + for (lt = 0; lt < NPC_MAX_LT; lt++) { + rsp->intf_extr_lt[NIX_INTF_RX][extr][lt] = + CN20K_GET_EXTR_LT(NIX_INTF_RX, extr, lt); + rsp->intf_extr_lt[NIX_INTF_TX][extr][lt] = + CN20K_GET_EXTR_LT(NIX_INTF_TX, extr, lt); + } + } + + memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); + return 0; +} + +static int *subbank_srch_order; + +static void npc_populate_restricted_idxs(int num_subbanks) +{ + npc_subbank_restricted_idxs[0] = num_subbanks - 1; + npc_subbank_restricted_idxs[1] = 0; +} + +static int npc_create_srch_order(int cnt) +{ + int val = 0; + + subbank_srch_order = kcalloc(cnt, sizeof(int), + GFP_KERNEL); + if (!subbank_srch_order) + return -ENOMEM; + + /* cnt(subbank depth) is always a power of 2. There is a check in + * npc_priv_init() to check the same. + */ + for (int i = 0; i < cnt; i += 2) { + subbank_srch_order[i] = cnt / 2 - val - 1; + subbank_srch_order[i + 1] = cnt / 2 + 1 + val; + val++; + } + + subbank_srch_order[cnt - 1] = cnt / 2; + return 0; +} + +static void npc_subbank_init(struct rvu *rvu, struct npc_subbank *sb, int idx) +{ + mutex_init(&sb->lock); + + sb->b0b = idx * npc_priv.subbank_depth; + sb->b0t = sb->b0b + npc_priv.subbank_depth - 1; + + sb->b1b = npc_priv.bank_depth + idx * npc_priv.subbank_depth; + sb->b1t = sb->b1b + npc_priv.subbank_depth - 1; + + sb->flags = NPC_SUBBANK_FLAG_FREE; + sb->idx = idx; + sb->arr_idx = subbank_srch_order[idx]; + + dev_dbg(rvu->dev, "%s: sb->idx=%u sb->arr_idx=%u\n", + __func__, sb->idx, sb->arr_idx); + + /* Keep first and last subbank at end of free array; so that + * it will be used at last + */ + xa_store(&npc_priv.xa_sb_free, sb->arr_idx, + xa_mk_value(sb->idx), GFP_KERNEL); +} + +static int npc_pcifunc_map_create(struct rvu *rvu) +{ + int pf, vf, numvfs; + int cnt = 0; + u16 pcifunc; + u64 cfg; + + for (pf = 0; pf < rvu->hw->total_pfs; pf++) { + cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); + numvfs = (cfg >> 12) & 0xFF; + + /* Skip not enabled PFs */ + if (!(cfg & BIT_ULL(20))) + goto chk_vfs; + + /* If Admin function, check on VFs */ + if (cfg & BIT_ULL(21)) + goto chk_vfs; + + pcifunc = pf << 9; + + xa_store(&npc_priv.xa_pf_map, (unsigned long)pcifunc, + xa_mk_value(cnt), GFP_KERNEL); + + cnt++; + +chk_vfs: + for (vf = 0; vf < numvfs; vf++) { + pcifunc = (pf << 9) | (vf + 1); + + xa_store(&npc_priv.xa_pf_map, (unsigned long)pcifunc, + xa_mk_value(cnt), GFP_KERNEL); + cnt++; + } + } + + return cnt; +} + +struct npc_defrag_node { + u8 idx; + u8 key_type; + bool valid; + bool refs; + u16 free_cnt; + u16 vidx_cnt; + u16 *vidx; + struct list_head list; +}; + +static bool npc_defrag_skip_restricted_sb(int sb_id) +{ + int i; + + if (!restrict_valid) + return false; + + for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) + if (sb_id == npc_subbank_restricted_idxs[i]) + return true; + return false; +} + +/* Find subbank with minimum number of virtual indexes */ +static struct npc_defrag_node *npc_subbank_min_vidx(struct list_head *lh) +{ + struct npc_defrag_node *node, *tnode = NULL; + int min = INT_MAX; + + list_for_each_entry(node, lh, list) { + if (!node->valid) + continue; + + /* if subbank has ref allocated mcam indexes, that subbank + * is not a good candidate to move out indexes. + */ + if (node->refs) + continue; + + if (min > node->vidx_cnt) { + min = node->vidx_cnt; + tnode = node; + } + } + + return tnode; +} + +/* Find subbank with maximum number of free spaces */ +static struct npc_defrag_node *npc_subbank_max_free(struct list_head *lh) +{ + struct npc_defrag_node *node, *tnode = NULL; + int max = INT_MIN; + + list_for_each_entry(node, lh, list) { + if (!node->valid) + continue; + + if (max < node->free_cnt) { + max = node->free_cnt; + tnode = node; + } + } + + return tnode; +} + +static int npc_defrag_alloc_free_slots(struct rvu *rvu, + struct npc_defrag_node *f, + int cnt, u16 *save) +{ + int alloc_cnt1, alloc_cnt2; + struct npc_subbank *sb; + int rc, sb_off, i; + bool deleted; + + sb = &npc_priv.sb[f->idx]; + + alloc_cnt1 = 0; + alloc_cnt2 = 0; + + rc = __npc_subbank_alloc(rvu, sb, + NPC_MCAM_KEY_X2, sb->b0b, + sb->b0t, + NPC_MCAM_LOWER_PRIO, + false, cnt, save, cnt, true, + &alloc_cnt1); + if (alloc_cnt1 < cnt) { + rc = __npc_subbank_alloc(rvu, sb, + NPC_MCAM_KEY_X2, sb->b1b, + sb->b1t, + NPC_MCAM_LOWER_PRIO, + false, cnt - alloc_cnt1, + save + alloc_cnt1, + cnt - alloc_cnt1, + true, &alloc_cnt2); + } + + if (alloc_cnt1 + alloc_cnt2 != cnt) { + dev_err(rvu->dev, + "%s: Failed to alloc cnt=%u alloc_cnt1=%u alloc_cnt2=%u\n", + __func__, cnt, alloc_cnt1, alloc_cnt2); + goto fail_free_alloc; + } + return 0; + +fail_free_alloc: + for (i = 0; i < alloc_cnt1 + alloc_cnt2; i++) { + rc = npc_mcam_idx_2_subbank_idx(rvu, save[i], + &sb, &sb_off); + if (rc) { + dev_err(rvu->dev, + "%s: Error to find subbank for mcam idx=%u\n", + __func__, save[i]); + break; + } + + deleted = __npc_subbank_free(rvu, sb, sb_off); + if (!deleted) { + dev_err(rvu->dev, + "%s: Error to free mcam idx=%u\n", + __func__, save[i]); + break; + } + } + + return rc; +} + +static int npc_defrag_add_2_show_list(struct rvu *rvu, u16 old_midx, + u16 new_midx, u16 vidx) +{ + struct npc_defrag_show_node *node; + + node = kcalloc(1, sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + node->old_midx = old_midx; + node->new_midx = new_midx; + node->vidx = vidx; + INIT_LIST_HEAD(&node->list); + + mutex_lock(&npc_priv.lock); + list_add_tail(&node->list, &npc_priv.defrag_lh); + mutex_unlock(&npc_priv.lock); + + return 0; +} + +static +int npc_defrag_move_vdx_to_free(struct rvu *rvu, + struct npc_defrag_node *f, + struct npc_defrag_node *v, + int cnt, u16 *save) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + int i, vidx_cnt, rc, sb_off; + u16 new_midx, old_midx, vidx; + struct npc_subbank *sb; + bool deleted; + u16 pcifunc; + int blkaddr; + void *map; + u8 bank; + u16 midx; + u64 stats; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + + vidx_cnt = v->vidx_cnt; + for (i = 0; i < cnt; i++) { + vidx = v->vidx[vidx_cnt - i - 1]; + old_midx = npc_vidx2idx(vidx); + new_midx = save[cnt - i - 1]; + + dev_dbg(rvu->dev, + "%s: Moving %u ---> %u (vidx=%u)\n", + __func__, + old_midx, new_midx, vidx); + + rc = npc_defrag_add_2_show_list(rvu, old_midx, new_midx, vidx); + if (rc) + dev_err(rvu->dev, + "%s: Error happened to add to show list vidx=%u\n", + __func__, vidx); + + /* Modify vidx to point to new mcam idx */ + rc = npc_vidx_maps_modify(rvu, vidx, new_midx); + if (rc) + return rc; + + midx = old_midx % mcam->banksize; + bank = old_midx / mcam->banksize; + stats = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, + bank)); + + npc_cn20k_enable_mcam_entry(rvu, blkaddr, old_midx, false); + npc_cn20k_copy_mcam_entry(rvu, blkaddr, old_midx, new_midx); + npc_cn20k_enable_mcam_entry(rvu, blkaddr, new_midx, true); + + midx = new_midx % mcam->banksize; + bank = new_midx / mcam->banksize; + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, bank), + stats); + + /* Free the old mcam idx */ + rc = npc_mcam_idx_2_subbank_idx(rvu, old_midx, + &sb, &sb_off); + if (rc) { + dev_err(rvu->dev, + "%s: Unable to calculate subbank off for mcamidx=%u\n", + __func__, old_midx); + return rc; + } + + deleted = __npc_subbank_free(rvu, sb, sb_off); + if (!deleted) { + dev_err(rvu->dev, + "%s: Failed to free mcamidx=%u sb=%u sb_off=%u\n", + __func__, old_midx, sb->idx, sb_off); + return -EFAULT; + } + + /* save pcifunc */ + map = xa_load(&npc_priv.xa_idx2pf_map, old_midx); + pcifunc = xa_to_value(map); + + /* delete from pf maps */ + rc = npc_del_from_pf_maps(rvu, old_midx); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to delete pf maps for mcamidx=%u\n", + __func__, old_midx); + return rc; + } + + /* add new mcam_idx to pf map */ + rc = npc_add_to_pf_maps(rvu, new_midx, pcifunc); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to add pf maps for mcamidx=%u\n", + __func__, new_midx); + return rc; + } + + /* Remove from mcam maps */ + mcam->entry2pfvf_map[old_midx] = NPC_MCAM_INVALID_MAP; + mcam->entry2cntr_map[old_midx] = NPC_MCAM_INVALID_MAP; + npc_mcam_clear_bit(mcam, old_midx); + + mcam->entry2pfvf_map[new_midx] = pcifunc; + /* Counter is not preserved */ + mcam->entry2cntr_map[new_midx] = new_midx; + npc_mcam_set_bit(mcam, new_midx); + + /* Mark as invalid */ + v->vidx[vidx_cnt - i - 1] = -1; + save[cnt - i - 1] = -1; + + f->free_cnt--; + v->vidx_cnt--; + } + + return 0; +} + +static int npc_defrag_process(struct rvu *rvu, struct list_head *lh) +{ + struct npc_defrag_node *v = NULL; + struct npc_defrag_node *f = NULL; + int rc = 0, cnt; + u16 *save; + + while (1) { + /* Find subbank with minimum vidx */ + if (!v) { + v = npc_subbank_min_vidx(lh); + if (!v) + break; + } + + /* Find subbank with maximum free slots */ + if (!f) { + f = npc_subbank_max_free(lh); + if (!f) + break; + } + + if (!v->vidx_cnt) { + list_del_init(&v->list); + v = NULL; + continue; + } + + if (!f->free_cnt) { + list_del_init(&f->list); + f = NULL; + continue; + } + + /* If both subbanks are same, choose vidx and + * search for free list again + */ + if (f == v) { + list_del_init(&f->list); + f = NULL; + continue; + } + + /* Calculate minimum free slots needs to be allocated */ + cnt = f->free_cnt > v->vidx_cnt ? v->vidx_cnt : + f->free_cnt; + + dev_dbg(rvu->dev, + "%s: cnt=%u free_cnt=%u(sb=%u) vidx_cnt=%u(sb=%u)\n", + __func__, cnt, f->free_cnt, f->idx, + v->vidx_cnt, v->idx); + + /* Allocate an array to store newly allocated + * free slots (mcam indexes) + */ + save = kcalloc(cnt, sizeof(*save), GFP_KERNEL); + if (!save) { + rc = -ENOMEM; + goto err; + } + + /* Alloc free slots for existing vidx */ + rc = npc_defrag_alloc_free_slots(rvu, f, cnt, save); + if (rc) { + kfree(save); + goto err; + } + + /* Move vidx to free slots; update pf_map and vidx maps, + * and free existing vidx mcam slots + */ + rc = npc_defrag_move_vdx_to_free(rvu, f, v, cnt, save); + if (rc) { + kfree(save); + goto err; + } + + kfree(save); + + if (!f->free_cnt) { + list_del_init(&f->list); + f = NULL; + } + + if (!v->vidx_cnt) { + list_del_init(&v->list); + v = NULL; + } + } + +err: + /* Whole defragmentation process is done within locks. if there + * is an error, it would be hard to roll back as index remove/add + * can fail again if it failed before. This would mean that there + * is bug in the index management algorithm. + * Return from here than rolling back. + */ + return rc; +} + +static void npc_defrag_list_clear(void) +{ + struct npc_defrag_show_node *node, *next; + + mutex_lock(&npc_priv.lock); + list_for_each_entry_safe(node, next, &npc_priv.defrag_lh, list) { + list_del_init(&node->list); + kfree(node); + } + + mutex_unlock(&npc_priv.lock); +} + +static void npc_lock_all_subbank(void) +{ + int i; + + for (i = 0; i < npc_priv.num_subbanks; i++) + mutex_lock(&npc_priv.sb[i].lock); +} + +static void npc_unlock_all_subbank(void) +{ + int i; + + for (i = npc_priv.num_subbanks - 1; i >= 0; i--) + mutex_unlock(&npc_priv.sb[i].lock); +} + +/* Only non-ref non-contigous mcam indexes + * are picked for defrag process + */ +int npc_cn20k_defrag(struct rvu *rvu) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + struct npc_defrag_node *node, *tnode; + struct list_head x4lh, x2lh, *lh; + int rc = 0, i, sb_off, tot; + struct npc_subbank *sb; + unsigned long index; + void *map; + u16 midx; + + /* Free previous show list */ + npc_defrag_list_clear(); + + INIT_LIST_HEAD(&x4lh); + INIT_LIST_HEAD(&x2lh); + + node = kcalloc(npc_priv.num_subbanks, sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + /* Lock mcam */ + mutex_lock(&mcam->lock); + npc_lock_all_subbank(); + + /* Fill in node with subbank properties */ + for (i = 0; i < npc_priv.num_subbanks; i++) { + sb = &npc_priv.sb[i]; + + node[i].idx = i; + node[i].key_type = sb->key_type; + node[i].free_cnt = sb->free_cnt; + node[i].vidx = kcalloc(npc_priv.subbank_depth * 2, + sizeof(*node[i].vidx), + GFP_KERNEL); + if (!node[i].vidx) { + rc = -ENOMEM; + goto free_vidx; + } + + /* If subbank is empty, dont include it in defrag + * process + */ + if (sb->flags & NPC_SUBBANK_FLAG_FREE) { + node[i].valid = false; + continue; + } + + if (npc_defrag_skip_restricted_sb(i)) { + node[i].valid = false; + continue; + } + + node[i].valid = true; + INIT_LIST_HEAD(&node[i].list); + + /* Add node to x2 or x4 list */ + lh = sb->key_type == NPC_MCAM_KEY_X2 ? &x2lh : &x4lh; + list_add_tail(&node[i].list, lh); + } + + /* Filling vidx[] array with all vidx in that subbank */ + xa_for_each_start(&npc_priv.xa_vidx2idx_map, index, map, + npc_priv.bank_depth * 2) { + midx = xa_to_value(map); + rc = npc_mcam_idx_2_subbank_idx(rvu, midx, + &sb, &sb_off); + if (rc) { + dev_err(rvu->dev, + "%s: Error to get mcam_idx for vidx=%lu\n", + __func__, index); + goto free_vidx; + } + + tnode = &node[sb->idx]; + tnode->vidx[tnode->vidx_cnt] = index; + tnode->vidx_cnt++; + } + + /* Mark all subbank which has ref allocation */ + for (i = 0; i < npc_priv.num_subbanks; i++) { + tnode = &node[i]; + + if (!tnode->valid) + continue; + + tot = (tnode->key_type == NPC_MCAM_KEY_X2) ? + npc_priv.subbank_depth * 2 : npc_priv.subbank_depth; + + if (node[i].vidx_cnt != tot - tnode->free_cnt) + tnode->refs = true; + } + + rc = npc_defrag_process(rvu, &x2lh); + if (rc) + goto free_vidx; + + rc = npc_defrag_process(rvu, &x4lh); + if (rc) + goto free_vidx; + +free_vidx: + npc_unlock_all_subbank(); + mutex_unlock(&mcam->lock); + for (i = 0; i < npc_priv.num_subbanks; i++) + kfree(node[i].vidx); + kfree(node); + return rc; +} + +int rvu_mbox_handler_npc_defrag(struct rvu *rvu, struct msg_req *req, + struct msg_rsp *rsp) +{ + return npc_cn20k_defrag(rvu); +} + +int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast, + u16 *mcast, u16 *promisc, u16 *ucast) +{ + u16 *ptr[4] = {promisc, mcast, bcast, ucast}; + unsigned long idx; + bool set = false; + void *val; + int i, j; + + if (!npc_priv.init_done) + return 0; + + if (is_lbk_vf(rvu, pcifunc)) { + if (!ptr[0]) + return -EINVAL; + + idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); + val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx); + if (!val) { + pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], + pcifunc); + + *ptr[0] = USHRT_MAX; + return -ESRCH; + } + + *ptr[0] = xa_to_value(val); + return 0; + } + + if (is_vf(pcifunc)) { + if (!ptr[3]) + return -EINVAL; + + idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); + val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx); + if (!val) { + pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], + pcifunc); + + *ptr[3] = USHRT_MAX; + return -ESRCH; + } + + *ptr[3] = xa_to_value(val); + return 0; + } + + for (i = NPC_DFT_RULE_START_ID, j = 0; i < NPC_DFT_RULE_MAX_ID; i++, + j++) { + if (!ptr[j]) + continue; + + idx = NPC_DFT_RULE_ID_MK(pcifunc, i); + val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx); + if (!val) { + pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", + __func__, + npc_dft_rule_name[i], pcifunc); + + *ptr[j] = USHRT_MAX; + continue; + } + + *ptr[j] = xa_to_value(val); + set = true; + } + + return set ? 0 : -ESRCH; +} + +int rvu_mbox_handler_npc_get_pfl_info(struct rvu *rvu, struct msg_req *req, + struct npc_get_pfl_info_rsp *rsp) +{ + if (!is_cn20k(rvu->pdev)) { + dev_err(rvu->dev, "Mbox support is only for cn20k\n"); + return -EOPNOTSUPP; + } + + rsp->kw_type = npc_priv.kw; + rsp->x4_slots = npc_priv.bank_depth; + return 0; +} + +int rvu_mbox_handler_npc_get_num_kws(struct rvu *rvu, + struct npc_get_num_kws_req *req, + struct npc_get_num_kws_rsp *rsp) +{ + u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; + u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; + struct rvu_npc_mcam_rule dummy = { 0 }; + struct mcam_entry_mdata mdata = { }; + struct npc_install_flow_req *fl; + int i, cnt = 0, blkaddr; + + if (!is_cn20k(rvu->pdev)) { + dev_err(rvu->dev, "Mbox support is only for cn20k\n"); + return -EOPNOTSUPP; + } + + fl = &req->fl; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) { + dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); + return NPC_MCAM_INVALID_REQ; + } + + mdata.kw = kw; + mdata.kw_mask = kw_mask; + + npc_update_flow(rvu, &mdata, fl->features, &fl->packet, + &fl->mask, &dummy, fl->intf, blkaddr); + + /* Find the most significant word valid. Traverse from + * MSB to LSB, check if cam0 or cam1 is set + */ + for (i = NPC_KWS_IN_KEY_SZ_MAX - 1; i >= 0; i--) { + if (kw[i] || kw_mask[i]) { + cnt = i + 1; + break; + } + } + + rsp->kws = cnt; + + return 0; +} + +int rvu_mbox_handler_npc_get_dft_rl_idxs(struct rvu *rvu, struct msg_req *req, + struct npc_get_dft_rl_idxs_rsp *rsp) +{ + u16 bcast, mcast, promisc, ucast; + u16 pcifunc; + int rc; + + if (!is_cn20k(rvu->pdev)) { + dev_err(rvu->dev, "Mbox support is only for cn20k\n"); + return -EOPNOTSUPP; + } + + pcifunc = req->hdr.pcifunc; + + rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, + &promisc, &ucast); + if (rc) + return rc; + + rsp->bcast = bcast; + rsp->mcast = mcast; + rsp->promisc = promisc; + rsp->ucast = ucast; + return 0; +} + +static bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc) +{ + return is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc)) || + is_lbk_vf(rvu, pcifunc); +} + +void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc) +{ + struct npc_mcam_free_entry_req free_req = { 0 }; + unsigned long index; + struct msg_rsp rsp; + u16 ptr[4]; + int rc, i; + void *map; + + if (!npc_priv.init_done) + return; + + if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { + dev_dbg(rvu->dev, + "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", + __func__, pcifunc); + return; + } + + rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1], + &ptr[2], &ptr[3]); + if (rc) + return; + + /* LBK */ + if (is_lbk_vf(rvu, pcifunc)) { + index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); + map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index); + if (!map) + dev_dbg(rvu->dev, + "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], + pcifunc); + + goto free_rules; + } + + /* VF */ + if (is_vf(pcifunc)) { + index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); + map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index); + if (!map) + dev_dbg(rvu->dev, + "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], + pcifunc); + + goto free_rules; + } + + /* PF */ + for (i = NPC_DFT_RULE_START_ID; i < NPC_DFT_RULE_MAX_ID; i++) { + index = NPC_DFT_RULE_ID_MK(pcifunc, i); + map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index); + if (!map) + dev_dbg(rvu->dev, + "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", + __func__, npc_dft_rule_name[i], + pcifunc); + } + +free_rules: + + free_req.hdr.pcifunc = pcifunc; + free_req.all = 1; + rc = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &rsp); + if (rc) + dev_err(rvu->dev, + "%s: Error deleting default entries (pcifunc=%#x\n", + __func__, pcifunc); +} + +int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc) +{ + struct npc_mcam_free_entry_req free_req = { 0 }; + u16 mcam_idx[4] = { 0 }, pf_ucast, pf_pcifunc; + struct npc_mcam_alloc_entry_req req = { 0 }; + struct npc_mcam_alloc_entry_rsp rsp = { 0 }; + int ret, eidx, i, k, pf, cnt; + struct rvu_pfvf *pfvf; + unsigned long index; + struct msg_rsp free_rsp; + u16 b, m, p, u; + + if (!npc_priv.init_done) + return 0; + + if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { + dev_dbg(rvu->dev, + "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", + __func__, pcifunc); + return 0; + } + + /* Check if default rules are already alloced for this pcifunc */ + ret = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &b, &m, &p, &u); + if (!ret) { + dev_dbg(rvu->dev, + "%s: default rules are already installed (pcifunc=%#x)\n", + __func__, pcifunc); + dev_dbg(rvu->dev, + "%s: bcast(%u) mcast(%u) promisc(%u) ucast(%u)\n", + __func__, b, m, p, u); + return 0; + } + + /* Set ref index as lowest priority index */ + eidx = 2 * npc_priv.bank_depth - 1; + + /* Install only UCAST for VF */ + cnt = is_vf(pcifunc) ? 1 : ARRAY_SIZE(mcam_idx); + + /* For VF pcifunc, allocate default mcam indexes by taking + * ref as PF's ucast index. + */ + if (is_vf(pcifunc)) { + pf = rvu_get_pf(rvu->pdev, pcifunc); + pf_pcifunc = pf << RVU_CN20K_PFVF_PF_SHIFT; + + /* Get PF's ucast entry index */ + ret = npc_cn20k_dft_rules_idx_get(rvu, pf_pcifunc, NULL, + NULL, NULL, &pf_ucast); + + /* There is no PF rules installed; and VF installation comes + * first. PF may come later. + * TODO: Install PF rules before installing VF rules. + */ + + /* Set PF's ucast as ref entry */ + if (!ret) + eidx = pf_ucast; + } + + pfvf = rvu_get_pfvf(rvu, pcifunc); + pfvf->hw_prio = NPC_DFT_RULE_PRIO; + + req.contig = false; + req.ref_prio = NPC_MCAM_HIGHER_PRIO; + req.ref_entry = eidx; + req.kw_type = NPC_MCAM_KEY_X2; + req.count = cnt; + req.hdr.pcifunc = pcifunc; + + ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); + + /* successfully allocated index */ + if (!ret) { + /* Copy indexes to local array */ + for (i = 0; i < cnt; i++) + mcam_idx[i] = rsp.entry_list[i]; + + goto chk_sanity; + } + + /* If there is no slots available and request is for PF, + * return error. + */ + if (!is_vf(pcifunc)) { + dev_err(rvu->dev, + "%s: Default index allocation failed for pcifunc=%#x\n", + __func__, pcifunc); + return ret; + } + + /* We could not find an index with higher priority index for VF. + * Find rule with lower priority index and set hardware priority + * as NPC_DFT_RULE_PRIO - 1 (higher hw priority) + */ + req.contig = false; + req.kw_type = NPC_MCAM_KEY_X2; + req.count = cnt; + req.hdr.pcifunc = pcifunc; + req.ref_prio = NPC_MCAM_LOWER_PRIO; + req.ref_entry = eidx + 1; + ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); + if (ret) { + dev_err(rvu->dev, + "%s: Default index allocation failed for pcifunc=%#x\n", + __func__, pcifunc); + return ret; + } + + /* Copy indexes to local array */ + for (i = 0; i < cnt; i++) + mcam_idx[i] = rsp.entry_list[i]; + + pfvf->hw_prio = NPC_DFT_RULE_PRIO - 1; + +chk_sanity: + /* LBK */ + if (is_lbk_vf(rvu, pcifunc)) { + index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); + ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index, + xa_mk_value(mcam_idx[0]), GFP_KERNEL); + if (ret) { + dev_err(rvu->dev, + "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], + pcifunc); + goto err; + } + + goto done; + } + + /* VF */ + if (is_vf(pcifunc)) { + index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); + ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index, + xa_mk_value(mcam_idx[0]), GFP_KERNEL); + if (ret) { + dev_err(rvu->dev, + "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], + pcifunc); + goto err; + } + + goto done; + } + + /* PF */ + for (i = NPC_DFT_RULE_START_ID, k = 0; i < NPC_DFT_RULE_MAX_ID && + k < cnt; i++, k++) { + index = NPC_DFT_RULE_ID_MK(pcifunc, i); + ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index, + xa_mk_value(mcam_idx[k]), GFP_KERNEL); + if (ret) { + dev_err(rvu->dev, + "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", + __func__, npc_dft_rule_name[i], + pcifunc); + for (int p = NPC_DFT_RULE_START_ID; p < i; p++) { + index = NPC_DFT_RULE_ID_MK(pcifunc, p); + xa_erase(&npc_priv.xa_pf2dfl_rmap, index); + } + goto err; + } + } + +done: + return 0; + +err: + free_req.hdr.pcifunc = pcifunc; + free_req.all = 1; + ret = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp); + if (ret) + dev_err(rvu->dev, + "%s: Error deleting default entries (pcifunc=%#x\n", + __func__, pcifunc); + + return -EFAULT; +} + +static int npc_priv_init(struct rvu *rvu) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + int blkaddr, num_banks, bank_depth; + int num_subbanks, subbank_depth; + u64 npc_const1, npc_const2 = 0; + struct npc_subbank *sb; + u64 cfg; + int i; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) { + dev_err(rvu->dev, "%s: NPC block not implemented\n", + __func__); + return -ENODEV; + } + + npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); + if (npc_const1 & BIT_ULL(63)) + npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); + + num_banks = mcam->banks; + bank_depth = mcam->banksize; + + num_subbanks = FIELD_GET(GENMASK_ULL(39, 32), npc_const2); + if (!num_subbanks) { + dev_err(rvu->dev, "Number of subbanks is zero\n"); + return -EFAULT; + } + + if (num_subbanks & (num_subbanks - 1)) { + dev_err(rvu->dev, + "subbanks cnt(%u) should be a power of 2\n", + num_subbanks); + return -EINVAL; + } + + npc_priv.num_subbanks = num_subbanks; + + subbank_depth = bank_depth / num_subbanks; + + npc_priv.bank_depth = bank_depth; + npc_priv.subbank_depth = subbank_depth; + + /* Get kex configured key size */ + cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(0)); + npc_priv.kw = FIELD_GET(GENMASK_ULL(34, 32), cfg); + + dev_info(rvu->dev, + "banks=%u depth=%u, subbanks=%u depth=%u, key type=%s\n", + num_banks, bank_depth, num_subbanks, subbank_depth, + npc_kw_name[npc_priv.kw]); + + npc_priv.sb = kcalloc(num_subbanks, sizeof(struct npc_subbank), + GFP_KERNEL); + if (!npc_priv.sb) + return -ENOMEM; + + xa_init_flags(&npc_priv.xa_sb_used, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_sb_free, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_idx2pf_map, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_pf_map, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_pf2dfl_rmap, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_idx2vidx_map, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_vidx2idx_map, XA_FLAGS_ALLOC); + + if (npc_create_srch_order(num_subbanks)) + goto fail1; + + npc_populate_restricted_idxs(num_subbanks); + + /* Initialize subbanks */ + for (i = 0, sb = npc_priv.sb; i < num_subbanks; i++, sb++) + npc_subbank_init(rvu, sb, i); + + /* Get number of pcifuncs in the system */ + npc_priv.pf_cnt = npc_pcifunc_map_create(rvu); + npc_priv.xa_pf2idx_map = kcalloc(npc_priv.pf_cnt, + sizeof(struct xarray), + GFP_KERNEL); + if (!npc_priv.xa_pf2idx_map) + goto fail2; + + for (i = 0; i < npc_priv.pf_cnt; i++) + xa_init_flags(&npc_priv.xa_pf2idx_map[i], XA_FLAGS_ALLOC); + + INIT_LIST_HEAD(&npc_priv.defrag_lh); + mutex_init(&npc_priv.lock); + + return 0; + +fail2: + kfree(subbank_srch_order); + subbank_srch_order = NULL; + +fail1: + xa_destroy(&npc_priv.xa_sb_used); + xa_destroy(&npc_priv.xa_sb_free); + xa_destroy(&npc_priv.xa_idx2pf_map); + xa_destroy(&npc_priv.xa_pf_map); + xa_destroy(&npc_priv.xa_pf2dfl_rmap); + xa_destroy(&npc_priv.xa_idx2vidx_map); + xa_destroy(&npc_priv.xa_vidx2idx_map); + kfree(npc_priv.sb); + npc_priv.sb = NULL; + return -ENOMEM; +} + +void npc_cn20k_deinit(struct rvu *rvu) +{ + int i; + + xa_destroy(&npc_priv.xa_sb_used); + xa_destroy(&npc_priv.xa_sb_free); + xa_destroy(&npc_priv.xa_idx2pf_map); + xa_destroy(&npc_priv.xa_pf_map); + xa_destroy(&npc_priv.xa_pf2dfl_rmap); + xa_destroy(&npc_priv.xa_idx2vidx_map); + xa_destroy(&npc_priv.xa_vidx2idx_map); + + for (i = 0; i < npc_priv.pf_cnt; i++) + xa_destroy(&npc_priv.xa_pf2idx_map[i]); + + kfree(npc_priv.xa_pf2idx_map); + /* No need to destroy mutex lock as it is + * part of subbank structure + */ + kfree(npc_priv.sb); + kfree(subbank_srch_order); +} + +static int npc_setup_mcam_section(struct rvu *rvu, int key_type) +{ + int blkaddr, sec; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) { + dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); + return -ENODEV; + } + + for (sec = 0; sec < npc_priv.num_subbanks; sec++) + rvu_write64(rvu, blkaddr, + NPC_AF_MCAM_SECTIONX_CFG_EXT(sec), key_type); + + return 0; +} + +int npc_cn20k_init(struct rvu *rvu) +{ + int err; + + err = npc_priv_init(rvu); + if (err) { + dev_err(rvu->dev, "%s: Error to init\n", + __func__); + return err; + } + + err = npc_setup_mcam_section(rvu, NPC_MCAM_KEY_X2); + if (err) { + dev_err(rvu->dev, "%s: mcam section configuration failure\n", + __func__); + return err; + } + + npc_priv.init_done = true; + + return 0; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h new file mode 100644 index 000000000000..815d0b257a7e --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h @@ -0,0 +1,340 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell RVU Admin Function driver + * + * Copyright (C) 2026 Marvell. + * + */ + +#ifndef NPC_CN20K_H +#define NPC_CN20K_H + +#define MKEX_CN20K_SIGN 0x19bbfdbd160 + +#define MAX_NUM_BANKS 2 +#define MAX_NUM_SUB_BANKS 32 +#define MAX_SUBBANK_DEPTH 256 + +/* strtoull of "mkexprof" with base:36 */ +#define MKEX_END_SIGN 0xdeadbeef + +#define NPC_CN20K_BYTESM GENMASK_ULL(18, 16) +#define NPC_CN20K_PARSE_NIBBLE GENMASK_ULL(22, 0) +#define NPC_CN20K_TOTAL_NIBBLE 23 + +#define CN20K_SET_EXTR_LT(intf, extr, ltype, cfg) \ + rvu_write64(rvu, BLKADDR_NPC, \ + NPC_AF_INTFX_EXTRACTORX_LTX_CFG(intf, extr, ltype), cfg) + +#define CN20K_GET_KEX_CFG(intf) \ + rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) + +#define CN20K_GET_EXTR_LID(intf, extr) \ + rvu_read64(rvu, BLKADDR_NPC, \ + NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr)) + +#define CN20K_SET_EXTR_LT(intf, extr, ltype, cfg) \ + rvu_write64(rvu, BLKADDR_NPC, \ + NPC_AF_INTFX_EXTRACTORX_LTX_CFG(intf, extr, ltype), cfg) + +#define CN20K_GET_EXTR_LT(intf, extr, ltype) \ + rvu_read64(rvu, BLKADDR_NPC, \ + NPC_AF_INTFX_EXTRACTORX_LTX_CFG(intf, extr, ltype)) + +/* NPC_PARSE_KEX_S nibble definitions for each field */ +#define NPC_CN20K_PARSE_NIBBLE_CHAN GENMASK_ULL(2, 0) +#define NPC_CN20K_PARSE_NIBBLE_ERRLEV BIT_ULL(3) +#define NPC_CN20K_PARSE_NIBBLE_ERRCODE GENMASK_ULL(5, 4) +#define NPC_CN20K_PARSE_NIBBLE_L2L3_BCAST BIT_ULL(6) +#define NPC_CN20K_PARSE_NIBBLE_LA_FLAGS BIT_ULL(7) +#define NPC_CN20K_PARSE_NIBBLE_LA_LTYPE BIT_ULL(8) +#define NPC_CN20K_PARSE_NIBBLE_LB_FLAGS BIT_ULL(9) +#define NPC_CN20K_PARSE_NIBBLE_LB_LTYPE BIT_ULL(10) +#define NPC_CN20K_PARSE_NIBBLE_LC_FLAGS BIT_ULL(11) +#define NPC_CN20K_PARSE_NIBBLE_LC_LTYPE BIT_ULL(12) +#define NPC_CN20K_PARSE_NIBBLE_LD_FLAGS BIT_ULL(13) +#define NPC_CN20K_PARSE_NIBBLE_LD_LTYPE BIT_ULL(14) +#define NPC_CN20K_PARSE_NIBBLE_LE_FLAGS BIT_ULL(15) +#define NPC_CN20K_PARSE_NIBBLE_LE_LTYPE BIT_ULL(16) +#define NPC_CN20K_PARSE_NIBBLE_LF_FLAGS BIT_ULL(17) +#define NPC_CN20K_PARSE_NIBBLE_LF_LTYPE BIT_ULL(18) +#define NPC_CN20K_PARSE_NIBBLE_LG_FLAGS BIT_ULL(19) +#define NPC_CN20K_PARSE_NIBBLE_LG_LTYPE BIT_ULL(20) +#define NPC_CN20K_PARSE_NIBBLE_LH_FLAGS BIT_ULL(21) +#define NPC_CN20K_PARSE_NIBBLE_LH_LTYPE BIT_ULL(22) + +/* Rx parse key extract nibble enable */ +#define NPC_CN20K_PARSE_NIBBLE_INTF_RX (NPC_CN20K_PARSE_NIBBLE_CHAN | \ + NPC_CN20K_PARSE_NIBBLE_L2L3_BCAST | \ + NPC_CN20K_PARSE_NIBBLE_LA_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LB_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LC_FLAGS | \ + NPC_CN20K_PARSE_NIBBLE_LC_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LD_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LE_LTYPE) + +/* Tx parse key extract nibble enable */ +#define NPC_CN20K_PARSE_NIBBLE_INTF_TX (NPC_CN20K_PARSE_NIBBLE_LA_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LB_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LC_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LD_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LE_LTYPE) + +/** + * enum npc_subbank_flag - NPC subbank status + * + * subbank flag indicates whether the subbank is free + * or used. + * + * @NPC_SUBBANK_FLAG_UNINIT: Subbank is not initialized. + * @NPC_SUBBANK_FLAG_FREE: Subbank is free. + * @NPC_SUBBANK_FLAG_USED: Subbank is used. + */ +enum npc_subbank_flag { + NPC_SUBBANK_FLAG_UNINIT, + NPC_SUBBANK_FLAG_FREE = BIT(0), + NPC_SUBBANK_FLAG_USED = BIT(1), +}; + +/** + * enum npc_dft_rule_id - Default rule type + * + * Mcam default rule type. + * + * @NPC_DFT_RULE_START_ID: Not used + * @NPC_DFT_RULE_PROMISC_ID: promiscuous rule + * @NPC_DFT_RULE_MCAST_ID: multicast rule + * @NPC_DFT_RULE_BCAST_ID: broadcast rule + * @NPC_DFT_RULE_UCAST_ID: unicast rule + * @NPC_DFT_RULE_MAX_ID: Maximum index. + */ +enum npc_dft_rule_id { + NPC_DFT_RULE_START_ID = 1, + NPC_DFT_RULE_PROMISC_ID = NPC_DFT_RULE_START_ID, + NPC_DFT_RULE_MCAST_ID, + NPC_DFT_RULE_BCAST_ID, + NPC_DFT_RULE_UCAST_ID, + NPC_DFT_RULE_MAX_ID, +}; + +/** + * struct npc_subbank - Subbank fields. + * @b0b: Subbanks bottom index for bank0 + * @b1b: Subbanks bottom index for bank1 + * @b0t: Subbanks top index for bank0 + * @b1t: Subbanks top index for bank1 + * @flags: Subbank flags + * @lock: Mutex lock for flags and rsrc mofiication + * @b0map: Bitmap map for bank0 indexes + * @b1map: Bitmap map for bank1 indexes + * @idx: Subbank index + * @arr_idx: Index to the free array or used array + * @free_cnt: Number of free slots in the subbank. + * @key_type: X4 or X2 subbank. + * + * MCAM resource is divided horizontally into multiple subbanks and + * Resource allocation from each subbank is managed by this data + * structure. + */ +struct npc_subbank { + u16 b0t, b0b, b1t, b1b; + enum npc_subbank_flag flags; + struct mutex lock; /* Protect subbank resources */ + DECLARE_BITMAP(b0map, MAX_SUBBANK_DEPTH); + DECLARE_BITMAP(b1map, MAX_SUBBANK_DEPTH); + u16 idx; + u16 arr_idx; + u16 free_cnt; + u8 key_type; +}; + +/** + * struct npc_defrag_show_node - Defragmentation show node + * @old_midx: Old mcam index. + * @new_midx: New mcam index. + * @vidx: Virtual index + * @list: Linked list of these nodes + * + * This structure holds information on last defragmentation + * executed on mcam resource. + */ +struct npc_defrag_show_node { + u16 old_midx; + u16 new_midx; + u16 vidx; + struct list_head list; +}; + +/** + * struct npc_priv_t - NPC private structure. + * @bank_depth: Total entries in each bank. + * @num_banks: Number of banks. + * @num_subbanks: Number of subbanks. + * @subbank_depth: Depth of subbank. + * @kw: Kex configured key type. + * @sb: Subbank array. + * @xa_sb_used: Array of used subbanks. + * @xa_sb_free: Array of free subbanks. + * @xa_pf2idx_map: PF to mcam index map. + * @xa_idx2pf_map: Mcam index to PF map. + * @xa_pf_map: Pcifunc to index map. + * @pf_cnt: Number of PFs. + * @init_done: Indicates MCAM initialization is done. + * @xa_pf2dfl_rmap: PF to default rule index map. + * @xa_idx2vidx_map: Mcam index to virtual index map. + * @xa_vidx2idx_map: virtual index to mcam index map. + * @defrag_lh: Defrag list head. + * @lock: Lock for defrag list + * + * This structure is populated during probing time by reading + * HW csr registers. + */ +struct npc_priv_t { + int bank_depth; + const int num_banks; + int num_subbanks; + int subbank_depth; + u8 kw; + struct npc_subbank *sb; + struct xarray xa_sb_used; + struct xarray xa_sb_free; + struct xarray *xa_pf2idx_map; + struct xarray xa_idx2pf_map; + struct xarray xa_pf_map; + struct xarray xa_pf2dfl_rmap; + struct xarray xa_idx2vidx_map; + struct xarray xa_vidx2idx_map; + struct list_head defrag_lh; + struct mutex lock; /* protect defrag nodes */ + int pf_cnt; + bool init_done; +}; + +struct npc_kpm_action0 { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 rsvd_63_57 : 7; + u64 byp_count : 3; + u64 capture_ena : 1; + u64 parse_done : 1; + u64 next_state : 8; + u64 rsvd_43 : 1; + u64 capture_lid : 3; + u64 capture_ltype : 4; + u64 rsvd_32_35 : 4; + u64 capture_flags : 4; + u64 ptr_advance : 8; + u64 var_len_offset : 8; + u64 var_len_mask : 8; + u64 var_len_right : 1; + u64 var_len_shift : 3; +#else + u64 var_len_shift : 3; + u64 var_len_right : 1; + u64 var_len_mask : 8; + u64 var_len_offset : 8; + u64 ptr_advance : 8; + u64 capture_flags : 4; + u64 rsvd_32_35 : 4; + u64 capture_ltype : 4; + u64 capture_lid : 3; + u64 rsvd_43 : 1; + u64 next_state : 8; + u64 parse_done : 1; + u64 capture_ena : 1; + u64 byp_count : 3; + u64 rsvd_63_57 : 7; +#endif +}; + +struct npc_mcam_kex_extr { + /* MKEX Profle Header */ + u64 mkex_sign; /* "mcam-kex-profile" (8 bytes/ASCII characters) */ + u8 name[MKEX_NAME_LEN]; /* MKEX Profile name */ + u64 cpu_model; /* Format as profiled by CPU hardware */ + u64 kpu_version; /* KPU firmware/profile version */ + u64 reserved; /* Reserved for extension */ + + /* MKEX Profle Data */ + u64 keyx_cfg[NPC_MAX_INTF]; /* NPC_AF_INTF(0..1)_KEX_CFG */ +#define NPC_MAX_EXTRACTOR 24 + /* MKEX Extractor data */ + u64 intf_extr_lid[NPC_MAX_INTF][NPC_MAX_EXTRACTOR]; + /* KEX configuration per extractor */ + u64 intf_extr_lt[NPC_MAX_INTF][NPC_MAX_EXTRACTOR][NPC_MAX_LT]; +} __packed; + +struct npc_cn20k_kpu_profile_fwdata { +#define KPU_SIGN 0x00666f727075706b +#define KPU_NAME_LEN 32 + /* Maximum number of custom KPU entries supported by + * the built-in profile. + */ +#define KPU_CN20K_MAX_CST_ENT 6 + /* KPU Profle Header */ + __le64 signature; /* "kpuprof\0" (8 bytes/ASCII characters) */ + u8 name[KPU_NAME_LEN]; /* KPU Profile name */ + __le64 version; /* KPU profile version */ + u8 kpus; + u8 reserved[7]; + + /* Default MKEX profile to be used with this KPU profile. May be + * overridden with mkex_profile module parameter. + * Format is same as for the MKEX profile to streamline processing. + */ + struct npc_mcam_kex_extr mkex; + /* LTYPE values for specific HW offloaded protocols. */ + struct npc_lt_def_cfg lt_def; + /* Dynamically sized data: + * Custom KPU CAM and ACTION configuration entries. + * struct npc_kpu_fwdata kpu[kpus]; + */ + u8 data[]; +} __packed; + +struct rvu; + +struct npc_priv_t *npc_priv_get(void); +int npc_cn20k_init(struct rvu *rvu); +void npc_cn20k_deinit(struct rvu *rvu); + +void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free, + int *x4_free, int *sb_free); + +int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type, + int prio, u16 *mcam_idx, int ref, int limit, + bool contig, int count, bool virt); +int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count); +void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr); +struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void); +void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr, + const char *mkex_profile); +int npc_cn20k_apply_custom_kpu(struct rvu *rvu, + struct npc_kpu_profile_adapter *profile); + +void +npc_cn20k_update_action_entries_n_flags(struct rvu *rvu, + struct npc_kpu_profile_adapter *pfl); + +int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc); +void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc); + +int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast, + u16 *mcast, u16 *promisc, u16 *ucast); + +void npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index, + u8 intf, struct cn20k_mcam_entry *entry, + bool enable, u8 hw_prio, u8 req_kw_type); +void npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr, + int index, bool enable); +void npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, + u16 src, u16 dest); +void npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index, + struct cn20k_mcam_entry *entry, u8 *intf, + u8 *ena, u8 *hw_prio); +void npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, + int bank, int index); +int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type); +u16 npc_cn20k_vidx2idx(u16 index); +u16 npc_cn20k_idx2vidx(u16 idx); +int npc_cn20k_defrag(struct rvu *rvu); + +#endif /* NPC_CN20K_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h index affb39803120..8bfaa507ee50 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h @@ -78,4 +78,69 @@ #define RVU_MBOX_VF_INT_ENA_W1C (0x38) #define RVU_MBOX_VF_VFAF_TRIGX(a) (0x2000 | (a) << 3) + +#define NIX_GINT_INT (0x200) +#define NIX_GINT_INT_W1S (0x208) + +#define ALTAF_FLR BIT_ULL(0) +#define ALTAF_RDY BIT_ULL(1) + +/* NPC registers */ +#define NPC_AF_INTFX_EXTRACTORX_CFG(a, b) \ + (0x20c000ull | (a) << 16 | (b) << 8) +#define NPC_AF_INTFX_EXTRACTORX_LTX_CFG(a, b, c) \ + (0x204000ull | (a) << 16 | (b) << 8 | (c) << 3) +#define NPC_AF_KPMX_ENTRYX_CAMX(a, b, c) \ + (0x20000ull | (a) << 12 | (b) << 3 | (c) << 16) +#define NPC_AF_KPMX_ENTRYX_ACTION0(a, b) \ + (0x40000ull | (a) << 12 | (b) << 3) +#define NPC_AF_KPMX_ENTRYX_ACTION1(a, b) \ + (0x50000ull | (a) << 12 | (b) << 3) +#define NPC_AF_KPMX_ENTRY_DISX(a, b) (0x60000ull | (a) << 12 | (b) << 3) +#define NPC_AF_KPM_PASS2_CFG 0x10210 +#define NPC_AF_KPMX_PASS2_OFFSET(a) (0x60040ull | (a) << 12) +#define NPC_AF_MCAM_SECTIONX_CFG_EXT(a) (0xf000000ull | (a) << 3) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0x8000000ull | (a) << 4 | (b) << 20 | (c) << 3); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0x9000000ull | (a) << 4 | (b) << 20 | (c) << 3); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0x9400000ull | (a) << 4 | (b) << 20 | (c) << 3); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0x9800000ull | (a) << 4 | (b) << 20 | (c) << 3); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0x9c00000ull | (a) << 4 | (b) << 20 | (c) << 3); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(a, b) ({ \ + u64 offset; \ + offset = (0xa000000ull | (a) << 4 | (b) << 20); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0xc000000ull | (a) << 4 | (b) << 20 | (c) << 22); \ + offset; }) + +#define NPC_AF_INTFX_MISS_ACTX(a, b) (0xf003000 | (a) << 6 | (b) << 4) + +#define NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(a, b) ({ \ + u64 offset; \ + offset = (0xb000000ull | (a) << 4 | (b) << 20); \ + offset; }) + #endif /* RVU_MBOX_REG_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/common.h b/drivers/net/ethernet/marvell/octeontx2/af/common.h index 8a08bebf08c2..779413a383b7 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/common.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/common.h @@ -177,10 +177,6 @@ enum nix_scheduler { #define NIX_TX_ACTIONOP_MCAST (0x3ull) #define NIX_TX_ACTIONOP_DROP_VIOL (0x5ull) -#define NPC_MCAM_KEY_X1 0 -#define NPC_MCAM_KEY_X2 1 -#define NPC_MCAM_KEY_X4 2 - #define NIX_INTFX_RX(a) (0x0ull | (a) << 1) #define NIX_INTFX_TX(a) (0x1ull | (a) << 1) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h index a3e273126e4e..dc42c81c0942 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h @@ -52,6 +52,14 @@ #define MBOX_DIR_PFVF_UP 6 /* PF sends messages to VF */ #define MBOX_DIR_VFPF_UP 7 /* VF replies to PF */ +enum { + NPC_MCAM_KEY_X1 = 0, + NPC_MCAM_KEY_DYN = NPC_MCAM_KEY_X1, + NPC_MCAM_KEY_X2, + NPC_MCAM_KEY_X4, + NPC_MCAM_KEY_MAX, +}; + enum { TYPE_AFVF, TYPE_AFPF, @@ -275,6 +283,33 @@ M(NPC_GET_FIELD_HASH_INFO, 0x6013, npc_get_field_hash_info, M(NPC_GET_FIELD_STATUS, 0x6014, npc_get_field_status, \ npc_get_field_status_req, \ npc_get_field_status_rsp) \ +M(NPC_CN20K_MCAM_GET_FREE_COUNT, 0x6015, npc_cn20k_get_fcnt, \ + msg_req, npc_cn20k_get_fcnt_rsp) \ +M(NPC_CN20K_GET_KEX_CFG, 0x6016, npc_cn20k_get_kex_cfg, \ + msg_req, npc_cn20k_get_kex_cfg_rsp) \ +M(NPC_CN20K_MCAM_WRITE_ENTRY, 0x6017, npc_cn20k_mcam_write_entry, \ + npc_cn20k_mcam_write_entry_req, msg_rsp) \ +M(NPC_CN20K_MCAM_ALLOC_AND_WRITE_ENTRY, 0x6018, \ +npc_cn20k_mcam_alloc_and_write_entry, \ + npc_cn20k_mcam_alloc_and_write_entry_req, \ + npc_mcam_alloc_and_write_entry_rsp) \ +M(NPC_CN20K_MCAM_READ_ENTRY, 0x6019, npc_cn20k_mcam_read_entry, \ + npc_mcam_read_entry_req, \ + npc_cn20k_mcam_read_entry_rsp) \ +M(NPC_CN20K_MCAM_READ_BASE_RULE, 0x601a, npc_cn20k_read_base_steer_rule, \ + msg_req, npc_cn20k_mcam_read_base_rule_rsp) \ +M(NPC_MCAM_DEFRAG, 0x601b, npc_defrag, \ + msg_req, \ + msg_rsp) \ +M(NPC_MCAM_GET_NUM_KWS, 0x601c, npc_get_num_kws, \ + npc_get_num_kws_req, \ + npc_get_num_kws_rsp) \ +M(NPC_MCAM_GET_DFT_RL_IDXS, 0x601d, npc_get_dft_rl_idxs, \ + msg_req, \ + npc_get_dft_rl_idxs_rsp)\ +M(NPC_MCAM_GET_NPC_PFL_INFO, 0x601e, npc_get_pfl_info, \ + msg_req, \ + npc_get_pfl_info_rsp) \ /* NIX mbox IDs (range 0x8000 - 0xFFFF) */ \ M(NIX_LF_ALLOC, 0x8000, nix_lf_alloc, \ nix_lf_alloc_req, nix_lf_alloc_rsp) \ @@ -1527,9 +1562,11 @@ struct npc_mcam_alloc_entry_req { #define NPC_MCAM_ANY_PRIO 0 #define NPC_MCAM_LOWER_PRIO 1 #define NPC_MCAM_HIGHER_PRIO 2 - u8 priority; /* Lower or higher w.r.t ref_entry */ + u8 ref_prio; /* Lower or higher w.r.t ref_entry */ u16 ref_entry; u16 count; /* Number of entries requested */ + u8 kw_type; /* entry key type, valid for cn20k */ + u8 virt; /* Request virtual index */ }; struct npc_mcam_alloc_entry_rsp { @@ -1548,14 +1585,47 @@ struct npc_mcam_free_entry_req { u8 all; /* If all entries allocated to this PFVF to be freed */ }; +struct mcam_entry_mdata { + u64 *kw; + u64 *kw_mask; + u64 *action; + u64 *vtag_action; + u8 max_kw; +}; + +enum npc_kws_in_key_sz { + NPC_KWS_IN_KEY_SZ_7 = 7, + NPC_KWS_IN_KEY_SZ_8 = 8, + NPC_KWS_IN_KEY_SZ_MAX, +}; + struct mcam_entry { -#define NPC_MAX_KWS_IN_KEY 7 /* Number of keywords in max keywidth */ - u64 kw[NPC_MAX_KWS_IN_KEY]; - u64 kw_mask[NPC_MAX_KWS_IN_KEY]; + u64 kw[NPC_KWS_IN_KEY_SZ_7]; + u64 kw_mask[NPC_KWS_IN_KEY_SZ_7]; u64 action; u64 vtag_action; }; +struct cn20k_mcam_entry { + u64 kw[NPC_KWS_IN_KEY_SZ_8]; + u64 kw_mask[NPC_KWS_IN_KEY_SZ_8]; + u64 action; + u64 vtag_action; + u64 action2; +}; + +struct npc_cn20k_mcam_write_entry_req { + struct mbox_msghdr hdr; + struct cn20k_mcam_entry entry_data; + u16 entry; /* MCAM entry to write this match key */ + u16 cntr; /* Counter for this MCAM entry */ + u8 intf; /* Rx or Tx interface */ + u8 enable_entry;/* Enable this MCAM entry ? */ + u8 hw_prio; /* hardware priority, valid for cn20k */ + u8 req_kw_type; /* Type of kw which should be written */ + u64 reserved; /* reserved for future use */ +}; + struct npc_mcam_write_entry_req { struct mbox_msghdr hdr; struct mcam_entry entry_data; @@ -1564,6 +1634,8 @@ struct npc_mcam_write_entry_req { u8 intf; /* Rx or Tx interface */ u8 enable_entry;/* Enable this MCAM entry ? */ u8 set_cntr; /* Set counter for this entry ? */ + u8 hw_prio; /* hardware priority, valid for cn20k */ + u64 reserved; /* reserved for future use */ }; /* Enable/Disable a given entry */ @@ -1622,12 +1694,38 @@ struct npc_mcam_alloc_and_write_entry_req { struct mbox_msghdr hdr; struct mcam_entry entry_data; u16 ref_entry; - u8 priority; /* Lower or higher w.r.t ref_entry */ + u8 ref_prio; /* Lower or higher w.r.t ref_entry */ u8 intf; /* Rx or Tx interface */ u8 enable_entry;/* Enable this MCAM entry ? */ u8 alloc_cntr; /* Allocate counter and map ? */ }; +struct npc_cn20k_mcam_alloc_and_write_entry_req { + struct mbox_msghdr hdr; + struct cn20k_mcam_entry entry_data; + u16 ref_entry; + u8 ref_prio; /* Lower or higher w.r.t ref_entry */ + u8 intf; /* Rx or Tx interface */ + u8 enable_entry;/* Enable this MCAM entry ? */ + u8 hw_prio; /* hardware priority, valid for cn20k */ + u8 virt; /* Allocate virtual index */ + u8 req_kw_type; /* Key type to be written */ + u16 reserved[4]; /* reserved for future use */ +}; + +struct npc_cn20k_mcam_read_entry_rsp { + struct mbox_msghdr hdr; + struct cn20k_mcam_entry entry_data; + u8 intf; + u8 enable; + u8 hw_prio; /* valid for cn20k */ +}; + +struct npc_cn20k_mcam_read_base_rule_rsp { + struct mbox_msghdr hdr; + struct cn20k_mcam_entry entry; +}; + struct npc_mcam_alloc_and_write_entry_rsp { struct mbox_msghdr hdr; u16 entry; @@ -1653,6 +1751,19 @@ struct npc_get_kex_cfg_rsp { u8 mkex_pfl_name[MKEX_NAME_LEN]; }; +struct npc_cn20k_get_kex_cfg_rsp { + struct mbox_msghdr hdr; + u64 rx_keyx_cfg; /* NPC_AF_INTF(0)_KEX_CFG */ + u64 tx_keyx_cfg; /* NPC_AF_INTF(1)_KEX_CFG */ +#define NPC_MAX_EXTRACTOR 24 + /* MKEX Extractor data */ + u64 intf_extr_lid[NPC_MAX_INTF][NPC_MAX_EXTRACTOR]; + /* KEX configuration per extractor */ + u64 intf_extr_lt[NPC_MAX_INTF][NPC_MAX_EXTRACTOR][NPC_MAX_LT]; +#define MKEX_NAME_LEN 128 + u8 mkex_pfl_name[MKEX_NAME_LEN]; +}; + struct ptp_get_cap_rsp { struct mbox_msghdr hdr; #define PTP_CAP_HW_ATOMIC_UPDATE BIT_ULL(0) @@ -1765,11 +1876,53 @@ struct npc_install_flow_req { u8 vtag1_op; /* old counter value */ u16 cntr_val; + u8 hw_prio; + u8 req_kw_type; /* Key type to be written */ + u8 alloc_entry; /* only for cn20k */ +/* For now use any priority, once AF driver is changed to + * allocate least priority entry instead of mid zone then make + * NPC_MCAM_LEAST_PRIO as 3 + */ +#define NPC_MCAM_LEAST_PRIO NPC_MCAM_ANY_PRIO + u16 ref_prio; + u16 ref_entry; }; struct npc_install_flow_rsp { struct mbox_msghdr hdr; int counter; /* negative if no counter else counter number */ + u16 entry; + u8 kw_type; +}; + +struct npc_get_num_kws_req { + struct mbox_msghdr hdr; + struct npc_install_flow_req fl; + u32 rsvd[4]; +}; + +struct npc_get_num_kws_rsp { + struct mbox_msghdr hdr; + int kws; + u32 rsvd[4]; +}; + +struct npc_get_dft_rl_idxs_rsp { + struct mbox_msghdr hdr; + u16 bcast; + u16 mcast; + u16 promisc; + u16 ucast; + u16 vf_ucast; + u16 rsvd[7]; +}; + +struct npc_get_pfl_info_rsp { + struct mbox_msghdr hdr; + u16 x4_slots; + u8 kw_type; + u8 rsvd1[3]; + u32 rsvd2[4]; }; struct npc_delete_flow_req { @@ -1795,6 +1948,15 @@ struct npc_mcam_read_entry_rsp { struct mcam_entry entry_data; u8 intf; u8 enable; + u8 hw_prio; /* valid for cn20k */ +}; + +/* Available entries to use */ +struct npc_cn20k_get_fcnt_rsp { + struct mbox_msghdr hdr; + int free_x2; + int free_x4; + int free_subbanks; }; struct npc_mcam_read_base_rule_rsp { diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/npc.h index 6c3aca6f278d..cefc5d70f3e4 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/npc.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/npc.h @@ -429,6 +429,7 @@ struct nix_rx_action { /* NPC_AF_INTFX_KEX_CFG field masks */ #define NPC_PARSE_NIBBLE GENMASK_ULL(30, 0) +#define NPC_TOTAL_NIBBLE 31 /* NPC_PARSE_KEX_S nibble definitions for each field */ #define NPC_PARSE_NIBBLE_CHAN GENMASK_ULL(2, 0) @@ -643,6 +644,7 @@ struct rvu_npc_mcam_rule { u16 chan; u16 chan_mask; u8 lxmb; + u8 hw_prio; }; #endif /* NPC_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h index 41de72c8607f..db74f7fdf028 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h @@ -321,6 +321,18 @@ enum npc_kpu_lb_lflag { NPC_F_LB_L_FDSA, }; +enum npc_cn20k_kpu_lc_uflag { + NPC_CN20K_F_LC_U_MPLS_IN_IP = 0x20, + NPC_CN20K_F_LC_U_IP6_TUN_IP6 = 0x40, + NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP = 0x80, +}; + +enum npc_cn20k_kpu_lc_lflag { + NPC_CN20K_F_LC_L_IP_FRAG = 2, + NPC_CN20K_F_LC_L_IP6_FRAG, + NPC_CN20K_F_LC_L_6TO4, +}; + enum npc_kpu_lc_uflag { NPC_F_LC_U_UNK_PROTO = 0x10, NPC_F_LC_U_IP_FRAG = 0x20, @@ -489,7 +501,7 @@ enum NPC_ERRLEV_E { 0, 0, 0, 0, \ } -static struct npc_kpu_profile_action ikpu_action_entries[] = { +static struct npc_kpu_profile_action ikpu_action_entries[] __maybe_unused = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 12, 16, 20, 0, 0, @@ -1068,7 +1080,7 @@ static struct npc_kpu_profile_action ikpu_action_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu1_cam_entries[] = { +static struct npc_kpu_profile_cam kpu1_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -1878,7 +1890,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu2_cam_entries[] = { +static struct npc_kpu_profile_cam kpu2_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -2823,7 +2835,7 @@ static struct npc_kpu_profile_cam kpu2_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu3_cam_entries[] = { +static struct npc_kpu_profile_cam kpu3_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -3804,7 +3816,7 @@ static struct npc_kpu_profile_cam kpu3_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu4_cam_entries[] = { +static struct npc_kpu_profile_cam kpu4_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -4119,7 +4131,7 @@ static struct npc_kpu_profile_cam kpu4_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu5_cam_entries[] = { +static struct npc_kpu_profile_cam kpu5_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -5172,7 +5184,7 @@ static struct npc_kpu_profile_cam kpu5_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu6_cam_entries[] = { +static struct npc_kpu_profile_cam kpu6_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -5901,7 +5913,7 @@ static struct npc_kpu_profile_cam kpu6_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu7_cam_entries[] = { +static struct npc_kpu_profile_cam kpu7_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -6252,7 +6264,7 @@ static struct npc_kpu_profile_cam kpu7_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu8_cam_entries[] = { +static struct npc_kpu_profile_cam kpu8_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -7089,7 +7101,7 @@ static struct npc_kpu_profile_cam kpu8_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu9_cam_entries[] = { +static struct npc_kpu_profile_cam kpu9_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -7575,7 +7587,7 @@ static struct npc_kpu_profile_cam kpu9_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu10_cam_entries[] = { +static struct npc_kpu_profile_cam kpu10_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -7746,7 +7758,7 @@ static struct npc_kpu_profile_cam kpu10_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu11_cam_entries[] = { +static struct npc_kpu_profile_cam kpu11_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8061,7 +8073,7 @@ static struct npc_kpu_profile_cam kpu11_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu12_cam_entries[] = { +static struct npc_kpu_profile_cam kpu12_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8322,7 +8334,7 @@ static struct npc_kpu_profile_cam kpu12_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu13_cam_entries[] = { +static struct npc_kpu_profile_cam kpu13_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8340,7 +8352,7 @@ static struct npc_kpu_profile_cam kpu13_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu14_cam_entries[] = { +static struct npc_kpu_profile_cam kpu14_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8358,7 +8370,7 @@ static struct npc_kpu_profile_cam kpu14_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu15_cam_entries[] = { +static struct npc_kpu_profile_cam kpu15_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8565,7 +8577,7 @@ static struct npc_kpu_profile_cam kpu15_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu16_cam_entries[] = { +static struct npc_kpu_profile_cam kpu16_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8628,7 +8640,7 @@ static struct npc_kpu_profile_cam kpu16_cam_entries[] = { }, }; -static struct npc_kpu_profile_action kpu1_action_entries[] = { +static struct npc_kpu_profile_action kpu1_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -9368,7 +9380,7 @@ static struct npc_kpu_profile_action kpu1_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu2_action_entries[] = { +static struct npc_kpu_profile_action kpu2_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -10209,7 +10221,7 @@ static struct npc_kpu_profile_action kpu2_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu3_action_entries[] = { +static struct npc_kpu_profile_action kpu3_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -11082,7 +11094,7 @@ static struct npc_kpu_profile_action kpu3_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu4_action_entries[] = { +static struct npc_kpu_profile_action kpu4_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -11363,7 +11375,7 @@ static struct npc_kpu_profile_action kpu4_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu5_action_entries[] = { +static struct npc_kpu_profile_action kpu5_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -12300,7 +12312,7 @@ static struct npc_kpu_profile_action kpu5_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu6_action_entries[] = { +static struct npc_kpu_profile_action kpu6_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -12949,7 +12961,7 @@ static struct npc_kpu_profile_action kpu6_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu7_action_entries[] = { +static struct npc_kpu_profile_action kpu7_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -13262,7 +13274,7 @@ static struct npc_kpu_profile_action kpu7_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu8_action_entries[] = { +static struct npc_kpu_profile_action kpu8_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -14007,7 +14019,7 @@ static struct npc_kpu_profile_action kpu8_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu9_action_entries[] = { +static struct npc_kpu_profile_action kpu9_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -14440,7 +14452,7 @@ static struct npc_kpu_profile_action kpu9_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu10_action_entries[] = { +static struct npc_kpu_profile_action kpu10_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -14593,7 +14605,7 @@ static struct npc_kpu_profile_action kpu10_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu11_action_entries[] = { +static struct npc_kpu_profile_action kpu11_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -14874,7 +14886,7 @@ static struct npc_kpu_profile_action kpu11_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu12_action_entries[] = { +static struct npc_kpu_profile_action kpu12_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -15107,7 +15119,7 @@ static struct npc_kpu_profile_action kpu12_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu13_action_entries[] = { +static struct npc_kpu_profile_action kpu13_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -15124,7 +15136,7 @@ static struct npc_kpu_profile_action kpu13_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu14_action_entries[] = { +static struct npc_kpu_profile_action kpu14_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -15141,7 +15153,7 @@ static struct npc_kpu_profile_action kpu14_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu15_action_entries[] = { +static struct npc_kpu_profile_action kpu15_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -15326,7 +15338,7 @@ static struct npc_kpu_profile_action kpu15_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu16_action_entries[] = { +static struct npc_kpu_profile_action kpu16_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -15383,7 +15395,7 @@ static struct npc_kpu_profile_action kpu16_action_entries[] = { }, }; -static struct npc_kpu_profile npc_kpu_profiles[] = { +static struct npc_kpu_profile npc_kpu_profiles[] __maybe_unused = { { ARRAY_SIZE(kpu1_cam_entries), ARRAY_SIZE(kpu1_action_entries), @@ -15482,7 +15494,7 @@ static struct npc_kpu_profile npc_kpu_profiles[] = { }, }; -static struct npc_lt_def_cfg npc_lt_defaults = { +static struct npc_lt_def_cfg npc_lt_defaults __maybe_unused = { .rx_ol2 = { .lid = NPC_LID_LA, .ltype_match = NPC_LT_LA_ETHER, @@ -15604,7 +15616,7 @@ static struct npc_lt_def_cfg npc_lt_defaults = { }, }; -static struct npc_mcam_kex npc_mkex_default = { +static struct npc_mcam_kex npc_mkex_default __maybe_unused = { .mkex_sign = MKEX_SIGN, .name = "default", .kpu_version = NPC_KPU_PROFILE_VER, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index 5b1129558e8b..1c8f657719b2 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -22,6 +22,7 @@ #include "rvu_npc_hash.h" #include "cn20k/reg.h" #include "cn20k/api.h" +#include "cn20k/npc.h" #define DRV_NAME "rvu_af" #define DRV_STRING "Marvell OcteonTX2 RVU Admin Function Driver" @@ -599,6 +600,7 @@ static void rvu_check_min_msix_vec(struct rvu *rvu, int nvecs, int pf, int vf) static int rvu_setup_msix_resources(struct rvu *rvu) { + struct altaf_intr_notify *altaf_intr_data; struct rvu_hwinfo *hw = rvu->hw; int pf, vf, numvfs, hwvf, err; int nvecs, offset, max_msix; @@ -705,7 +707,30 @@ static int rvu_setup_msix_resources(struct rvu *rvu) rvu->msix_base_iova = iova; rvu->msixtr_base_phy = phy_addr; + if (is_rvu_otx2(rvu) || is_cn20k(rvu->pdev)) + return 0; + + if (!rvu->fwdata) + goto fail; + + altaf_intr_data = &rvu->fwdata->altaf_intr_info; + if (altaf_intr_data->gint_paddr) { + iova = dma_map_resource(rvu->dev, altaf_intr_data->gint_paddr, + PCI_MSIX_ENTRY_SIZE, + DMA_BIDIRECTIONAL, 0); + + if (dma_mapping_error(rvu->dev, iova)) + goto fail; + + altaf_intr_data->gint_iova_addr = iova; + } + return 0; + +fail: + dma_unmap_resource(rvu->dev, phy_addr, max_msix * PCI_MSIX_ENTRY_SIZE, + DMA_BIDIRECTIONAL, 0); + return -EFAULT; } static void rvu_reset_msix(struct rvu *rvu) @@ -1396,7 +1421,6 @@ static void rvu_detach_block(struct rvu *rvu, int pcifunc, int blktype) if (blkaddr < 0) return; - block = &hw->block[blkaddr]; num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr); @@ -1467,6 +1491,13 @@ static int rvu_detach_rsrcs(struct rvu *rvu, struct rsrc_detach *detach, else if ((blkid == BLKADDR_CPT1) && !detach->cptlfs) continue; } + + if (detach_all || + (detach && (blkid == BLKADDR_NIX0 || + blkid == BLKADDR_NIX1) && + detach->nixlf)) + npc_cn20k_dft_rules_free(rvu, pcifunc); + rvu_detach_block(rvu, pcifunc, block->type); } @@ -1750,6 +1781,12 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, err = rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1, attach); if (err) goto fail2; + + if (is_cn20k(rvu->pdev)) { + err = npc_cn20k_dft_rules_alloc(rvu, pcifunc); + if (err) + goto fail3; + } } if (attach->sso) { @@ -1763,7 +1800,7 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, err = rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO, attach->sso, attach); if (err) - goto fail3; + goto fail4; } if (attach->ssow) { @@ -1772,7 +1809,7 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, err = rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW, attach->ssow, attach); if (err) - goto fail4; + goto fail5; } if (attach->timlfs) { @@ -1781,7 +1818,7 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, err = rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM, attach->timlfs, attach); if (err) - goto fail5; + goto fail6; } if (attach->cptlfs) { @@ -1791,24 +1828,28 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, err = rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT, attach->cptlfs, attach); if (err) - goto fail6; + goto fail7; } mutex_unlock(&rvu->rsrc_lock); return 0; -fail6: +fail7: if (attach->timlfs) rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM); -fail5: +fail6: if (attach->ssow) rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW); -fail4: +fail5: if (attach->sso) rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO); +fail4: + if (is_cn20k(rvu->pdev)) + npc_cn20k_dft_rules_free(rvu, pcifunc); + fail3: if (attach->nixlf) rvu_detach_block(rvu, pcifunc, BLKTYPE_NIX); @@ -2185,6 +2226,33 @@ int rvu_mbox_handler_ndc_sync_op(struct rvu *rvu, return 0; } +static void rvu_notify_altaf(struct rvu *rvu, u16 pcifunc, u64 op) +{ + int pf, vf; + + if (!rvu->fwdata) + return; + + if (op == ALTAF_FLR) { + pf = rvu_get_pf(rvu->pdev, pcifunc); + set_bit(pf, rvu->fwdata->altaf_intr_info.flr_pf_bmap); + if (pcifunc & RVU_PFVF_FUNC_MASK) { + vf = pcifunc & RVU_PFVF_FUNC_MASK; + if (vf >= 128) { + WARN(1, + "flr_vf_bmap size is 128 bits, vf=%u\n", + vf); + return; + } + + set_bit(vf, rvu->fwdata->altaf_intr_info.flr_vf_bmap); + } + } + + rvu_write64(rvu, BLKADDR_NIX0, AF_BAR2_ALIASX(0, NIX_GINT_INT_W1S), op); + usleep_range(5000, 6000); +} + static int rvu_process_mbox_msg(struct otx2_mbox *mbox, int devid, struct mbox_msghdr *req) { @@ -2268,7 +2336,8 @@ static void __rvu_mbox_handler(struct rvu_work *mwork, int type, bool poll) offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN); - if (req_hdr->sig && !(is_rvu_otx2(rvu) || is_cn20k(rvu->pdev))) { + if (req_hdr->sig && rvu->altaf_ready && + !(is_rvu_otx2(rvu) || is_cn20k(rvu->pdev))) { req_hdr->opt_msg = mw->mbox_wrk[devid].num_msgs; rvu_write64(rvu, BLKADDR_NIX0, RVU_AF_BAR2_SEL, RVU_AF_BAR2_PFID); @@ -2777,6 +2846,16 @@ static void rvu_blklf_teardown(struct rvu *rvu, u16 pcifunc, u8 blkaddr) block = &rvu->hw->block[blkaddr]; num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc), block->addr); + + if (block->addr == BLKADDR_TIM && rvu->altaf_ready) { + rvu_notify_altaf(rvu, pcifunc, ALTAF_FLR); + return; + } + + if ((block->addr == BLKADDR_SSO || block->addr == BLKADDR_SSOW) && + rvu->altaf_ready) + return; + if (!num_lfs) return; for (slot = 0; slot < num_lfs; slot++) { @@ -3060,12 +3139,12 @@ static int rvu_afvf_msix_vectors_num_ok(struct rvu *rvu) static int rvu_register_interrupts(struct rvu *rvu) { - int ret, offset, pf_vec_start; + int i, ret, offset, pf_vec_start; rvu->num_vec = pci_msix_vec_count(rvu->pdev); - rvu->irq_name = devm_kmalloc_array(rvu->dev, rvu->num_vec, - NAME_SIZE, GFP_KERNEL); + rvu->irq_name = devm_kcalloc(rvu->dev, rvu->num_vec, + NAME_SIZE, GFP_KERNEL); if (!rvu->irq_name) return -ENOMEM; @@ -3251,6 +3330,13 @@ static int rvu_register_interrupts(struct rvu *rvu) if (ret) goto fail; + for (i = 0; i < rvu->num_vec; i++) { + if (strstr(&rvu->irq_name[i * NAME_SIZE], "Mbox") || + strstr(&rvu->irq_name[i * NAME_SIZE], "FLR")) + irq_set_affinity(pci_irq_vector(rvu->pdev, i), + cpumask_of(0)); + } + return 0; fail: @@ -3279,8 +3365,8 @@ static int rvu_flr_init(struct rvu *rvu) cfg | BIT_ULL(22)); } - rvu->flr_wq = alloc_ordered_workqueue("rvu_afpf_flr", - WQ_HIGHPRI | WQ_MEM_RECLAIM); + rvu->flr_wq = alloc_workqueue("rvu_afpf_flr", + WQ_HIGHPRI | WQ_MEM_RECLAIM, 0); if (!rvu->flr_wq) return -ENOMEM; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index e85dac2c806d..a466181cf908 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -197,7 +197,7 @@ struct npc_key_field { /* Masks where all set bits indicate position * of a field in the key */ - u64 kw_mask[NPC_MAX_KWS_IN_KEY]; + u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX]; /* Number of words in the key a field spans. If a field is * of 16 bytes and key offset is 4 then the field will use * 4 bytes in KW0, 8 bytes in KW1 and 4 bytes in KW2 and @@ -308,6 +308,7 @@ struct rvu_pfvf { u64 lmt_map_ent_w1; /* Preseving the word1 of lmtst map table entry*/ unsigned long flags; struct sdp_node_info *sdp_info; + u8 hw_prio; /* Hw priority of default rules */ }; enum rvu_pfvf_flags { @@ -447,9 +448,11 @@ struct rvu_hwinfo { u8 sdp_links; u8 cpt_links; /* Number of CPT links */ u8 npc_kpus; /* No of parser units */ + u8 npc_kpms; /* Number of enhanced parser units */ + u8 npc_kex_extr; /* Number of LDATA extractors per KEX */ u8 npc_pkinds; /* No of port kinds */ u8 npc_intfs; /* No of interfaces */ - u8 npc_kpu_entries; /* No of KPU entries */ + u16 npc_kpu_entries; /* No of KPU entries */ u16 npc_counters; /* No of match stats counters */ u32 lbk_bufsize; /* FIFO size supported by LBK */ bool npc_ext_set; /* Extended register set */ @@ -552,7 +555,11 @@ struct npc_kpu_profile_adapter { const struct npc_lt_def_cfg *lt_def; const struct npc_kpu_profile_action *ikpu; /* array[pkinds] */ const struct npc_kpu_profile *kpu; /* array[kpus] */ - struct npc_mcam_kex *mkex; + union npc_mcam_key_prfl { + struct npc_mcam_kex *mkex; + /* used for cn9k and cn10k */ + struct npc_mcam_kex_extr *mkex_extr; /* used for cn20k */ + } mcam_kex_prfl; struct npc_mcam_kex_hash *mkex_hash; bool custom; size_t pkinds; @@ -1117,8 +1124,8 @@ int rvu_cgx_cfg_pause_frm(struct rvu *rvu, u16 pcifunc, u8 tx_pause, u8 rx_pause void rvu_mac_reset(struct rvu *rvu, u16 pcifunc); u32 rvu_cgx_get_lmac_fifolen(struct rvu *rvu, int cgx, int lmac); void cgx_start_linkup(struct rvu *rvu); -int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, u16 pcifunc, int nixlf, - int type); +int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, + u16 pcifunc, int nixlf, int type); bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, int index); int rvu_npc_init(struct rvu *rvu); @@ -1184,4 +1191,5 @@ int rvu_rep_pf_init(struct rvu *rvu); int rvu_rep_install_mcam_rules(struct rvu *rvu); void rvu_rep_update_rules(struct rvu *rvu, u16 pcifunc, bool ena); int rvu_rep_notify_pfvf_state(struct rvu *rvu, u16 pcifunc, bool enable); +int npc_mcam_verify_entry(struct npc_mcam *mcam, u16 pcifunc, int entry); #endif /* RVU_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index 15d3cb0b9da6..413f9fa40b33 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -21,7 +21,9 @@ #include "rvu_npc_hash.h" #include "mcs.h" +#include "cn20k/reg.h" #include "cn20k/debugfs.h" +#include "cn20k/npc.h" #define DEBUGFS_DIR_NAME "octeontx2" @@ -3196,7 +3198,9 @@ static void rvu_print_npc_mcam_info(struct seq_file *s, static int rvu_dbg_npc_mcam_info_display(struct seq_file *filp, void *unsued) { struct rvu *rvu = filp->private; + int x4_free, x2_free, sb_free; int pf, vf, numvfs, blkaddr; + struct npc_priv_t *npc_priv; struct npc_mcam *mcam; u16 pcifunc, counters; u64 cfg; @@ -3210,16 +3214,34 @@ static int rvu_dbg_npc_mcam_info_display(struct seq_file *filp, void *unsued) seq_puts(filp, "\nNPC MCAM info:\n"); /* MCAM keywidth on receive and transmit sides */ - cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX)); - cfg = (cfg >> 32) & 0x07; - seq_printf(filp, "\t\t RX keywidth \t: %s\n", (cfg == NPC_MCAM_KEY_X1) ? - "112bits" : ((cfg == NPC_MCAM_KEY_X2) ? - "224bits" : "448bits")); - cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX)); - cfg = (cfg >> 32) & 0x07; - seq_printf(filp, "\t\t TX keywidth \t: %s\n", (cfg == NPC_MCAM_KEY_X1) ? - "112bits" : ((cfg == NPC_MCAM_KEY_X2) ? - "224bits" : "448bits")); + if (is_cn20k(rvu->pdev)) { + npc_priv = npc_priv_get(); + seq_printf(filp, "\t\t RX keywidth \t: %s\n", + (npc_priv->kw == NPC_MCAM_KEY_X1) ? + "256bits" : "512bits"); + + npc_cn20k_subbank_calc_free(rvu, &x2_free, &x4_free, &sb_free); + seq_printf(filp, "\t\t free x4 slots\t: %d\n", x4_free); + + seq_printf(filp, "\t\t free x2 slots\t: %d\n", x2_free); + + seq_printf(filp, "\t\t free subbanks\t: %d\n", sb_free); + } else { + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX)); + cfg = (cfg >> 32) & 0x07; + seq_printf(filp, "\t\t RX keywidth \t: %s\n", + (cfg == NPC_MCAM_KEY_X1) ? + "112bits" : ((cfg == NPC_MCAM_KEY_X2) ? + "224bits" : "448bits")); + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX)); + cfg = (cfg >> 32) & 0x07; + seq_printf(filp, "\t\t TX keywidth \t: %s\n", + (cfg == NPC_MCAM_KEY_X1) ? + "112bits" : ((cfg == NPC_MCAM_KEY_X2) ? + "224bits" : "448bits")); + } mutex_lock(&mcam->lock); /* MCAM entries */ @@ -3506,11 +3528,11 @@ static int rvu_dbg_npc_mcam_show_rules(struct seq_file *s, void *unused) struct rvu_npc_mcam_rule *iter; struct rvu *rvu = s->private; struct npc_mcam *mcam; - int pf, vf = -1; + int pf, vf = -1, bank; + u16 target, index; bool enabled; + u64 hits, off; int blkaddr; - u16 target; - u64 hits; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) @@ -3554,6 +3576,15 @@ static int rvu_dbg_npc_mcam_show_rules(struct seq_file *s, void *unused) enabled = is_mcam_entry_enabled(rvu, mcam, blkaddr, iter->entry); seq_printf(s, "\tenabled: %s\n", enabled ? "yes" : "no"); + if (is_cn20k(rvu->pdev)) { + seq_printf(s, "\tpriority: %u\n", iter->hw_prio); + index = iter->entry & (mcam->banksize - 1); + bank = npc_get_bank(mcam, iter->entry); + off = NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank); + hits = rvu_read64(rvu, blkaddr, off); + seq_printf(s, "\thits: %lld\n", hits); + continue; + } if (!iter->has_cntr) continue; @@ -3698,9 +3729,9 @@ static int rvu_dbg_npc_exact_drop_cnt(struct seq_file *s, void *unused) struct npc_exact_table *table; struct rvu *rvu = s->private; struct npc_key_field *field; + u64 cfg, cam1, off; u16 chan, pcifunc; int blkaddr, i; - u64 cfg, cam1; char *str; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); @@ -3721,11 +3752,17 @@ static int rvu_dbg_npc_exact_drop_cnt(struct seq_file *s, void *unused) chan = field->kw_mask[0] & cam1; str = (cfg & 1) ? "enabled" : "disabled"; + if (is_cn20k(rvu->pdev)) { + off = NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(i, 0); + seq_printf(s, "0x%x\t%d\t\t%llu\t0x%x\t%s\n", pcifunc, + i, rvu_read64(rvu, blkaddr, off), chan, str); + } else { + off = NPC_AF_MATCH_STATX(table->counter_idx[i]); + seq_printf(s, "0x%x\t%d\t\t%llu\t0x%x\t%s\n", pcifunc, + i, rvu_read64(rvu, blkaddr, off), + chan, str); + } - seq_printf(s, "0x%x\t%d\t\t%llu\t0x%x\t%s\n", pcifunc, i, - rvu_read64(rvu, blkaddr, - NPC_AF_MATCH_STATX(table->counter_idx[i])), - chan, str); } return 0; @@ -3745,6 +3782,9 @@ static void rvu_dbg_npc_init(struct rvu *rvu) debugfs_create_file("rx_miss_act_stats", 0444, rvu->rvu_dbg.npc, rvu, &rvu_dbg_npc_rx_miss_act_fops); + if (is_cn20k(rvu->pdev)) + npc_cn20k_debugfs_init(rvu); + if (!rvu->hw->cap.npc_exact_match_enabled) return; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c index fb15c794efc9..287ff0eda152 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c @@ -11,6 +11,7 @@ #include "rvu_reg.h" #include "rvu_struct.h" #include "rvu_npc_hash.h" +#include "cn20k/npc.h" #define DRV_NAME "octeontx2-af" @@ -1256,9 +1257,71 @@ enum rvu_af_dl_param_id { RVU_AF_DEVLINK_PARAM_ID_NPC_MCAM_ZONE_PERCENT, RVU_AF_DEVLINK_PARAM_ID_NPC_EXACT_FEATURE_DISABLE, RVU_AF_DEVLINK_PARAM_ID_NPC_DEF_RULE_CNTR_ENABLE, + RVU_AF_DEVLINK_PARAM_ID_NPC_DEFRAG, RVU_AF_DEVLINK_PARAM_ID_NIX_MAXLF, }; +static int rvu_af_npc_defrag_feature_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + bool enabled; + + enabled = is_cn20k(rvu->pdev); + + snprintf(ctx->val.vstr, sizeof(ctx->val.vstr), "%s", + enabled ? "enabled" : "disabled"); + + return 0; +} + +static int rvu_af_npc_defrag(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + + /* It is hard to roll back if defrag process fails. + * print a error message and return fault. + */ + if (npc_cn20k_defrag(rvu)) { + dev_err(rvu->dev, "Defrag process failed\n"); + return -EFAULT; + } + return 0; +} + +static int rvu_af_npc_defrag_feature_validate(struct devlink *devlink, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + u64 enable; + + if (kstrtoull(val.vstr, 10, &enable)) { + NL_SET_ERR_MSG_MOD(extack, + "Only 1 value is supported"); + return -EINVAL; + } + + if (enable != 1) { + NL_SET_ERR_MSG_MOD(extack, + "Only initiating defrag is supported"); + return -EINVAL; + } + + if (is_cn20k(rvu->pdev)) + return 0; + + NL_SET_ERR_MSG_MOD(extack, + "Can defrag NPC only in cn20k silicon"); + return -EFAULT; +} + static int rvu_af_npc_exact_feature_get(struct devlink *devlink, u32 id, struct devlink_param_gset_ctx *ctx, struct netlink_ext_ack *extack) @@ -1561,6 +1624,15 @@ static const struct devlink_ops rvu_devlink_ops = { .eswitch_mode_set = rvu_devlink_eswitch_mode_set, }; +static const struct devlink_param rvu_af_dl_param_defrag[] = { + DEVLINK_PARAM_DRIVER(RVU_AF_DEVLINK_PARAM_ID_NPC_DEFRAG, + "npc_defrag", DEVLINK_PARAM_TYPE_STRING, + BIT(DEVLINK_PARAM_CMODE_RUNTIME), + rvu_af_npc_defrag_feature_get, + rvu_af_npc_defrag, + rvu_af_npc_defrag_feature_validate), +}; + int rvu_register_dl(struct rvu *rvu) { struct rvu_devlink *rvu_dl; @@ -1593,6 +1665,17 @@ int rvu_register_dl(struct rvu *rvu) goto err_dl_health; } + if (is_cn20k(rvu->pdev)) { + err = devlink_params_register(dl, rvu_af_dl_param_defrag, + ARRAY_SIZE(rvu_af_dl_param_defrag)); + if (err) { + dev_err(rvu->dev, + "devlink defrag params register failed with error %d", + err); + goto err_dl_defrag; + } + } + /* Register exact match devlink only for CN10K-B */ if (!rvu_npc_exact_has_match_table(rvu)) goto done; @@ -1601,7 +1684,8 @@ int rvu_register_dl(struct rvu *rvu) ARRAY_SIZE(rvu_af_dl_param_exact_match)); if (err) { dev_err(rvu->dev, - "devlink exact match params register failed with error %d", err); + "devlink exact match params register failed with error %d", + err); goto err_dl_exact_match; } @@ -1610,6 +1694,11 @@ int rvu_register_dl(struct rvu *rvu) return 0; err_dl_exact_match: + if (is_cn20k(rvu->pdev)) + devlink_params_unregister(dl, rvu_af_dl_param_defrag, + ARRAY_SIZE(rvu_af_dl_param_defrag)); + +err_dl_defrag: devlink_params_unregister(dl, rvu_af_dl_params, ARRAY_SIZE(rvu_af_dl_params)); err_dl_health: @@ -1627,6 +1716,10 @@ void rvu_unregister_dl(struct rvu *rvu) devlink_params_unregister(dl, rvu_af_dl_params, ARRAY_SIZE(rvu_af_dl_params)); + if (is_cn20k(rvu->pdev)) + devlink_params_unregister(dl, rvu_af_dl_param_defrag, + ARRAY_SIZE(rvu_af_dl_param_defrag)); + /* Unregister exact match devlink only for CN10K-B */ if (rvu_npc_exact_has_match_table(rvu)) devlink_params_unregister(dl, rvu_af_dl_param_exact_match, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 98dd68137a09..badfa1d64252 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -5289,7 +5289,6 @@ int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req, /* Disable the interface if it is in any multicast list */ nix_mcast_update_mce_entry(rvu, pcifunc, 0); - pfvf = rvu_get_pfvf(rvu, pcifunc); clear_bit(NIXLF_INITIALIZED, &pfvf->flags); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index 8658cb2143df..c2ca5ed1d028 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -16,6 +16,9 @@ #include "cgx.h" #include "npc_profile.h" #include "rvu_npc_hash.h" +#include "cn20k/npc.h" +#include "rvu_npc.h" +#include "cn20k/reg.h" #define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ @@ -149,10 +152,33 @@ int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, { struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); struct rvu *rvu = hw->rvu; - int pf = rvu_get_pf(rvu->pdev, pcifunc); + u16 bcast, mcast, promisc, ucast; int index; + int rc; + int pf; + + if (is_cn20k(rvu->pdev)) { + rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, + &promisc, &ucast); + if (rc) + return -EFAULT; + + switch (type) { + case NIXLF_BCAST_ENTRY: + return bcast; + case NIXLF_ALLMULTI_ENTRY: + return mcast; + case NIXLF_PROMISC_ENTRY: + return promisc; + case NIXLF_UCAST_ENTRY: + return ucast; + default: + return -EINVAL; + } + } /* Check if this is for a PF */ + pf = rvu_get_pf(rvu->pdev, pcifunc); if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { /* Reserved entries exclude PF0 */ pf--; @@ -173,7 +199,12 @@ int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, int npc_get_bank(struct npc_mcam *mcam, int index) { + struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); int bank = index / mcam->banksize; + struct rvu *rvu = hw->rvu; + + if (is_cn20k(rvu->pdev)) + return bank; /* 0,1 & 2,3 banks are combined for this keysize */ if (mcam->keysize == NPC_MCAM_KEY_X2) @@ -189,7 +220,14 @@ bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, u64 cfg; index &= (mcam->banksize - 1); - cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); + if (is_cn20k(rvu->pdev)) + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, + bank)); + else + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_MCAMEX_BANKX_CFG(index, bank)); + return (cfg & 1); } @@ -199,6 +237,13 @@ void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, int bank = npc_get_bank(mcam, index); int actbank = bank; + if (is_cn20k(rvu->pdev)) { + if (index < 0 || index >= mcam->banksize * mcam->banks) + return; + + return npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable); + } + index &= (mcam->banksize - 1); for (; bank < (actbank + mcam->banks_per_entry); bank++) { rvu_write64(rvu, blkaddr, @@ -367,6 +412,7 @@ static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, u16 pf_func) { int bank, nixlf, index; + u64 reg; /* get ucast entry rule entry index */ if (nix_get_nixlf(rvu, pf_func, &nixlf, NULL)) { @@ -381,8 +427,12 @@ static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, bank = npc_get_bank(mcam, index); index &= (mcam->banksize - 1); - return rvu_read64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); + else + reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); + + return rvu_read64(rvu, blkaddr, reg); } static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, @@ -547,6 +597,9 @@ static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, u64 cfg, sreg, dreg; int bank, i; + if (is_cn20k(rvu->pdev)) + return npc_cn20k_copy_mcam_entry(rvu, blkaddr, src, dest); + src &= (mcam->banksize - 1); dest &= (mcam->banksize - 1); @@ -583,20 +636,31 @@ u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, int index) { int bank = npc_get_bank(mcam, index); + u64 reg; index &= (mcam->banksize - 1); - return rvu_read64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); + + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); + else + reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); + return rvu_read64(rvu, blkaddr, reg); } void npc_set_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, int index, u64 cfg) { int bank = npc_get_bank(mcam, index); + u64 reg; index &= (mcam->banksize - 1); - return rvu_write64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index, bank), cfg); + + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); + else + reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); + + return rvu_write64(rvu, blkaddr, reg, cfg); } void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, @@ -651,6 +715,9 @@ void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, req.match_id = action.match_id; req.flow_key_alg = action.flow_key_alg; + if (is_cn20k(rvu->pdev)) + req.hw_prio = pfvf->hw_prio; + rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); } @@ -739,6 +806,9 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, req.match_id = action.match_id; req.flow_key_alg = flow_key_alg; + if (is_cn20k(rvu->pdev)) + req.hw_prio = pfvf->hw_prio; + rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); } @@ -819,6 +889,9 @@ void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, req.hdr.pcifunc = 0; /* AF is requester */ req.vf = pcifunc; + if (is_cn20k(rvu->pdev)) + req.hw_prio = pfvf->hw_prio; + rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); } @@ -907,6 +980,9 @@ void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, req.match_id = action.match_id; req.flow_key_alg = flow_key_alg; + if (is_cn20k(rvu->pdev)) + req.hw_prio = pfvf->hw_prio; + rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); } @@ -940,33 +1016,42 @@ static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, mutex_lock(&mcam->lock); for (index = 0; index < mcam->bmap_entries; index++) { - if (mcam->entry2target_pffunc[index] == pcifunc) { - update = true; - /* update not needed for the rules added via ntuple filters */ - list_for_each_entry(rule, &mcam->mcam_rules, list) { - if (rule->entry == index) - update = false; - } - if (!update) - continue; - bank = npc_get_bank(mcam, index); - actindex = index; - entry = index & (mcam->banksize - 1); + if (mcam->entry2target_pffunc[index] != pcifunc) + continue; + update = true; + /* update not needed for the rules added via ntuple filters */ + list_for_each_entry(rule, &mcam->mcam_rules, list) { + if (rule->entry == index) + update = false; + } + if (!update) + continue; + bank = npc_get_bank(mcam, index); + actindex = index; + entry = index & (mcam->banksize - 1); - /* read vf flow entry enable status */ - enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, - actindex); - /* disable before mcam entry update */ - npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, - false); - /* update 'action' */ + /* read vf flow entry enable status */ + enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, + actindex); + /* disable before mcam entry update */ + npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, + false); + + /* update 'action' */ + if (is_cn20k(rvu->pdev)) + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(entry, + bank, + 0), + rx_action); + else rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_ACTION(entry, bank), rx_action); - if (enable) - npc_enable_mcam_entry(rvu, mcam, blkaddr, - actindex, true); - } + + if (enable) + npc_enable_mcam_entry(rvu, mcam, blkaddr, + actindex, true); } mutex_unlock(&mcam->lock); } @@ -979,6 +1064,7 @@ static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_act struct npc_mcam *mcam = &rvu->hw->mcam; struct rvu_hwinfo *hw = rvu->hw; int bank, op_rss; + u64 reg; if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_index)) return; @@ -988,15 +1074,19 @@ static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_act bank = npc_get_bank(mcam, mcam_index); mcam_index &= (mcam->banksize - 1); + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_index, + bank, 0); + else + reg = NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank); + /* If Rx action is MCAST update only RSS algorithm index */ if (!op_rss) { - *(u64 *)&action = rvu_read64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank)); + *(u64 *)&action = rvu_read64(rvu, blkaddr, reg); action.flow_key_alg = alg_idx; } - rvu_write64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank), *(u64 *)&action); + rvu_write64(rvu, blkaddr, reg, *(u64 *)&action); } void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, @@ -1006,6 +1096,7 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, struct nix_rx_action action; int blkaddr, index, bank; struct rvu_pfvf *pfvf; + u64 reg; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) @@ -1028,8 +1119,12 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, bank = npc_get_bank(mcam, index); index &= (mcam->banksize - 1); - *(u64 *)&action = rvu_read64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); + else + reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); + + *(u64 *)&action = rvu_read64(rvu, blkaddr, reg); /* Ignore if no action was set earlier */ if (!*(u64 *)&action) return; @@ -1039,8 +1134,11 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, action.index = group; action.flow_key_alg = alg_idx; - rvu_write64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); + rvu_write64(rvu, blkaddr, reg, *(u64 *)&action); + /* update the VF flow rule action with the VF default entry action */ + if (mcam_index < 0) + npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, + *(u64 *)&action); /* update the action change in default rule */ pfvf = rvu_get_pfvf(rvu, pcifunc); @@ -1338,8 +1436,8 @@ static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, npc_program_mkex_hash(rvu, blkaddr); } -static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, - u64 *size) +int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, + u64 *size) { u64 prfl_addr, prfl_sz; @@ -1395,7 +1493,7 @@ static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, */ if (!is_rvu_96xx_B0(rvu) || mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) - rvu->kpu.mkex = mcam_kex; + rvu->kpu.mcam_kex_prfl.mkex = mcam_kex; goto program_mkex; } @@ -1405,16 +1503,17 @@ static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); program_mkex: - dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name); + dev_info(rvu->dev, "Using %s mkex profile\n", + rvu->kpu.mcam_kex_prfl.mkex->name); /* Program selected mkex profile */ - npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); + npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mcam_kex_prfl.mkex); if (mkex_prfl_addr) iounmap(mkex_prfl_addr); } -static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, - const struct npc_kpu_profile_action *kpuaction, - int kpu, int entry, bool pkind) +void npc_config_kpuaction(struct rvu *rvu, int blkaddr, + const struct npc_kpu_profile_action *kpuaction, + int kpu, int entry, bool pkind) { struct npc_kpu_action0 action0 = {0}; struct npc_kpu_action1 action1 = {0}; @@ -1477,7 +1576,7 @@ static void npc_config_kpucam(struct rvu *rvu, int blkaddr, NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); } -static inline u64 enable_mask(int count) +u64 npc_enable_mask(int count) { return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); } @@ -1510,7 +1609,7 @@ static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, /* Enable all programmed entries */ num_entries = min_t(int, profile->action_entries, profile->cam_entries); - entry_mask = enable_mask(num_entries); + entry_mask = npc_enable_mask(num_entries); /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ if (!rvu->kpu.custom) entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); @@ -1519,14 +1618,15 @@ static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, if (num_entries > 64) { rvu_write64(rvu, blkaddr, NPC_AF_KPUX_ENTRY_DISX(kpu, 1), - enable_mask(num_entries - 64)); + npc_enable_mask(num_entries - 64)); } /* Enable this KPU */ rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); } -static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) +static void npc_prepare_default_kpu(struct rvu *rvu, + struct npc_kpu_profile_adapter *profile) { profile->custom = 0; profile->name = def_pfl_name; @@ -1536,23 +1636,38 @@ static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) profile->kpu = npc_kpu_profiles; profile->kpus = ARRAY_SIZE(npc_kpu_profiles); profile->lt_def = &npc_lt_defaults; - profile->mkex = &npc_mkex_default; profile->mkex_hash = &npc_mkex_hash_default; - return 0; + if (!is_cn20k(rvu->pdev)) { + profile->mcam_kex_prfl.mkex = &npc_mkex_default; + return; + } + + profile->mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get(); + ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].offset = 6; + ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].mask = 0xe0; + ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].shift = 0x5; + ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].right = 0x1; + + npc_cn20k_update_action_entries_n_flags(rvu, profile); } static int npc_apply_custom_kpu(struct rvu *rvu, struct npc_kpu_profile_adapter *profile) { size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; - struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata; struct npc_kpu_profile_action *action; + struct npc_kpu_profile_fwdata *fw; struct npc_kpu_profile_cam *cam; struct npc_kpu_fwdata *fw_kpu; int entries; u16 kpu, entry; + if (is_cn20k(rvu->pdev)) + return npc_cn20k_apply_custom_kpu(rvu, profile); + + fw = rvu->kpu_fwdata; + if (rvu->kpu_fwdata_sz < hdr_sz) { dev_warn(rvu->dev, "Invalid KPU profile size\n"); return -EINVAL; @@ -1593,7 +1708,7 @@ static int npc_apply_custom_kpu(struct rvu *rvu, profile->custom = 1; profile->name = fw->name; profile->version = le64_to_cpu(fw->version); - profile->mkex = &fw->mkex; + profile->mcam_kex_prfl.mkex = &fw->mkex; profile->lt_def = &fw->lt_def; for (kpu = 0; kpu < fw->kpus; kpu++) { @@ -1707,7 +1822,7 @@ static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile) return ret; } -static void npc_load_kpu_profile(struct rvu *rvu) +void npc_load_kpu_profile(struct rvu *rvu) { struct npc_kpu_profile_adapter *profile = &rvu->kpu; const char *kpu_profile = rvu->kpu_pfl_name; @@ -1718,7 +1833,7 @@ static void npc_load_kpu_profile(struct rvu *rvu) if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) goto revert_to_default; /* First prepare default KPU, then we'll customize top entries. */ - npc_prepare_default_kpu(profile); + npc_prepare_default_kpu(rvu, profile); /* Order of preceedence for load loading NPC profile (high to low) * Firmware binary in filesystem. @@ -1781,7 +1896,7 @@ static void npc_load_kpu_profile(struct rvu *rvu) return; revert_to_default: - npc_prepare_default_kpu(profile); + npc_prepare_default_kpu(rvu, profile); } static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) @@ -1842,39 +1957,56 @@ int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) int cntr; u64 cfg; - /* Actual number of MCAM entries vary by entry size */ cfg = (rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; - mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; mcam->keysize = cfg; /* Number of banks combined per MCAM entry */ - if (cfg == NPC_MCAM_KEY_X4) - mcam->banks_per_entry = 4; - else if (cfg == NPC_MCAM_KEY_X2) - mcam->banks_per_entry = 2; - else - mcam->banks_per_entry = 1; + if (is_cn20k(rvu->pdev)) { + /* In cn20k, x2 entries is allowed for x4 profile. + * set total_entries as 8192 * 2 and key size as x2. + */ + mcam->total_entries = mcam->banks * mcam->banksize; + if (cfg == NPC_MCAM_KEY_X2) + mcam->banks_per_entry = 1; + else + mcam->banks_per_entry = 2; - /* Reserve one MCAM entry for each of the NIX LF to - * guarantee space to install default matching DMAC rule. - * Also reserve 2 MCAM entries for each PF for default - * channel based matching or 'bcast & promisc' matching to - * support BCAST and PROMISC modes of operation for PFs. - * PF0 is excluded. - */ - rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + - ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); - if (mcam->total_entries <= rsvd) { - dev_warn(rvu->dev, - "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", - mcam->total_entries); - return -ENOMEM; + rsvd = 0; + } else { + mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * + mcam->banksize; + + if (cfg == NPC_MCAM_KEY_X4) + mcam->banks_per_entry = 4; + else if (cfg == NPC_MCAM_KEY_X2) + mcam->banks_per_entry = 2; + else + mcam->banks_per_entry = 1; + + /* Reserve one MCAM entry for each of the NIX LF to + * guarantee space to install default matching DMAC rule. + * Also reserve 2 MCAM entries for each PF for default + * channel based matching or 'bcast & promisc' matching to + * support BCAST and PROMISC modes of operation for PFs. + * PF0 is excluded. + */ + rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + + ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); + if (mcam->total_entries <= rsvd) { + dev_warn(rvu->dev, + "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", + mcam->total_entries); + return -ENOMEM; + } } mcam->bmap_entries = mcam->total_entries - rsvd; - mcam->nixlf_offset = mcam->bmap_entries; - mcam->pf_offset = mcam->nixlf_offset + nixlf_count; + /* cn20k does not need offsets to alloc mcam entries */ + if (!is_cn20k(rvu->pdev)) { + mcam->nixlf_offset = mcam->bmap_entries; + mcam->pf_offset = mcam->nixlf_offset + nixlf_count; + } /* Allocate bitmaps for managing MCAM entries */ mcam->bmap = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL); @@ -1898,13 +2030,15 @@ int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) * allocations and another 1/8th at the top for high priority * allocations. */ - mcam->lprio_count = mcam->bmap_entries / 8; - if (mcam->lprio_count > BITS_PER_LONG) - mcam->lprio_count = round_down(mcam->lprio_count, - BITS_PER_LONG); - mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; - mcam->hprio_count = mcam->lprio_count; - mcam->hprio_end = mcam->hprio_count; + if (!is_cn20k(rvu->pdev)) { + mcam->lprio_count = mcam->bmap_entries / 8; + if (mcam->lprio_count > BITS_PER_LONG) + mcam->lprio_count = round_down(mcam->lprio_count, + BITS_PER_LONG); + mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; + mcam->hprio_count = mcam->lprio_count; + mcam->hprio_end = mcam->hprio_count; + } /* Allocate bitmap for managing MCAM counters and memory * for saving counter to RVU PFFUNC allocation mapping. @@ -1984,6 +2118,19 @@ static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL; hw->npc_kpu_entries = npc_const1 & 0xFFFULL; hw->npc_kpus = (npc_const >> 8) & 0x1FULL; + /* For Cn20k silicon, check if enhanced parser + * is present, then set the NUM_KPMS = NUM_KPUS / 2 and + * number of LDATA extractors per KEX. + */ + if (is_cn20k(rvu->pdev)) { + if (!(npc_const1 & BIT_ULL(62))) { + WARN(1, "Enhanced parser is not supported\n"); + return; + } + hw->npc_kpms = hw->npc_kpus / 2; + hw->npc_kex_extr = (npc_const1 >> 36) & 0x3FULL; + } + hw->npc_intfs = npc_const & 0xFULL; hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; @@ -2014,28 +2161,36 @@ static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) { - struct npc_mcam_kex *mkex = rvu->kpu.mkex; + struct npc_mcam_kex_extr *mkex_extr = rvu->kpu.mcam_kex_prfl.mkex_extr; + struct npc_mcam_kex *mkex = rvu->kpu.mcam_kex_prfl.mkex; struct npc_mcam *mcam = &rvu->hw->mcam; struct rvu_hwinfo *hw = rvu->hw; u64 nibble_ena, rx_kex, tx_kex; + u64 *keyx_cfg, reg; u8 intf; - /* Reserve last counter for MCAM RX miss action which is set to - * drop packet. This way we will know how many pkts didn't match - * any MCAM entry. - */ - mcam->counters.max--; - mcam->rx_miss_act_cntr = mcam->counters.max; + if (is_cn20k(rvu->pdev)) { + keyx_cfg = mkex_extr->keyx_cfg; + } else { + keyx_cfg = mkex->keyx_cfg; + /* Reserve last counter for MCAM RX miss action which is set to + * drop packet. This way we will know how many pkts didn't + * match any MCAM entry. + */ + mcam->counters.max--; + mcam->rx_miss_act_cntr = mcam->counters.max; + } + + rx_kex = keyx_cfg[NIX_INTF_RX]; + tx_kex = keyx_cfg[NIX_INTF_TX]; - rx_kex = mkex->keyx_cfg[NIX_INTF_RX]; - tx_kex = mkex->keyx_cfg[NIX_INTF_TX]; nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); if (nibble_ena) { tx_kex &= ~NPC_PARSE_NIBBLE; tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); - mkex->keyx_cfg[NIX_INTF_TX] = tx_kex; + keyx_cfg[NIX_INTF_TX] = tx_kex; } /* Configure RX interfaces */ @@ -2047,12 +2202,19 @@ static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), rx_kex); + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_INTFX_MISS_ACTX(intf, 0); + else + reg = NPC_AF_INTFX_MISS_ACT(intf); + /* If MCAM lookup doesn't result in a match, drop the received * packet. And map this action to a counter to count dropped * packets. */ - rvu_write64(rvu, blkaddr, - NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP); + rvu_write64(rvu, blkaddr, reg, NIX_RX_ACTIONOP_DROP); + + if (is_cn20k(rvu->pdev)) + continue; /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] @@ -2072,12 +2234,15 @@ static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), tx_kex); + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_INTFX_MISS_ACTX(intf, 0); + else + reg = NPC_AF_INTFX_MISS_ACT(intf); + /* Set TX miss action to UCAST_DEFAULT i.e * transmit the packet on NIX LF SQ's default channel. */ - rvu_write64(rvu, blkaddr, - NPC_AF_INTFX_MISS_ACT(intf), - NIX_TX_ACTIONOP_UCAST_DEFAULT); + rvu_write64(rvu, blkaddr, reg, NIX_TX_ACTIONOP_UCAST_DEFAULT); } } @@ -2118,7 +2283,10 @@ int rvu_npc_init(struct rvu *rvu) return -ENOMEM; /* Configure KPU profile */ - npc_parser_profile_init(rvu, blkaddr); + if (is_cn20k(rvu->pdev)) + npc_cn20k_parser_profile_init(rvu, blkaddr); + else + npc_parser_profile_init(rvu, blkaddr); /* Config Outer L2, IPv4's NPC layer info */ rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, @@ -2149,7 +2317,10 @@ int rvu_npc_init(struct rvu *rvu) npc_config_secret_key(rvu, blkaddr); /* Configure MKEX profile */ - npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); + if (is_cn20k(rvu->pdev)) + npc_cn20k_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); + else + npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); err = npc_mcam_rsrcs_init(rvu, blkaddr); if (err) @@ -2159,9 +2330,15 @@ int rvu_npc_init(struct rvu *rvu) if (err) { dev_err(rvu->dev, "Incorrect mkex profile loaded using default mkex\n"); - npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); + if (is_cn20k(rvu->pdev)) + npc_cn20k_load_mkex_profile(rvu, blkaddr, def_pfl_name); + else + npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); } + if (is_cn20k(rvu->pdev)) + return npc_cn20k_init(rvu); + return 0; } @@ -2177,6 +2354,9 @@ void rvu_npc_freemem(struct rvu *rvu) else kfree(rvu->kpu_fwdata); mutex_destroy(&mcam->lock); + + if (is_cn20k(rvu->pdev)) + npc_cn20k_deinit(rvu); } void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, @@ -2217,8 +2397,8 @@ void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, } } -static int npc_mcam_verify_entry(struct npc_mcam *mcam, - u16 pcifunc, int entry) +int npc_mcam_verify_entry(struct npc_mcam *mcam, + u16 pcifunc, int entry) { /* verify AF installed entries */ if (is_pffunc_af(pcifunc)) @@ -2260,6 +2440,10 @@ static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, /* Set mapping and increment counter's refcnt */ mcam->entry2cntr_map[entry] = cntr; mcam->cntr_refcnt[cntr]++; + + if (is_cn20k(rvu->pdev)) + return; + /* Enable stats */ rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), @@ -2276,6 +2460,10 @@ static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, /* Remove mapping and reduce counter's refcnt */ mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; mcam->cntr_refcnt[cntr]--; + + if (is_cn20k(rvu->pdev)) + return; + /* Disable stats */ rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); @@ -2285,7 +2473,7 @@ static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, * reverse bitmap too. Should be called with * 'mcam->lock' held. */ -static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) +void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) { u16 entry, rentry; @@ -2301,7 +2489,7 @@ static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) * reverse bitmap too. Should be called with * 'mcam->lock' held. */ -static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) +void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) { u16 entry, rentry; @@ -2317,6 +2505,7 @@ static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, u16 pcifunc) { u16 index, cntr; + int rc; /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ for (index = 0; index < mcam->bmap_entries; index++) { @@ -2334,6 +2523,13 @@ static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, blkaddr, index, cntr); mcam->entry2target_pffunc[index] = 0x0; + if (is_cn20k(rvu->pdev)) { + rc = npc_cn20k_idx_free(rvu, &index, 1); + if (rc) + dev_err(rvu->dev, + "Failed to free mcam idx=%u pcifunc=%#x\n", + index, pcifunc); + } } } } @@ -2423,7 +2619,7 @@ npc_get_mcam_search_range_priority(struct npc_mcam *mcam, { u16 fcnt; - if (req->priority == NPC_MCAM_HIGHER_PRIO) + if (req->ref_prio == NPC_MCAM_HIGHER_PRIO) goto hprio; /* For a low priority entry allocation @@ -2480,14 +2676,80 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, struct npc_mcam_alloc_entry_req *req, struct npc_mcam_alloc_entry_rsp *rsp) { + struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; u16 fcnt, hp_fcnt, lp_fcnt; + struct rvu *rvu = hw->rvu; u16 start, end, index; int entry, next_start; bool reverse = false; unsigned long *bmap; + int ret, limit = 0; u16 max_contig; + if (!is_cn20k(rvu->pdev)) + goto not_cn20k; + + /* Only x2 or x4 key types are accepted */ + if (req->kw_type != NPC_MCAM_KEY_X2 && req->kw_type != NPC_MCAM_KEY_X4) + return NPC_MCAM_INVALID_REQ; + + /* The below table is being followed during allocation, + * + * 1. ref_entry == 0 && prio == HIGH && count == 1 : user wants to + * allocate 0th index + * 2. ref_entry == 0 && prio == HIGH && count > 1 : Invalid request + * 3. ref_entry == 0 && prio == LOW && count >= 1 : limit = 0 + * 4. ref_entry != 0 && prio == HIGH && count >= 1 : limit = 0 + * 5. ref_entry != 0 && prio == LOW && count >=1 : limit = Max + * (X2 2*8192, X4 8192) + */ + if (req->ref_entry && req->ref_prio == NPC_MCAM_LOWER_PRIO) { + if (req->kw_type == NPC_MCAM_KEY_X2) + limit = 2 * mcam->bmap_entries; + else + limit = mcam->bmap_entries; + } + + ret = npc_cn20k_ref_idx_alloc(rvu, pcifunc, req->kw_type, + req->ref_prio, rsp->entry_list, + req->ref_entry, limit, + req->contig, req->count, !!req->virt); + + if (ret) { + rsp->count = 0; + return NPC_MCAM_ALLOC_FAILED; + } + + rsp->count = req->count; + if (req->contig) + rsp->entry = rsp->entry_list[0]; + + /* cn20k, entries allocation algorithm is different. + * This common API updates some bitmap on usage etc, which + * will be used by other functions. So update those for + * cn20k as well. + */ + + mutex_lock(&mcam->lock); + /* Mark the allocated entries as used and set nixlf mapping */ + for (entry = 0; entry < rsp->count; entry++) { + index = npc_cn20k_vidx2idx(rsp->entry_list[entry]); + npc_mcam_set_bit(mcam, index); + mcam->entry2pfvf_map[index] = pcifunc; + mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; + } + + /* cn20k, free count is provided thru different mbox message. + * one counter to indicate free x2 slots and free x4 slots + * does not provide any useful information to the user. + */ + rsp->free_count = -1; + mutex_unlock(&mcam->lock); + + return 0; + +not_cn20k: mutex_lock(&mcam->lock); /* Check if there are any free entries */ @@ -2523,7 +2785,7 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, goto lprio_alloc; /* Get the search range for priority allocation request */ - if (req->priority) { + if (req->ref_prio) { npc_get_mcam_search_range_priority(mcam, req, &start, &end, &reverse); goto alloc; @@ -2564,11 +2826,11 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, * and not in mid zone. */ if (!(pcifunc & RVU_PFVF_FUNC_MASK) && - req->priority == NPC_MCAM_HIGHER_PRIO) + req->ref_prio == NPC_MCAM_HIGHER_PRIO) end = req->ref_entry; if (!(pcifunc & RVU_PFVF_FUNC_MASK) && - req->priority == NPC_MCAM_LOWER_PRIO) + req->ref_prio == NPC_MCAM_LOWER_PRIO) start = req->ref_entry; } @@ -2617,7 +2879,7 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, /* If allocating requested no of entries is unsucessful, * expand the search range to full bitmap length and retry. */ - if (!req->priority && (rsp->count < req->count) && + if (!req->ref_prio && rsp->count < req->count && ((end - start) != mcam->bmap_entries)) { reverse = true; start = 0; @@ -2628,14 +2890,14 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, /* For priority entry allocation requests, if allocation is * failed then expand search to max possible range and retry. */ - if (req->priority && rsp->count < req->count) { - if (req->priority == NPC_MCAM_LOWER_PRIO && + if (req->ref_prio && rsp->count < req->count) { + if (req->ref_prio == NPC_MCAM_LOWER_PRIO && (start != (req->ref_entry + 1))) { start = req->ref_entry + 1; end = mcam->bmap_entries; reverse = false; goto alloc; - } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && + } else if ((req->ref_prio == NPC_MCAM_HIGHER_PRIO) && ((end - start) != req->ref_entry)) { start = 0; end = req->ref_entry; @@ -2689,6 +2951,10 @@ int npc_config_cntr_default_entries(struct rvu *rvu, bool enable) struct rvu_npc_mcam_rule *rule; int blkaddr; + /* Counter is set for each rule by default */ + if (is_cn20k(rvu->pdev)) + return -EINVAL; + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return -EINVAL; @@ -2749,9 +3015,9 @@ int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, /* ref_entry can't be '0' if requested priority is high. * Can't be last entry if requested priority is low. */ - if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || - ((req->ref_entry == mcam->bmap_entries) && - req->priority == NPC_MCAM_LOWER_PRIO)) + if ((!req->ref_entry && req->ref_prio == NPC_MCAM_HIGHER_PRIO) || + (req->ref_entry == mcam->bmap_entries && + req->ref_prio == NPC_MCAM_LOWER_PRIO)) return NPC_MCAM_INVALID_REQ; /* Since list of allocated indices needs to be sent to requester, @@ -2780,6 +3046,8 @@ int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, int blkaddr, rc = 0; u16 cntr; + req->entry = npc_cn20k_vidx2idx(req->entry); + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; @@ -2808,6 +3076,14 @@ int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, req->entry, cntr); + if (is_cn20k(rvu->pdev)) { + rc = npc_cn20k_idx_free(rvu, &req->entry, 1); + if (rc) + dev_err(rvu->dev, + "Failed to free index=%u\n", + req->entry); + } + goto exit; free_all: @@ -2861,10 +3137,13 @@ int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, if (rc) goto exit; - if (req->set_cntr && - npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { - rc = NPC_MCAM_INVALID_REQ; - goto exit; + if (!is_cn20k(rvu->pdev)) { + /* Verify counter in SoCs other than cn20k */ + if (req->set_cntr && + npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { + rc = NPC_MCAM_INVALID_REQ; + goto exit; + } } if (!is_npc_interface_valid(rvu, req->intf)) { @@ -2902,6 +3181,8 @@ int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, u16 pcifunc = req->hdr.pcifunc; int blkaddr, rc; + req->entry = npc_cn20k_vidx2idx(req->entry); + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; @@ -2925,6 +3206,8 @@ int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, u16 pcifunc = req->hdr.pcifunc; int blkaddr, rc; + req->entry = npc_cn20k_vidx2idx(req->entry); + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; @@ -2959,8 +3242,8 @@ int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, mutex_lock(&mcam->lock); for (index = 0; index < req->shift_count; index++) { - old_entry = req->curr_entry[index]; - new_entry = req->new_entry[index]; + old_entry = npc_cn20k_vidx2idx(req->curr_entry[index]); + new_entry = npc_cn20k_vidx2idx(req->new_entry[index]); /* Check if both old and new entries are valid and * does belong to this PFFUNC or not. @@ -3032,7 +3315,6 @@ static int __npc_mcam_alloc_counter(struct rvu *rvu, if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) return NPC_MCAM_INVALID_REQ; - /* Check if unused counters are available or not */ if (!rvu_rsrc_free_count(&mcam->counters)) { return NPC_MCAM_ALLOC_FAILED; @@ -3077,6 +3359,10 @@ int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, struct npc_mcam *mcam = &rvu->hw->mcam; int err; + /* Counter is not supported for CN20K */ + if (is_cn20k(rvu->pdev)) + return NPC_MCAM_INVALID_REQ; + mutex_lock(&mcam->lock); err = __npc_mcam_alloc_counter(rvu, req, rsp); @@ -3131,6 +3417,10 @@ int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, struct npc_mcam *mcam = &rvu->hw->mcam; int err; + /* Counter is not supported for CN20K */ + if (is_cn20k(rvu->pdev)) + return NPC_MCAM_INVALID_REQ; + mutex_lock(&mcam->lock); err = __npc_mcam_free_counter(rvu, req, rsp); @@ -3189,6 +3479,10 @@ int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, u16 index, entry = 0; int blkaddr, rc; + /* Counter is not supported for CN20K */ + if (is_cn20k(rvu->pdev)) + return NPC_MCAM_INVALID_REQ; + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; @@ -3233,12 +3527,23 @@ int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) { struct npc_mcam *mcam = &rvu->hw->mcam; - int blkaddr, err; + int blkaddr, err, index, bank; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; + /* For cn20k, npc mcam index is passed as cntr, as each + * mcam entry has corresponding counter. + */ + if (is_cn20k(rvu->pdev)) { + index = req->cntr & (mcam->banksize - 1); + bank = npc_get_bank(mcam, req->cntr); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0); + return 0; + } + mutex_lock(&mcam->lock); err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); mutex_unlock(&mcam->lock); @@ -3255,12 +3560,26 @@ int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, struct npc_mcam_oper_counter_rsp *rsp) { struct npc_mcam *mcam = &rvu->hw->mcam; - int blkaddr, err; + int blkaddr, err, index, bank; + u64 regval; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; + /* In CN20K, mcam index is passed cntr. Each cn20k mcam entry + * has its own counter. No need to verify the counter index. + */ + if (is_cn20k(rvu->pdev)) { + index = req->cntr & (mcam->banksize - 1); + bank = npc_get_bank(mcam, req->cntr); + regval = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, + bank)); + rsp->stat = regval; + return 0; + } + mutex_lock(&mcam->lock); err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); mutex_unlock(&mcam->lock); @@ -3298,7 +3617,7 @@ int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, /* Try to allocate a MCAM entry */ entry_req.hdr.pcifunc = req->hdr.pcifunc; entry_req.contig = true; - entry_req.priority = req->priority; + entry_req.ref_prio = req->ref_prio; entry_req.ref_entry = req->ref_entry; entry_req.count = 1; @@ -3555,13 +3874,26 @@ int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu, if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; - mutex_lock(&mcam->lock); + req->entry = npc_cn20k_vidx2idx(req->entry); index = req->entry & (mcam->banksize - 1); bank = npc_get_bank(mcam, req->entry); + mutex_lock(&mcam->lock); + + if (is_cn20k(rvu->pdev)) { + regval = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, + bank)); + rsp->stat_ena = 1; + rsp->stat = regval; + mutex_unlock(&mcam->lock); + return 0; + } + /* read MCAM entry STAT_ACT register */ - regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank)); + regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, + bank)); if (!(regval & rvu->hw->npc_stat_ena)) { rsp->stat_ena = 0; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.h new file mode 100644 index 000000000000..83c5e32e2afc --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell RVU Admin Function driver + * + * Copyright (C) 2026 Marvell. + * + */ + +#ifndef RVU_NPC_H +#define RVU_NPC_H + +u64 npc_enable_mask(int count); +void npc_load_kpu_profile(struct rvu *rvu); +void npc_config_kpuaction(struct rvu *rvu, int blkaddr, + const struct npc_kpu_profile_action *kpuaction, + int kpu, int entry, bool pkind); +int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, + u64 *size); + +void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index); +void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index); +#endif /* RVU_NPC_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c index 1930b54e72f2..b45798d9fdab 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c @@ -12,6 +12,8 @@ #include "npc.h" #include "rvu_npc_fs.h" #include "rvu_npc_hash.h" +#include "cn20k/reg.h" +#include "cn20k/npc.h" static const char * const npc_flow_names[] = { [NPC_DMAC] = "dmac", @@ -81,19 +83,26 @@ const char *npc_get_field_name(u8 hdr) /* Compute keyword masks and figure out the number of keywords a field * spans in the key. */ -static void npc_set_kw_masks(struct npc_mcam *mcam, u8 type, +static void npc_set_kw_masks(struct rvu *rvu, struct npc_mcam *mcam, u8 type, u8 nr_bits, int start_kwi, int offset, u8 intf) { struct npc_key_field *field = &mcam->rx_key_fields[type]; u8 bits_in_kw; int max_kwi; - if (mcam->banks_per_entry == 1) - max_kwi = 1; /* NPC_MCAM_KEY_X1 */ - else if (mcam->banks_per_entry == 2) - max_kwi = 3; /* NPC_MCAM_KEY_X2 */ - else - max_kwi = 6; /* NPC_MCAM_KEY_X4 */ + if (is_cn20k(rvu->pdev)) { + if (mcam->banks_per_entry == 1) + max_kwi = 3; /* NPC_MCAM_KEY_X2 */ + else + max_kwi = 7; /* NPC_MCAM_KEY_X4 */ + } else { + if (mcam->banks_per_entry == 1) + max_kwi = 1; /* NPC_MCAM_KEY_X1 */ + else if (mcam->banks_per_entry == 2) + max_kwi = 3; /* NPC_MCAM_KEY_X2 */ + else + max_kwi = 6; /* NPC_MCAM_KEY_X4 */ + } if (is_npc_intf_tx(intf)) field = &mcam->tx_key_fields[type]; @@ -155,7 +164,8 @@ static bool npc_is_same(struct npc_key_field *input, sizeof(struct npc_layer_mdata)) == 0; } -static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type, +static void npc_set_layer_mdata(struct rvu *rvu, + struct npc_mcam *mcam, enum key_fields type, u64 cfg, u8 lid, u8 lt, u8 intf) { struct npc_key_field *input = &mcam->rx_key_fields[type]; @@ -165,13 +175,17 @@ static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type, input->layer_mdata.hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); input->layer_mdata.key = FIELD_GET(NPC_KEY_OFFSET, cfg); - input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1; + if (is_cn20k(rvu->pdev)) + input->layer_mdata.len = FIELD_GET(NPC_CN20K_BYTESM, cfg) + 1; + else + input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1; input->layer_mdata.ltype = lt; input->layer_mdata.lid = lid; } static bool npc_check_overlap_fields(struct npc_key_field *input1, - struct npc_key_field *input2) + struct npc_key_field *input2, + int max_kw) { int kwi; @@ -182,7 +196,7 @@ static bool npc_check_overlap_fields(struct npc_key_field *input1, input1->layer_mdata.ltype != input2->layer_mdata.ltype) return false; - for (kwi = 0; kwi < NPC_MAX_KWS_IN_KEY; kwi++) { + for (kwi = 0; kwi < max_kw; kwi++) { if (input1->kw_mask[kwi] & input2->kw_mask[kwi]) return true; } @@ -202,6 +216,7 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr, struct npc_key_field *dummy, *input; int start_kwi, offset; u8 nr_bits, lid, lt, ld; + int extr, kws; u64 cfg; dummy = &mcam->rx_key_fields[NPC_UNKNOWN]; @@ -212,6 +227,11 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr, input = &mcam->tx_key_fields[type]; } + kws = NPC_KWS_IN_KEY_SZ_7; + + if (is_cn20k(rvu->pdev)) + goto skip_cn10k_config; + for (lid = start_lid; lid < NPC_MAX_LID; lid++) { for (lt = 0; lt < NPC_MAX_LT; lt++) { for (ld = 0; ld < NPC_MAX_LD; ld++) { @@ -221,8 +241,8 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr, if (!FIELD_GET(NPC_LDATA_EN, cfg)) continue; memset(dummy, 0, sizeof(struct npc_key_field)); - npc_set_layer_mdata(mcam, NPC_UNKNOWN, cfg, - lid, lt, intf); + npc_set_layer_mdata(rvu, mcam, NPC_UNKNOWN, + cfg, lid, lt, intf); /* exclude input */ if (npc_is_same(input, dummy)) continue; @@ -230,16 +250,49 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr, offset = (dummy->layer_mdata.key * 8) % 64; nr_bits = dummy->layer_mdata.len * 8; /* form KW masks */ - npc_set_kw_masks(mcam, NPC_UNKNOWN, nr_bits, - start_kwi, offset, intf); + npc_set_kw_masks(rvu, mcam, NPC_UNKNOWN, + nr_bits, start_kwi, + offset, intf); /* check any input field bits falls in any * other field bits. */ - if (npc_check_overlap_fields(dummy, input)) + if (npc_check_overlap_fields(dummy, input, kws)) return true; } } } + return false; + +skip_cn10k_config: + for (extr = 0 ; extr < rvu->hw->npc_kex_extr; extr++) { + lid = CN20K_GET_EXTR_LID(intf, extr); + if (lid < start_lid) + continue; + for (lt = 0; lt < NPC_MAX_LT; lt++) { + cfg = CN20K_GET_EXTR_LT(intf, extr, lt); + if (!FIELD_GET(NPC_LDATA_EN, cfg)) + continue; + + memset(dummy, 0, sizeof(struct npc_key_field)); + npc_set_layer_mdata(rvu, mcam, NPC_UNKNOWN, cfg, + lid, lt, intf); + /* exclude input */ + if (npc_is_same(input, dummy)) + continue; + start_kwi = dummy->layer_mdata.key / 8; + offset = (dummy->layer_mdata.key * 8) % 64; + nr_bits = dummy->layer_mdata.len * 8; + /* form KW masks */ + npc_set_kw_masks(rvu, mcam, NPC_UNKNOWN, nr_bits, + start_kwi, offset, intf); + /* check any input field bits falls in any other + * field bits + */ + if (npc_check_overlap_fields(dummy, input, + NPC_KWS_IN_KEY_SZ_8)) + return true; + } + } return false; } @@ -253,7 +306,8 @@ static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type, return true; } -static void npc_scan_exact_result(struct npc_mcam *mcam, u8 bit_number, +static void npc_scan_exact_result(struct rvu *rvu, + struct npc_mcam *mcam, u8 bit_number, u8 key_nibble, u8 intf) { u8 offset = (key_nibble * 4) % 64; /* offset within key word */ @@ -269,10 +323,63 @@ static void npc_scan_exact_result(struct npc_mcam *mcam, u8 bit_number, default: return; } - npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); + npc_set_kw_masks(rvu, mcam, type, nr_bits, kwi, offset, intf); } -static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, +static void npc_cn20k_scan_parse_result(struct rvu *rvu, struct npc_mcam *mcam, + u8 bit_number, u8 key_nibble, u8 intf) +{ + u8 offset = (key_nibble * 4) % 64; /* offset within key word */ + u8 kwi = (key_nibble * 4) / 64; /* which word in key */ + u8 nr_bits = 4; /* bits in a nibble */ + u8 type; + + switch (bit_number) { + case 0 ... 2: + type = NPC_CHAN; + break; + case 3: + type = NPC_ERRLEV; + break; + case 4 ... 5: + type = NPC_ERRCODE; + break; + case 6: + type = NPC_LXMB; + break; + case 8: + type = NPC_LA; + break; + case 10: + type = NPC_LB; + break; + case 12: + type = NPC_LC; + break; + case 14: + type = NPC_LD; + break; + case 16: + type = NPC_LE; + break; + case 18: + type = NPC_LF; + break; + case 20: + type = NPC_LG; + break; + case 22: + type = NPC_LH; + break; + default: + return; + } + + npc_set_kw_masks(rvu, mcam, type, nr_bits, kwi, offset, intf); +} + +static void npc_scan_parse_result(struct rvu *rvu, + struct npc_mcam *mcam, u8 bit_number, u8 key_nibble, u8 intf) { u8 offset = (key_nibble * 4) % 64; /* offset within key word */ @@ -280,6 +387,12 @@ static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, u8 nr_bits = 4; /* bits in a nibble */ u8 type; + if (is_cn20k(rvu->pdev)) { + npc_cn20k_scan_parse_result(rvu, mcam, bit_number, + key_nibble, intf); + return; + } + switch (bit_number) { case 0 ... 2: type = NPC_CHAN; @@ -322,7 +435,7 @@ static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, return; } - npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); + npc_set_kw_masks(rvu, mcam, type, nr_bits, kwi, offset, intf); } static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) @@ -343,8 +456,13 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) /* Inner VLAN TCI for double tagged frames */ struct npc_key_field *vlan_tag3; u64 *features; + int i, max_kw; u8 start_lid; - int i; + + if (is_cn20k(rvu->pdev)) + max_kw = NPC_KWS_IN_KEY_SZ_8; + else + max_kw = NPC_KWS_IN_KEY_SZ_7; key_fields = mcam->rx_key_fields; features = &mcam->rx_features; @@ -382,7 +500,7 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) /* if key profile programmed extracts Ethertype from multiple layers */ if (etype_ether->nr_kws && etype_tag1->nr_kws) { - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + for (i = 0; i < max_kw; i++) { if (etype_ether->kw_mask[i] != etype_tag1->kw_mask[i]) { dev_err(rvu->dev, "mkex: Etype pos is different for untagged and tagged pkts.\n"); goto vlan_tci; @@ -391,7 +509,7 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) key_fields[NPC_ETYPE] = *etype_tag1; } if (etype_ether->nr_kws && etype_tag2->nr_kws) { - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + for (i = 0; i < max_kw; i++) { if (etype_ether->kw_mask[i] != etype_tag2->kw_mask[i]) { dev_err(rvu->dev, "mkex: Etype pos is different for untagged and double tagged pkts.\n"); goto vlan_tci; @@ -400,7 +518,7 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) key_fields[NPC_ETYPE] = *etype_tag2; } if (etype_tag1->nr_kws && etype_tag2->nr_kws) { - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + for (i = 0; i < max_kw; i++) { if (etype_tag1->kw_mask[i] != etype_tag2->kw_mask[i]) { dev_err(rvu->dev, "mkex: Etype pos is different for tagged and double tagged pkts.\n"); goto vlan_tci; @@ -431,7 +549,7 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) /* if key profile extracts outer vlan tci from multiple layers */ if (vlan_tag1->nr_kws && vlan_tag2->nr_kws) { - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + for (i = 0; i < max_kw; i++) { if (vlan_tag1->kw_mask[i] != vlan_tag2->kw_mask[i]) { dev_err(rvu->dev, "mkex: Out vlan tci pos is different for tagged and double tagged pkts.\n"); goto done; @@ -466,7 +584,11 @@ static void npc_scan_ldata(struct rvu *rvu, int blkaddr, u8 lid, /* starting KW index and starting bit position */ int start_kwi, offset; - nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1; + if (is_cn20k(rvu->pdev)) + nr_bytes = FIELD_GET(NPC_CN20K_BYTESM, cfg) + 1; + else + nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1; + hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); key = FIELD_GET(NPC_KEY_OFFSET, cfg); @@ -489,11 +611,12 @@ do { \ if ((hstart) >= hdr && \ ((hstart) + (hlen)) <= (hdr + nr_bytes)) { \ bit_offset = (hdr + nr_bytes - (hstart) - (hlen)) * 8; \ - npc_set_layer_mdata(mcam, (name), cfg, lid, lt, intf); \ + npc_set_layer_mdata(rvu, mcam, (name), cfg, lid, lt, \ + intf); \ offset += bit_offset; \ start_kwi += offset / 64; \ offset %= 64; \ - npc_set_kw_masks(mcam, (name), (hlen) * 8, \ + npc_set_kw_masks(rvu, mcam, (name), (hlen) * 8, \ start_kwi, offset, intf); \ } \ } \ @@ -636,6 +759,7 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) u8 lid, lt, ld, bitnr; u64 cfg, masked_cfg; u8 key_nibble = 0; + int extr; /* Scan and note how parse result is going to be in key. * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from @@ -643,10 +767,22 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) * will be concatenated in key. */ cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); - masked_cfg = cfg & NPC_PARSE_NIBBLE; - for_each_set_bit(bitnr, (unsigned long *)&masked_cfg, 31) { - npc_scan_parse_result(mcam, bitnr, key_nibble, intf); - key_nibble++; + if (is_cn20k(rvu->pdev)) { + masked_cfg = cfg & NPC_CN20K_PARSE_NIBBLE; + for_each_set_bit(bitnr, (unsigned long *)&masked_cfg, + NPC_CN20K_TOTAL_NIBBLE) { + npc_scan_parse_result(rvu, mcam, bitnr, + key_nibble, intf); + key_nibble++; + } + } else { + masked_cfg = cfg & NPC_PARSE_NIBBLE; + for_each_set_bit(bitnr, (unsigned long *)&masked_cfg, + NPC_TOTAL_NIBBLE) { + npc_scan_parse_result(rvu, mcam, bitnr, + key_nibble, intf); + key_nibble++; + } } /* Ignore exact match bits for mcam entries except the first rule @@ -656,10 +792,13 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) masked_cfg = cfg & NPC_EXACT_NIBBLE; bitnr = NPC_EXACT_NIBBLE_START; for_each_set_bit_from(bitnr, (unsigned long *)&masked_cfg, NPC_EXACT_NIBBLE_END + 1) { - npc_scan_exact_result(mcam, bitnr, key_nibble, intf); + npc_scan_exact_result(rvu, mcam, bitnr, key_nibble, intf); key_nibble++; } + if (is_cn20k(rvu->pdev)) + goto skip_cn10k_config; + /* Scan and note how layer data is going to be in key */ for (lid = 0; lid < NPC_MAX_LID; lid++) { for (lt = 0; lt < NPC_MAX_LT; lt++) { @@ -676,6 +815,19 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) } return 0; + +skip_cn10k_config: + for (extr = 0 ; extr < rvu->hw->npc_kex_extr; extr++) { + lid = CN20K_GET_EXTR_LID(intf, extr); + for (lt = 0; lt < NPC_MAX_LT; lt++) { + cfg = CN20K_GET_EXTR_LT(intf, extr, lt); + if (!FIELD_GET(NPC_LDATA_EN, cfg)) + continue; + npc_scan_ldata(rvu, blkaddr, lid, lt, cfg, + intf); + } + } + return 0; } static int npc_scan_verify_kex(struct rvu *rvu, int blkaddr) @@ -751,15 +903,17 @@ static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf) * dont care. */ void npc_update_entry(struct rvu *rvu, enum key_fields type, - struct mcam_entry *entry, u64 val_lo, + struct mcam_entry_mdata *mdata, u64 val_lo, u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf) { + u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; + u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; struct npc_mcam *mcam = &rvu->hw->mcam; - struct mcam_entry dummy = { {0} }; struct npc_key_field *field; u64 kw1, kw2, kw3; + u64 *val, *mask; + int i, max_kw; u8 shift; - int i; field = &mcam->rx_key_fields[type]; if (is_npc_intf_tx(intf)) @@ -768,17 +922,22 @@ void npc_update_entry(struct rvu *rvu, enum key_fields type, if (!field->nr_kws) return; - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + if (is_cn20k(rvu->pdev)) + max_kw = NPC_KWS_IN_KEY_SZ_8; + else + max_kw = NPC_KWS_IN_KEY_SZ_7; + + for (i = 0; i < max_kw; i++) { if (!field->kw_mask[i]) continue; /* place key value in kw[x] */ shift = __ffs64(field->kw_mask[i]); /* update entry value */ kw1 = (val_lo << shift) & field->kw_mask[i]; - dummy.kw[i] = kw1; + kw[i] = kw1; /* update entry mask */ kw1 = (mask_lo << shift) & field->kw_mask[i]; - dummy.kw_mask[i] = kw1; + kw_mask[i] = kw1; if (field->nr_kws == 1) break; @@ -788,12 +947,12 @@ void npc_update_entry(struct rvu *rvu, enum key_fields type, kw2 = shift ? val_lo >> (64 - shift) : 0; kw2 |= (val_hi << shift); kw2 &= field->kw_mask[i + 1]; - dummy.kw[i + 1] = kw2; + kw[i + 1] = kw2; /* update entry mask */ kw2 = shift ? mask_lo >> (64 - shift) : 0; kw2 |= (mask_hi << shift); kw2 &= field->kw_mask[i + 1]; - dummy.kw_mask[i + 1] = kw2; + kw_mask[i + 1] = kw2; break; } /* place remaining bits of key value in kw[x + 1], kw[x + 2] */ @@ -804,34 +963,40 @@ void npc_update_entry(struct rvu *rvu, enum key_fields type, kw2 &= field->kw_mask[i + 1]; kw3 = shift ? val_hi >> (64 - shift) : 0; kw3 &= field->kw_mask[i + 2]; - dummy.kw[i + 1] = kw2; - dummy.kw[i + 2] = kw3; + kw[i + 1] = kw2; + kw[i + 2] = kw3; /* update entry mask */ kw2 = shift ? mask_lo >> (64 - shift) : 0; kw2 |= (mask_hi << shift); kw2 &= field->kw_mask[i + 1]; kw3 = shift ? mask_hi >> (64 - shift) : 0; kw3 &= field->kw_mask[i + 2]; - dummy.kw_mask[i + 1] = kw2; - dummy.kw_mask[i + 2] = kw3; + kw_mask[i + 1] = kw2; + kw_mask[i + 2] = kw3; break; } } /* dummy is ready with values and masks for given key * field now clear and update input entry with those */ - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + + val = mdata->kw; + mask = mdata->kw_mask; + + for (i = 0; i < max_kw; i++, val++, mask++) { if (!field->kw_mask[i]) continue; - entry->kw[i] &= ~field->kw_mask[i]; - entry->kw_mask[i] &= ~field->kw_mask[i]; - entry->kw[i] |= dummy.kw[i]; - entry->kw_mask[i] |= dummy.kw_mask[i]; + *val &= ~field->kw_mask[i]; + *mask &= ~field->kw_mask[i]; + + *val |= kw[i]; + *mask |= kw_mask[i]; } } -static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry, +static void npc_update_ipv6_flow(struct rvu *rvu, + struct mcam_entry_mdata *mdata, u64 features, struct flow_msg *pkt, struct flow_msg *mask, struct rvu_npc_mcam_rule *output, u8 intf) @@ -857,7 +1022,7 @@ static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry, val_hi = (u64)src_ip[0] << 32 | src_ip[1]; val_lo = (u64)src_ip[2] << 32 | src_ip[3]; - npc_update_entry(rvu, NPC_SIP_IPV6, entry, val_lo, val_hi, + npc_update_entry(rvu, NPC_SIP_IPV6, mdata, val_lo, val_hi, mask_lo, mask_hi, intf); memcpy(opkt->ip6src, pkt->ip6src, sizeof(opkt->ip6src)); memcpy(omask->ip6src, mask->ip6src, sizeof(omask->ip6src)); @@ -871,14 +1036,15 @@ static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry, val_hi = (u64)dst_ip[0] << 32 | dst_ip[1]; val_lo = (u64)dst_ip[2] << 32 | dst_ip[3]; - npc_update_entry(rvu, NPC_DIP_IPV6, entry, val_lo, val_hi, + npc_update_entry(rvu, NPC_DIP_IPV6, mdata, val_lo, val_hi, mask_lo, mask_hi, intf); memcpy(opkt->ip6dst, pkt->ip6dst, sizeof(opkt->ip6dst)); memcpy(omask->ip6dst, mask->ip6dst, sizeof(omask->ip6dst)); } } -static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry, +static void npc_update_vlan_features(struct rvu *rvu, + struct mcam_entry_mdata *mdata, u64 features, u8 intf) { bool ctag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_CTAG)); @@ -887,24 +1053,24 @@ static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry, /* If only VLAN id is given then always match outer VLAN id */ if (vid && !ctag && !stag) { - npc_update_entry(rvu, NPC_LB, entry, + npc_update_entry(rvu, NPC_LB, mdata, NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG, 0, NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG, 0, intf); return; } if (ctag) - npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_CTAG, 0, + npc_update_entry(rvu, NPC_LB, mdata, NPC_LT_LB_CTAG, 0, ~0ULL, 0, intf); if (stag) - npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_STAG_QINQ, 0, + npc_update_entry(rvu, NPC_LB, mdata, NPC_LT_LB_STAG_QINQ, 0, ~0ULL, 0, intf); } -static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry, - u64 features, struct flow_msg *pkt, - struct flow_msg *mask, - struct rvu_npc_mcam_rule *output, u8 intf, - int blkaddr) +void npc_update_flow(struct rvu *rvu, struct mcam_entry_mdata *mdata, + u64 features, struct flow_msg *pkt, + struct flow_msg *mask, + struct rvu_npc_mcam_rule *output, u8 intf, + int blkaddr) { u64 dmac_mask = ether_addr_to_u64(mask->dmac); u64 smac_mask = ether_addr_to_u64(mask->smac); @@ -918,39 +1084,39 @@ static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry, /* For tcp/udp/sctp LTYPE should be present in entry */ if (features & BIT_ULL(NPC_IPPROTO_TCP)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_TCP, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_TCP, 0, ~0ULL, 0, intf); if (features & BIT_ULL(NPC_IPPROTO_UDP)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_UDP, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_UDP, 0, ~0ULL, 0, intf); if (features & BIT_ULL(NPC_IPPROTO_SCTP)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_SCTP, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_SCTP, 0, ~0ULL, 0, intf); if (features & BIT_ULL(NPC_IPPROTO_ICMP)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_ICMP, 0, ~0ULL, 0, intf); if (features & BIT_ULL(NPC_IPPROTO_ICMP6)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP6, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_ICMP6, 0, ~0ULL, 0, intf); /* For AH, LTYPE should be present in entry */ if (features & BIT_ULL(NPC_IPPROTO_AH)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_AH, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_AH, 0, ~0ULL, 0, intf); /* For ESP, LTYPE should be present in entry */ if (features & BIT_ULL(NPC_IPPROTO_ESP)) - npc_update_entry(rvu, NPC_LE, entry, NPC_LT_LE_ESP, + npc_update_entry(rvu, NPC_LE, mdata, NPC_LT_LE_ESP, 0, ~0ULL, 0, intf); if (features & BIT_ULL(NPC_LXMB)) { output->lxmb = is_broadcast_ether_addr(pkt->dmac) ? 2 : 1; - npc_update_entry(rvu, NPC_LXMB, entry, output->lxmb, 0, + npc_update_entry(rvu, NPC_LXMB, mdata, output->lxmb, 0, output->lxmb, 0, intf); } #define NPC_WRITE_FLOW(field, member, val_lo, val_hi, mask_lo, mask_hi) \ do { \ if (features & BIT_ULL((field))) { \ - npc_update_entry(rvu, (field), entry, (val_lo), (val_hi), \ + npc_update_entry(rvu, (field), mdata, (val_lo), (val_hi), \ (mask_lo), (mask_hi), intf); \ memcpy(&opkt->member, &pkt->member, sizeof(pkt->member)); \ memcpy(&omask->member, &mask->member, sizeof(mask->member)); \ @@ -1038,10 +1204,10 @@ do { \ NPC_WRITE_FLOW(NPC_IPFRAG_IPV6, next_header, pkt->next_header, 0, mask->next_header, 0); - npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf); - npc_update_vlan_features(rvu, entry, features, intf); + npc_update_ipv6_flow(rvu, mdata, features, pkt, mask, output, intf); + npc_update_vlan_features(rvu, mdata, features, intf); - npc_update_field_hash(rvu, intf, entry, blkaddr, features, + npc_update_field_hash(rvu, intf, mdata, blkaddr, features, pkt, mask, opkt, omask); } @@ -1083,6 +1249,10 @@ static void rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc, { struct npc_mcam *mcam = &rvu->hw->mcam; + /* There is no counter allotted for cn20k */ + if (is_cn20k(rvu->pdev)) + return; + mutex_lock(&mcam->lock); __rvu_mcam_remove_counter_from_rule(rvu, pcifunc, rule); @@ -1129,8 +1299,29 @@ static int npc_mcast_update_action_index(struct rvu *rvu, struct npc_install_flo return 0; } +void +npc_populate_mcam_mdata(struct rvu *rvu, + struct mcam_entry_mdata *mdata, + struct cn20k_mcam_entry *cn20k_entry, + struct mcam_entry *entry) +{ + if (is_cn20k(rvu->pdev)) { + mdata->kw = cn20k_entry->kw; + mdata->kw_mask = cn20k_entry->kw_mask; + mdata->action = &cn20k_entry->action; + mdata->vtag_action = &cn20k_entry->vtag_action; + mdata->max_kw = NPC_KWS_IN_KEY_SZ_8; + return; + } + mdata->kw = entry->kw; + mdata->kw_mask = entry->kw_mask; + mdata->action = &entry->action; + mdata->vtag_action = &entry->vtag_action; + mdata->max_kw = NPC_KWS_IN_KEY_SZ_7; +} + static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, - struct mcam_entry *entry, + struct mcam_entry_mdata *mdata, struct npc_install_flow_req *req, u16 target, bool pf_set_vfs_mac) { @@ -1141,7 +1332,7 @@ static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, if (rswitch->mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && pf_set_vfs_mac) req->chan_mask = 0x0; /* Do not care channel */ - npc_update_entry(rvu, NPC_CHAN, entry, req->channel, 0, req->chan_mask, + npc_update_entry(rvu, NPC_CHAN, mdata, req->channel, 0, req->chan_mask, 0, NIX_INTF_RX); *(u64 *)&action = 0x00; @@ -1173,12 +1364,12 @@ static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, action.match_id = req->match_id; } - entry->action = *(u64 *)&action; + *mdata->action = *(u64 *)&action; /* VTAG0 starts at 0th byte of LID_B. * VTAG1 starts at 4th byte of LID_B. */ - entry->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) | + *mdata->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) | FIELD_PREP(RX_VTAG0_TYPE_MASK, req->vtag0_type) | FIELD_PREP(RX_VTAG0_LID_MASK, NPC_LID_LB) | FIELD_PREP(RX_VTAG0_RELPTR_MASK, 0) | @@ -1191,7 +1382,7 @@ static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, } static int npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, - struct mcam_entry *entry, + struct mcam_entry_mdata *mdata, struct npc_install_flow_req *req, u16 target) { struct nix_tx_action action; @@ -1204,7 +1395,7 @@ static int npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, if (is_pffunc_af(req->hdr.pcifunc)) mask = 0; - npc_update_entry(rvu, NPC_PF_FUNC, entry, (__force u16)htons(target), + npc_update_entry(rvu, NPC_PF_FUNC, mdata, (__force u16)htons(target), 0, mask, 0, NIX_INTF_TX); *(u64 *)&action = 0x00; @@ -1217,12 +1408,12 @@ static int npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, action.match_id = req->match_id; - entry->action = *(u64 *)&action; + *mdata->action = *(u64 *)&action; /* VTAG0 starts at 0th byte of LID_B. * VTAG1 starts at 4th byte of LID_B. */ - entry->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) | + *mdata->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) | FIELD_PREP(TX_VTAG0_OP_MASK, req->vtag0_op) | FIELD_PREP(TX_VTAG0_LID_MASK, NPC_LID_LA) | FIELD_PREP(TX_VTAG0_RELPTR_MASK, 20) | @@ -1241,9 +1432,12 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, bool pf_set_vfs_mac) { struct rvu_npc_mcam_rule *def_ucast_rule = pfvf->def_ucast_rule; + struct npc_cn20k_mcam_write_entry_req cn20k_wreq = { 0 }; u64 features, installed_features, missing_features = 0; struct npc_mcam_write_entry_req write_req = { 0 }; struct npc_mcam *mcam = &rvu->hw->mcam; + struct cn20k_mcam_entry *cn20k_entry; + struct mcam_entry_mdata mdata = { }; struct rvu_npc_mcam_rule dummy = { 0 }; struct rvu_npc_mcam_rule *rule; u16 owner = req->hdr.pcifunc; @@ -1255,18 +1449,23 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, installed_features = req->features; features = req->features; - entry = &write_req.entry_data; entry_index = req->entry; - npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy, + cn20k_entry = &cn20k_wreq.entry_data; + entry = &write_req.entry_data; + + npc_populate_mcam_mdata(rvu, &mdata, cn20k_entry, entry); + + npc_update_flow(rvu, &mdata, features, &req->packet, &req->mask, &dummy, req->intf, blkaddr); if (is_npc_intf_rx(req->intf)) { - err = npc_update_rx_entry(rvu, pfvf, entry, req, target, pf_set_vfs_mac); + err = npc_update_rx_entry(rvu, pfvf, &mdata, req, target, + pf_set_vfs_mac); if (err) return err; } else { - err = npc_update_tx_entry(rvu, pfvf, entry, req, target); + err = npc_update_tx_entry(rvu, pfvf, &mdata, req, target); if (err) return err; } @@ -1286,7 +1485,7 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, missing_features = (def_ucast_rule->features ^ features) & def_ucast_rule->features; if (missing_features) - npc_update_flow(rvu, entry, missing_features, + npc_update_flow(rvu, &mdata, missing_features, &def_ucast_rule->packet, &def_ucast_rule->mask, &dummy, req->intf, @@ -1303,51 +1502,91 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, new = true; } - /* allocate new counter if rule has no counter */ - if (!req->default_rule && req->set_cntr && !rule->has_cntr) - rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp); + if (!is_cn20k(rvu->pdev)) { + write_req.hdr.pcifunc = owner; - /* if user wants to delete an existing counter for a rule then - * free the counter - */ - if (!req->set_cntr && rule->has_cntr) - rvu_mcam_remove_counter_from_rule(rvu, owner, rule); + /* allocate new counter if rule has no counter */ + if (!req->default_rule && req->set_cntr && !rule->has_cntr) + rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp); - write_req.hdr.pcifunc = owner; + /* if user wants to delete an existing counter for a rule then + * free the counter + */ + if (!req->set_cntr && rule->has_cntr) + rvu_mcam_remove_counter_from_rule(rvu, owner, rule); - /* AF owns the default rules so change the owner just to relax - * the checks in rvu_mbox_handler_npc_mcam_write_entry - */ - if (req->default_rule) - write_req.hdr.pcifunc = 0; + /* AF owns the default rules so change the owner just to relax + * the checks in rvu_mbox_handler_npc_mcam_write_entry + */ + if (req->default_rule) + write_req.hdr.pcifunc = 0; - write_req.entry = entry_index; - write_req.intf = req->intf; - write_req.enable_entry = (u8)enable; - /* if counter is available then clear and use it */ - if (req->set_cntr && rule->has_cntr) { - rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), req->cntr_val); - write_req.set_cntr = 1; - write_req.cntr = rule->cntr; + write_req.entry = entry_index; + write_req.intf = req->intf; + write_req.enable_entry = (u8)enable; + /* if counter is available then clear and use it */ + if (req->set_cntr && rule->has_cntr) { + rvu_write64(rvu, blkaddr, + NPC_AF_MATCH_STATX(rule->cntr), + req->cntr_val); + write_req.set_cntr = 1; + write_req.cntr = rule->cntr; + } + goto update_rule; } + cn20k_wreq.hdr.pcifunc = owner; + + if (req->default_rule) + cn20k_wreq.hdr.pcifunc = 0; + + cn20k_wreq.entry = entry_index; + cn20k_wreq.intf = req->intf; + cn20k_wreq.enable_entry = (u8)enable; + cn20k_wreq.hw_prio = req->hw_prio; + cn20k_wreq.req_kw_type = req->req_kw_type; + +update_rule: + /* update rule */ memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet)); memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask)); rule->entry = entry_index; - memcpy(&rule->rx_action, &entry->action, sizeof(struct nix_rx_action)); - if (is_npc_intf_tx(req->intf)) - memcpy(&rule->tx_action, &entry->action, - sizeof(struct nix_tx_action)); - rule->vtag_action = entry->vtag_action; + if (is_cn20k(rvu->pdev)) { + memcpy(&rule->rx_action, &cn20k_entry->action, + sizeof(struct nix_rx_action)); + if (is_npc_intf_tx(req->intf)) + memcpy(&rule->tx_action, &cn20k_entry->action, + sizeof(struct nix_tx_action)); + rule->vtag_action = cn20k_entry->vtag_action; + } else { + memcpy(&rule->rx_action, &entry->action, + sizeof(struct nix_rx_action)); + if (is_npc_intf_tx(req->intf)) + memcpy(&rule->tx_action, &entry->action, + sizeof(struct nix_tx_action)); + rule->vtag_action = entry->vtag_action; + } + rule->features = installed_features; rule->default_rule = req->default_rule; rule->owner = owner; rule->enable = enable; - rule->chan_mask = write_req.entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK; - rule->chan = write_req.entry_data.kw[0] & NPC_KEX_CHAN_MASK; + + if (is_cn20k(rvu->pdev)) { + rule->chan_mask = cn20k_wreq.entry_data.kw_mask[0] & + NPC_KEX_CHAN_MASK; + rule->chan = cn20k_wreq.entry_data.kw[0] & + NPC_KEX_CHAN_MASK; + } else { + rule->chan_mask = write_req.entry_data.kw_mask[0] & + NPC_KEX_CHAN_MASK; + rule->chan = write_req.entry_data.kw[0] & NPC_KEX_CHAN_MASK; + } + rule->chan &= rule->chan_mask; rule->lxmb = dummy.lxmb; + rule->hw_prio = req->hw_prio; if (is_npc_intf_tx(req->intf)) rule->intf = pfvf->nix_tx_intf; else @@ -1359,8 +1598,14 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, pfvf->def_ucast_rule = rule; /* write to mcam entry registers */ - err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, - &write_rsp); + if (is_cn20k(rvu->pdev)) + err = rvu_mbox_handler_npc_cn20k_mcam_write_entry(rvu, + &cn20k_wreq, + &write_rsp); + else + err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, + &write_rsp); + if (err) { rvu_mcam_remove_counter_from_rule(rvu, owner, rule); if (new) { @@ -1393,6 +1638,79 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, return 0; } +static int +rvu_npc_free_entry_for_flow_install(struct rvu *rvu, u16 pcifunc, + bool free_entry, int mcam_idx) +{ + struct npc_mcam_free_entry_req free_req = { 0 }; + struct msg_rsp rsp; + int rc; + + if (!free_entry) + return 0; + + free_req.hdr.pcifunc = pcifunc; + free_req.entry = mcam_idx; + rc = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &rsp); + return rc; +} + +static int +rvu_npc_alloc_entry_for_flow_install(struct rvu *rvu, + struct npc_install_flow_req *fl_req, + u16 *mcam_idx, u8 *kw_type, + bool *allocated) +{ + struct npc_mcam_alloc_entry_req entry_req; + struct npc_mcam_alloc_entry_rsp entry_rsp; + struct npc_get_num_kws_req kws_req; + struct npc_get_num_kws_rsp kws_rsp; + int off, kw_bits, rc; + u8 *src, *dst; + + if (!is_cn20k(rvu->pdev)) { + *kw_type = -1; + return 0; + } + + if (!fl_req->alloc_entry) { + *kw_type = -1; + return 0; + } + + off = offsetof(struct npc_install_flow_req, packet); + dst = (u8 *)&kws_req.fl + off; + src = (u8 *)fl_req + off; + memcpy(dst, src, sizeof(struct npc_install_flow_req) - off); + rc = rvu_mbox_handler_npc_get_num_kws(rvu, &kws_req, &kws_rsp); + if (rc) + return rc; + + kw_bits = kws_rsp.kws * 64; + + *kw_type = NPC_MCAM_KEY_X2; + if (kw_bits > 256) + *kw_type = NPC_MCAM_KEY_X4; + + memset(&entry_req, 0, sizeof(entry_req)); + memset(&entry_rsp, 0, sizeof(entry_rsp)); + + entry_req.hdr.pcifunc = fl_req->hdr.pcifunc; + entry_req.ref_prio = fl_req->ref_prio; + entry_req.ref_entry = fl_req->ref_entry; + entry_req.kw_type = *kw_type; + entry_req.count = 1; + rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, + &entry_req, + &entry_rsp); + if (rc) + return rc; + + *mcam_idx = entry_rsp.entry_list[0]; + *allocated = true; + return 0; +} + int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, struct npc_install_flow_req *req, struct npc_install_flow_rsp *rsp) @@ -1403,7 +1721,9 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, int blkaddr, nixlf, err; struct rvu_pfvf *pfvf; bool pf_set_vfs_mac = false; + bool allocated = false; bool enable = true; + u8 kw_type; u16 target; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); @@ -1415,6 +1735,17 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, if (!is_npc_interface_valid(rvu, req->intf)) return NPC_FLOW_INTF_INVALID; + err = rvu_npc_alloc_entry_for_flow_install(rvu, req, &req->entry, + &kw_type, &allocated); + if (err) { + dev_err(rvu->dev, + "%s: Error to alloc mcam entry for pcifunc=%#x\n", + __func__, req->hdr.pcifunc); + return err; + } + + req->entry = npc_cn20k_vidx2idx(req->entry); + /* If DMAC is not extracted in MKEX, rules installed by AF * can rely on L2MB bit set by hardware protocol checker for * broadcast and multicast addresses. @@ -1428,6 +1759,10 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, dev_warn(rvu->dev, "%s: mkex profile does not support ucast flow\n", __func__); + rvu_npc_free_entry_for_flow_install(rvu, + req->hdr.pcifunc, + allocated, + req->entry); return NPC_FLOW_NOT_SUPPORTED; } @@ -1435,6 +1770,10 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, dev_warn(rvu->dev, "%s: mkex profile does not support bcast/mcast flow", __func__); + rvu_npc_free_entry_for_flow_install(rvu, + req->hdr.pcifunc, + allocated, + req->entry); return NPC_FLOW_NOT_SUPPORTED; } @@ -1444,8 +1783,11 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, } process_flow: - if (from_vf && req->default_rule) + if (from_vf && req->default_rule) { + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); return NPC_FLOW_VF_PERM_DENIED; + } /* Each PF/VF info is maintained in struct rvu_pfvf. * rvu_pfvf for the target PF/VF needs to be retrieved @@ -1473,8 +1815,11 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, req->chan_mask = 0xFFF; err = npc_check_unsupported_flows(rvu, req->features, req->intf); - if (err) + if (err) { + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); return NPC_FLOW_NOT_SUPPORTED; + } pfvf = rvu_get_pfvf(rvu, target); @@ -1493,8 +1838,11 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, /* Proceed if NIXLF is attached or not for TX rules */ err = nix_get_nixlf(rvu, target, &nixlf, NULL); - if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac) + if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac) { + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); return NPC_FLOW_NO_NIXLF; + } /* don't enable rule when nixlf not attached or initialized */ if (!(is_nixlf_attached(rvu, target) && @@ -1509,20 +1857,31 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, enable = true; /* Do not allow requests from uninitialized VFs */ - if (from_vf && !enable) + if (from_vf && !enable) { + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); return NPC_FLOW_VF_NOT_INIT; + } /* PF sets VF mac & VF NIXLF is not attached, update the mac addr */ if (pf_set_vfs_mac && !enable) { ether_addr_copy(pfvf->default_mac, req->packet.dmac); ether_addr_copy(pfvf->mac_addr, req->packet.dmac); set_bit(PF_SET_VF_MAC, &pfvf->flags); + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); return 0; } mutex_lock(&rswitch->switch_lock); err = npc_install_flow(rvu, blkaddr, target, nixlf, pfvf, req, rsp, enable, pf_set_vfs_mac); + if (err) + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); + + rsp->kw_type = kw_type; + rsp->entry = req->entry; mutex_unlock(&rswitch->switch_lock); return err; @@ -1559,6 +1918,10 @@ int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu, struct list_head del_list; int blkaddr; + req->entry = npc_cn20k_vidx2idx(req->entry); + req->start = npc_cn20k_vidx2idx(req->start); + req->end = npc_cn20k_vidx2idx(req->end); + INIT_LIST_HEAD(&del_list); mutex_lock(&mcam->lock); @@ -1601,28 +1964,49 @@ static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr, struct rvu_npc_mcam_rule *rule, struct rvu_pfvf *pfvf) { + struct npc_cn20k_mcam_write_entry_req cn20k_wreq = { 0 }; struct npc_mcam_write_entry_req write_req = { 0 }; - struct mcam_entry *entry = &write_req.entry_data; + struct mcam_entry_mdata mdata = { }; struct npc_mcam *mcam = &rvu->hw->mcam; + struct cn20k_mcam_entry *cn20k_entry; + struct mcam_entry *entry; + u8 intf, enable, hw_prio; struct msg_rsp rsp; - u8 intf, enable; int err; + cn20k_entry = &cn20k_wreq.entry_data; + entry = &write_req.entry_data; + npc_populate_mcam_mdata(rvu, &mdata, cn20k_entry, entry); + ether_addr_copy(rule->packet.dmac, pfvf->mac_addr); - npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry, - entry, &intf, &enable); + if (is_cn20k(rvu->pdev)) + npc_cn20k_read_mcam_entry(rvu, npcblkaddr, rule->entry, + cn20k_entry, &intf, + &enable, &hw_prio); + else + npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry, + entry, &intf, &enable); - npc_update_entry(rvu, NPC_DMAC, entry, + npc_update_entry(rvu, NPC_DMAC, &mdata, ether_addr_to_u64(pfvf->mac_addr), 0, 0xffffffffffffull, 0, intf); - write_req.hdr.pcifunc = rule->owner; - write_req.entry = rule->entry; - write_req.intf = pfvf->nix_rx_intf; - mutex_unlock(&mcam->lock); - err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, &rsp); + if (is_cn20k(rvu->pdev)) { + cn20k_wreq.hdr.pcifunc = rule->owner; + cn20k_wreq.entry = rule->entry; + cn20k_wreq.intf = pfvf->nix_rx_intf; + err = rvu_mbox_handler_npc_cn20k_mcam_write_entry(rvu, + &cn20k_wreq, + &rsp); + } else { + write_req.hdr.pcifunc = rule->owner; + write_req.entry = rule->entry; + write_req.intf = pfvf->nix_rx_intf; + err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, + &rsp); + } mutex_lock(&mcam->lock); return err; @@ -1710,10 +2094,12 @@ int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, u64 chan_val, u64 chan_mask, u64 exact_val, u64 exact_mask, u64 bcast_mcast_val, u64 bcast_mcast_mask) { + struct npc_cn20k_mcam_write_entry_req cn20k_req = { 0 }; struct npc_mcam_alloc_counter_req cntr_req = { 0 }; struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; struct npc_mcam_write_entry_req req = { 0 }; struct npc_mcam *mcam = &rvu->hw->mcam; + struct mcam_entry_mdata mdata = { }; struct rvu_npc_mcam_rule *rule; struct msg_rsp rsp; bool enabled; @@ -1757,26 +2143,51 @@ int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, /* Reserve slot 0 */ npc_mcam_rsrcs_reserve(rvu, blkaddr, mcam_idx); - /* Allocate counter for this single drop on non hit rule */ - cntr_req.hdr.pcifunc = 0; /* AF request */ - cntr_req.contig = true; - cntr_req.count = 1; - err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); - if (err) { - dev_err(rvu->dev, "%s: Err to allocate cntr for drop rule (err=%d)\n", - __func__, err); - return -EFAULT; + if (!is_cn20k(rvu->pdev)) { + /* Allocate counter for this single drop on non hit rule */ + cntr_req.hdr.pcifunc = 0; /* AF request */ + cntr_req.contig = true; + cntr_req.count = 1; + err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, + &cntr_rsp); + if (err) { + dev_err(rvu->dev, + "%s: Err to allocate cntr for drop rule (err=%d)\n", + __func__, err); + return -EFAULT; + } + *counter_idx = cntr_rsp.cntr; } - *counter_idx = cntr_rsp.cntr; + + npc_populate_mcam_mdata(rvu, &mdata, + &cn20k_req.entry_data, + &req.entry_data); /* Fill in fields for this mcam entry */ - npc_update_entry(rvu, NPC_EXACT_RESULT, &req.entry_data, exact_val, 0, + npc_update_entry(rvu, NPC_EXACT_RESULT, &mdata, exact_val, 0, exact_mask, 0, NIX_INTF_RX); - npc_update_entry(rvu, NPC_CHAN, &req.entry_data, chan_val, 0, + npc_update_entry(rvu, NPC_CHAN, &mdata, chan_val, 0, chan_mask, 0, NIX_INTF_RX); - npc_update_entry(rvu, NPC_LXMB, &req.entry_data, bcast_mcast_val, 0, + npc_update_entry(rvu, NPC_LXMB, &mdata, bcast_mcast_val, 0, bcast_mcast_mask, 0, NIX_INTF_RX); + if (is_cn20k(rvu->pdev)) { + cn20k_req.intf = NIX_INTF_RX; + cn20k_req.entry = mcam_idx; + + err = rvu_mbox_handler_npc_cn20k_mcam_write_entry(rvu, + &cn20k_req, + &rsp); + if (err) { + dev_err(rvu->dev, + "%s: Installation of single drop on non hit rule at %d failed\n", + __func__, mcam_idx); + return err; + } + + goto enable_entry; + } + req.intf = NIX_INTF_RX; req.set_cntr = true; req.cntr = cntr_rsp.cntr; @@ -1784,14 +2195,17 @@ int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &req, &rsp); if (err) { - dev_err(rvu->dev, "%s: Installation of single drop on non hit rule at %d failed\n", + dev_err(rvu->dev, + "%s: Installation of single drop on non hit rule at %d failed\n", __func__, mcam_idx); return err; } - dev_err(rvu->dev, "%s: Installed single drop on non hit rule at %d, cntr=%d\n", + dev_err(rvu->dev, + "%s: Installed single drop on non hit rule at %d, cntr=%d\n", __func__, mcam_idx, req.cntr); +enable_entry: /* disable entry at Bank 0, index 0 */ npc_enable_mcam_entry(rvu, mcam, blkaddr, mcam_idx, false); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h index 3f5c9042d10e..d3ba86c23959 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h @@ -15,7 +15,19 @@ #define NPC_LDATA_EN BIT_ULL(7) void npc_update_entry(struct rvu *rvu, enum key_fields type, - struct mcam_entry *entry, u64 val_lo, + struct mcam_entry_mdata *mdata, u64 val_lo, u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf); +void npc_update_flow(struct rvu *rvu, struct mcam_entry_mdata *mdata, + u64 features, struct flow_msg *pkt, + struct flow_msg *mask, + struct rvu_npc_mcam_rule *output, u8 intf, + int blkaddr); + +void +npc_populate_mcam_mdata(struct rvu *rvu, + struct mcam_entry_mdata *mdata, + struct cn20k_mcam_entry *cn20k_entry, + struct mcam_entry *entry); + #endif /* RVU_NPC_FS_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c index 906d712cef19..03bb485a1aca 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c @@ -125,6 +125,9 @@ static void npc_program_mkex_hash_rx(struct rvu *rvu, int blkaddr, struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash; int lid, lt, ld, hash_cnt = 0; + if (is_cn20k(rvu->pdev)) + return; + if (is_npc_intf_tx(intf)) return; @@ -165,6 +168,9 @@ static void npc_program_mkex_hash_tx(struct rvu *rvu, int blkaddr, struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash; int lid, lt, ld, hash_cnt = 0; + if (is_cn20k(rvu->pdev)) + return; + if (is_npc_intf_rx(intf)) return; @@ -224,6 +230,9 @@ void npc_program_mkex_hash(struct rvu *rvu, int blkaddr) struct rvu_hwinfo *hw = rvu->hw; u64 cfg; + if (is_cn20k(rvu->pdev)) + return; + /* Check if hardware supports hash extraction */ if (!hwcap->npc_hash_extract) return; @@ -273,7 +282,7 @@ void npc_program_mkex_hash(struct rvu *rvu, int blkaddr) } void npc_update_field_hash(struct rvu *rvu, u8 intf, - struct mcam_entry *entry, + struct mcam_entry_mdata *mdata, int blkaddr, u64 features, struct flow_msg *pkt, @@ -284,9 +293,13 @@ void npc_update_field_hash(struct rvu *rvu, u8 intf, struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash; struct npc_get_field_hash_info_req req; struct npc_get_field_hash_info_rsp rsp; + u8 hash_idx, lid, ltype, ltype_mask; u64 ldata[2], cfg; u32 field_hash; - u8 hash_idx; + bool en; + + if (is_cn20k(rvu->pdev)) + return; if (!rvu->hw->cap.npc_hash_extract) { dev_dbg(rvu->dev, "%s: Field hash extract feature is not supported\n", __func__); @@ -298,60 +311,60 @@ void npc_update_field_hash(struct rvu *rvu, u8 intf, for (hash_idx = 0; hash_idx < NPC_MAX_HASH; hash_idx++) { cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_HASHX_CFG(intf, hash_idx)); - if ((cfg & BIT_ULL(11)) && (cfg & BIT_ULL(12))) { - u8 lid = (cfg & GENMASK_ULL(10, 8)) >> 8; - u8 ltype = (cfg & GENMASK_ULL(7, 4)) >> 4; - u8 ltype_mask = cfg & GENMASK_ULL(3, 0); + en = !!(cfg & BIT_ULL(11)) && (cfg & BIT_ULL(12)); + if (!en) + continue; - if (mkex_hash->lid_lt_ld_hash_en[intf][lid][ltype][hash_idx]) { - switch (ltype & ltype_mask) { - /* If hash extract enabled is supported for IPv6 then - * 128 bit IPv6 source and destination addressed - * is hashed to 32 bit value. - */ - case NPC_LT_LC_IP6: - /* ld[0] == hash_idx[0] == Source IPv6 - * ld[1] == hash_idx[1] == Destination IPv6 - */ - if ((features & BIT_ULL(NPC_SIP_IPV6)) && !hash_idx) { - u32 src_ip[IPV6_WORDS]; + lid = (cfg & GENMASK_ULL(10, 8)) >> 8; + ltype = (cfg & GENMASK_ULL(7, 4)) >> 4; + ltype_mask = cfg & GENMASK_ULL(3, 0); - be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS); - ldata[1] = (u64)src_ip[0] << 32 | src_ip[1]; - ldata[0] = (u64)src_ip[2] << 32 | src_ip[3]; - field_hash = npc_field_hash_calc(ldata, - rsp, - intf, - hash_idx); - npc_update_entry(rvu, NPC_SIP_IPV6, entry, - field_hash, 0, - GENMASK(31, 0), 0, intf); - memcpy(&opkt->ip6src, &pkt->ip6src, - sizeof(pkt->ip6src)); - memcpy(&omask->ip6src, &mask->ip6src, - sizeof(mask->ip6src)); - } else if ((features & BIT_ULL(NPC_DIP_IPV6)) && hash_idx) { - u32 dst_ip[IPV6_WORDS]; + if (!mkex_hash->lid_lt_ld_hash_en[intf][lid][ltype][hash_idx]) + continue; - be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS); - ldata[1] = (u64)dst_ip[0] << 32 | dst_ip[1]; - ldata[0] = (u64)dst_ip[2] << 32 | dst_ip[3]; - field_hash = npc_field_hash_calc(ldata, - rsp, - intf, - hash_idx); - npc_update_entry(rvu, NPC_DIP_IPV6, entry, - field_hash, 0, - GENMASK(31, 0), 0, intf); - memcpy(&opkt->ip6dst, &pkt->ip6dst, - sizeof(pkt->ip6dst)); - memcpy(&omask->ip6dst, &mask->ip6dst, - sizeof(mask->ip6dst)); - } + /* If hash extract enabled is supported for IPv6 then + * 128 bit IPv6 source and destination addressed + * is hashed to 32 bit value. + */ + if ((ltype & ltype_mask) != NPC_LT_LC_IP6) + continue; - break; - } - } + /* ld[0] == hash_idx[0] == Source IPv6 + * ld[1] == hash_idx[1] == Destination IPv6 + */ + if ((features & BIT_ULL(NPC_SIP_IPV6)) && !hash_idx) { + u32 src_ip[IPV6_WORDS]; + + be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS); + ldata[1] = (u64)src_ip[0] << 32 | src_ip[1]; + ldata[0] = (u64)src_ip[2] << 32 | src_ip[3]; + field_hash = npc_field_hash_calc(ldata, rsp, intf, + hash_idx); + npc_update_entry(rvu, NPC_SIP_IPV6, mdata, field_hash, + 0, GENMASK(31, 0), 0, intf); + memcpy(&opkt->ip6src, &pkt->ip6src, + sizeof(pkt->ip6src)); + memcpy(&omask->ip6src, &mask->ip6src, + sizeof(mask->ip6src)); + continue; + } + + if ((features & BIT_ULL(NPC_DIP_IPV6)) && hash_idx) { + u32 dst_ip[IPV6_WORDS]; + + be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS); + ldata[1] = (u64)dst_ip[0] << 32 | dst_ip[1]; + ldata[0] = (u64)dst_ip[2] << 32 | dst_ip[3]; + field_hash = npc_field_hash_calc(ldata, rsp, intf, + hash_idx); + npc_update_entry(rvu, NPC_DIP_IPV6, mdata, + field_hash, 0, GENMASK(31, 0), + 0, intf); + memcpy(&opkt->ip6dst, &pkt->ip6dst, + sizeof(pkt->ip6dst)); + memcpy(&omask->ip6dst, &mask->ip6dst, + sizeof(mask->ip6dst)); + continue; } } } @@ -1874,6 +1887,9 @@ int rvu_npc_exact_init(struct rvu *rvu) u64 cfg; bool rc; + if (is_cn20k(rvu->pdev)) + return 0; + /* Read NPC_AF_CONST3 and check for have exact * match functionality is present */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h index cb25cf478f1f..4cbcae69b6d3 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h @@ -53,7 +53,7 @@ struct npc_mcam_kex_hash { } __packed; void npc_update_field_hash(struct rvu *rvu, u8 intf, - struct mcam_entry *entry, + struct mcam_entry_mdata *mdata, int blkaddr, u64 features, struct flow_msg *pkt, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c index a60f8cf53feb..a5a8f4558717 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c @@ -251,6 +251,272 @@ static u8 cn20k_aura_bpid_idx(struct otx2_nic *pfvf, int aura_id) #endif } +static int cn20k_tc_get_entry_index(struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) +{ + struct otx2_tc_flow *tmp; + int index = 0; + + list_for_each_entry(tmp, &flow_cfg->flow_list_tc, list) { + if (tmp == node) + return index; + + index++; + } + + return -1; +} + +int cn20k_tc_free_mcam_entry(struct otx2_nic *nic, u16 entry) +{ + struct npc_mcam_free_entry_req *req; + int err; + + mutex_lock(&nic->mbox.lock); + req = otx2_mbox_alloc_msg_npc_mcam_free_entry(&nic->mbox); + if (!req) { + mutex_unlock(&nic->mbox.lock); + return -ENOMEM; + } + + req->entry = entry; + /* Send message to AF to free MCAM entries */ + err = otx2_sync_mbox_msg(&nic->mbox); + if (err) { + mutex_unlock(&nic->mbox.lock); + return err; + } + + mutex_unlock(&nic->mbox.lock); + + return 0; +} + +static bool cn20k_tc_check_entry_shiftable(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node, int index, + bool error) +{ + struct otx2_tc_flow *first, *tmp, *n; + u32 prio = 0; + int i = 0; + u8 type; + + first = list_first_entry(&flow_cfg->flow_list_tc, struct otx2_tc_flow, + list); + type = first->kw_type; + + /* Check all the nodes from start to given index (including index) has + * same type i.e, either X2 or X4 + */ + list_for_each_entry_safe(tmp, n, &flow_cfg->flow_list_tc, list) { + if (i > index) + break; + + if (type != tmp->kw_type) { + /* List has both X2 and X4 entries so entries cannot be + * shifted to save MCAM space. + */ + if (error) + dev_err(nic->dev, "Rule %d cannot be shifted to %d\n", + tmp->prio, prio); + return false; + } + + type = tmp->kw_type; + prio = tmp->prio; + i++; + } + + return true; +} + +void cn20k_tc_update_mcam_table_del_req(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) +{ + struct otx2_tc_flow *first, *tmp, *n; + int i = 0, index; + u16 cntr_val = 0; + u16 entry; + + index = cn20k_tc_get_entry_index(flow_cfg, node); + if (index < 0) { + netdev_dbg(nic->netdev, "Could not find node\n"); + return; + } + + first = list_first_entry(&flow_cfg->flow_list_tc, struct otx2_tc_flow, + list); + entry = first->entry; + + /* If entries cannot be shifted then delete given entry + * and free it to AF too. + */ + if (!cn20k_tc_check_entry_shiftable(nic, flow_cfg, node, + index, false)) { + list_del(&node->list); + entry = node->entry; + goto free_mcam_entry; + } + + /* Find and delete the entry from the list and re-install + * all the entries from beginning to the index of the + * deleted entry to higher mcam indexes. + */ + list_for_each_entry_safe(tmp, n, &flow_cfg->flow_list_tc, list) { + if (node == tmp) { + list_del(&tmp->list); + break; + } + + otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val); + tmp->entry = (list_next_entry(tmp, list))->entry; + tmp->req.entry = tmp->entry; + tmp->req.cntr_val = cntr_val; + } + + list_for_each_entry_safe(tmp, n, &flow_cfg->flow_list_tc, list) { + if (i == index) + break; + + otx2_add_mcam_flow_entry(nic, &tmp->req); + i++; + } + +free_mcam_entry: + if (cn20k_tc_free_mcam_entry(nic, entry)) + netdev_err(nic->netdev, "Freeing entry %d to AF failed\n", + entry); +} + +int cn20k_tc_update_mcam_table_add_req(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) +{ + struct otx2_tc_flow *tmp; + u16 cntr_val = 0; + int list_idx, i; + int entry, prev; + + /* Find the index of the entry(list_idx) whose priority + * is greater than the new entry and re-install all + * the entries from beginning to list_idx to higher + * mcam indexes. + */ + list_idx = otx2_tc_add_to_flow_list(flow_cfg, node); + entry = node->entry; + if (!cn20k_tc_check_entry_shiftable(nic, flow_cfg, node, + list_idx, true)) + /* Due to mix of X2 and X4, entries cannot be shifted. + * In this case free the entry allocated for this rule. + */ + return -EINVAL; + + for (i = 0; i < list_idx; i++) { + tmp = otx2_tc_get_entry_by_index(flow_cfg, i); + if (!tmp) + return -ENOMEM; + + otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val); + prev = tmp->entry; + tmp->entry = entry; + tmp->req.entry = tmp->entry; + tmp->req.cntr_val = cntr_val; + otx2_add_mcam_flow_entry(nic, &tmp->req); + entry = prev; + } + + return entry; +} + +#define MAX_TC_HW_PRIORITY 125 +#define MAX_TC_VF_PRIORITY 126 +#define MAX_TC_PF_PRIORITY 127 + +static int __cn20k_tc_alloc_entry(struct otx2_nic *nic, + struct npc_install_flow_req *flow_req, + u16 *entry, u8 *type, + u32 tc_priority, bool hw_priority) +{ + struct otx2_flow_config *flow_cfg = nic->flow_cfg; + struct npc_install_flow_req *req; + struct npc_install_flow_rsp *rsp; + struct otx2_tc_flow *tmp; + int ret = 0; + + req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); + if (!req) + return -ENOMEM; + + memcpy(&flow_req->hdr, &req->hdr, sizeof(struct mbox_msghdr)); + memcpy(req, flow_req, sizeof(struct npc_install_flow_req)); + req->alloc_entry = 1; + + /* Allocate very least priority for first rule */ + if (hw_priority || list_empty(&flow_cfg->flow_list_tc)) { + req->ref_prio = NPC_MCAM_LEAST_PRIO; + } else { + req->ref_prio = NPC_MCAM_HIGHER_PRIO; + tmp = list_first_entry(&flow_cfg->flow_list_tc, + struct otx2_tc_flow, list); + req->ref_entry = tmp->entry; + } + + ret = otx2_sync_mbox_msg(&nic->mbox); + if (ret) + return ret; + + rsp = (struct npc_install_flow_rsp *)otx2_mbox_get_rsp(&nic->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) + return -EFAULT; + + if (entry) + *entry = rsp->entry; + if (type) + *type = rsp->kw_type; + + return ret; +} + +int cn20k_tc_alloc_entry(struct otx2_nic *nic, + struct flow_cls_offload *tc_flow_cmd, + struct otx2_tc_flow *new_node, + struct npc_install_flow_req *flow_req) +{ + bool hw_priority = false; + u16 entry_from_af; + u8 entry_type; + int ret; + + if (is_otx2_vf(nic->pcifunc)) + flow_req->hw_prio = MAX_TC_VF_PRIORITY; + else + flow_req->hw_prio = MAX_TC_PF_PRIORITY; + + if (new_node->prio <= MAX_TC_HW_PRIORITY) { + flow_req->hw_prio = new_node->prio; + hw_priority = true; + } + + mutex_lock(&nic->mbox.lock); + + ret = __cn20k_tc_alloc_entry(nic, flow_req, &entry_from_af, &entry_type, + new_node->prio, hw_priority); + if (ret) { + mutex_unlock(&nic->mbox.lock); + return ret; + } + + new_node->kw_type = entry_type; + new_node->entry = entry_from_af; + + mutex_unlock(&nic->mbox.lock); + + return 0; +} + static int cn20k_aura_aq_init(struct otx2_nic *pfvf, int aura_id, int pool_id, int numptrs) { diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h index 832adaf8c57f..b5e527f6d7eb 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h @@ -10,8 +10,22 @@ #include "otx2_common.h" +struct otx2_flow_config; +struct otx2_tc_flow; + void cn20k_init(struct otx2_nic *pfvf); int cn20k_register_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs); void cn20k_disable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs); void cn20k_enable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs); +void cn20k_tc_update_mcam_table_del_req(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node); +int cn20k_tc_update_mcam_table_add_req(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node); +int cn20k_tc_alloc_entry(struct otx2_nic *nic, + struct flow_cls_offload *tc_flow_cmd, + struct otx2_tc_flow *new_node, + struct npc_install_flow_req *dummy); +int cn20k_tc_free_mcam_entry(struct otx2_nic *nic, u16 entry); #endif /* CN20K_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index 255c7e2633bb..eecee612b7b2 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -366,6 +366,31 @@ struct otx2_flow_config { u16 ntuple_cnt; }; +struct otx2_tc_flow_stats { + u64 bytes; + u64 pkts; + u64 used; +}; + +struct otx2_tc_flow { + struct list_head list; + unsigned long cookie; + struct rcu_head rcu; + struct otx2_tc_flow_stats stats; + spinlock_t lock; /* lock for stats */ + u16 rq; + u16 entry; + u16 leaf_profile; + bool is_act_police; + u32 prio; + struct npc_install_flow_req req; + u64 rate; + u32 burst; + u32 mcast_grp_idx; + bool is_pps; + u8 kw_type; /* X2/X4 */ +}; + struct dev_hw_ops { int (*sq_aq_init)(void *dev, u16 qidx, u8 chan_offset, u16 sqb_aura); @@ -1223,4 +1248,14 @@ void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg); int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx); void otx2_queue_vf_work(struct mbox *mw, struct workqueue_struct *mbox_wq, int first, int mdevs, u64 intr); +int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry, + u16 *cntr_val); +int otx2_add_mcam_flow_entry(struct otx2_nic *nic, + struct npc_install_flow_req *req); +int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node); + +struct otx2_tc_flow * +otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg, + int index); #endif /* OTX2_COMMON_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c index 12c001ee34e2..38cc539d724d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c @@ -37,6 +37,98 @@ static void otx2_clear_ntuple_flow_info(struct otx2_nic *pfvf, struct otx2_flow_ flow_cfg->max_flows = 0; } +static int otx2_mcam_pfl_info_get(struct otx2_nic *pfvf, bool *is_x2, + u16 *x4_slots) +{ + struct npc_get_pfl_info_rsp *rsp; + struct msg_req *req; + static struct { + bool is_set; + bool is_x2; + u16 x4_slots; + } pfl_info; + + /* Avoid sending mboxes for constant information + * like x4_slots + */ + mutex_lock(&pfvf->mbox.lock); + if (pfl_info.is_set) { + *is_x2 = pfl_info.is_x2; + *x4_slots = pfl_info.x4_slots; + mutex_unlock(&pfvf->mbox.lock); + return 0; + } + + req = otx2_mbox_alloc_msg_npc_get_pfl_info(&pfvf->mbox); + if (!req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + /* Send message to AF */ + if (otx2_sync_mbox_msg(&pfvf->mbox)) { + mutex_unlock(&pfvf->mbox.lock); + return -EFAULT; + } + + rsp = (struct npc_get_pfl_info_rsp *)otx2_mbox_get_rsp + (&pfvf->mbox.mbox, 0, &req->hdr); + + if (IS_ERR(rsp)) { + mutex_unlock(&pfvf->mbox.lock); + return -EFAULT; + } + + *is_x2 = (rsp->kw_type == NPC_MCAM_KEY_X2); + if (*is_x2) + *x4_slots = 0; + else + *x4_slots = rsp->x4_slots; + + pfl_info.is_x2 = *is_x2; + pfl_info.x4_slots = *x4_slots; + pfl_info.is_set = true; + + mutex_unlock(&pfvf->mbox.lock); + return 0; +} + +static int otx2_get_dft_rl_idx(struct otx2_nic *pfvf, u16 *mcam_idx) +{ + struct npc_get_dft_rl_idxs_rsp *rsp; + struct msg_req *req; + + mutex_lock(&pfvf->mbox.lock); + + req = otx2_mbox_alloc_msg_npc_get_dft_rl_idxs(&pfvf->mbox); + if (!req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + /* Send message to AF */ + if (otx2_sync_mbox_msg(&pfvf->mbox)) { + mutex_unlock(&pfvf->mbox.lock); + return -EINVAL; + } + + rsp = (struct npc_get_dft_rl_idxs_rsp *)otx2_mbox_get_rsp + (&pfvf->mbox.mbox, 0, &req->hdr); + + if (IS_ERR(rsp)) { + mutex_unlock(&pfvf->mbox.lock); + return -EFAULT; + } + + if (is_otx2_lbkvf(pfvf->pdev)) + *mcam_idx = rsp->promisc; + else + *mcam_idx = rsp->ucast; + + mutex_unlock(&pfvf->mbox.lock); + return 0; +} + static int otx2_free_ntuple_mcam_entries(struct otx2_nic *pfvf) { struct otx2_flow_config *flow_cfg = pfvf->flow_cfg; @@ -69,7 +161,10 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count) struct otx2_flow_config *flow_cfg = pfvf->flow_cfg; struct npc_mcam_alloc_entry_req *req; struct npc_mcam_alloc_entry_rsp *rsp; - int ent, allocated = 0; + u16 dft_idx = 0, x4_slots = 0; + int ent, allocated = 0, ref; + bool is_x2 = false; + int rc; /* Free current ones and allocate new ones with requested count */ otx2_free_ntuple_mcam_entries(pfvf); @@ -86,6 +181,22 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count) return -ENOMEM; } + if (is_cn20k(pfvf->pdev)) { + rc = otx2_mcam_pfl_info_get(pfvf, &is_x2, &x4_slots); + if (rc) { + netdev_err(pfvf->netdev, "Error to retrieve profile info\n"); + return rc; + } + + rc = otx2_get_dft_rl_idx(pfvf, &dft_idx); + if (rc) { + netdev_err(pfvf->netdev, + "Error to retrieve ucast mcam idx for pcifunc %#x\n", + pfvf->pcifunc); + return rc; + } + } + mutex_lock(&pfvf->mbox.lock); /* In a single request a max of NPC_MAX_NONCONTIG_ENTRIES MCAM entries @@ -96,18 +207,31 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count) if (!req) goto exit; + req->kw_type = is_x2 ? NPC_MCAM_KEY_X2 : NPC_MCAM_KEY_X4; req->contig = false; req->count = (count - allocated) > NPC_MAX_NONCONTIG_ENTRIES ? NPC_MAX_NONCONTIG_ENTRIES : count - allocated; + ref = 0; + + if (is_cn20k(pfvf->pdev)) { + req->ref_prio = NPC_MCAM_HIGHER_PRIO; + ref = dft_idx; + } + /* Allocate higher priority entries for PFs, so that VF's entries * will be on top of PF. */ if (!is_otx2_vf(pfvf->pcifunc)) { - req->priority = NPC_MCAM_HIGHER_PRIO; - req->ref_entry = flow_cfg->def_ent[0]; + req->ref_prio = NPC_MCAM_HIGHER_PRIO; + ref = flow_cfg->def_ent[0]; } + if (is_cn20k(pfvf->pdev)) + ref = is_x2 ? ref : ref & (x4_slots - 1); + + req->ref_entry = ref; + /* Send message to AF */ if (otx2_sync_mbox_msg(&pfvf->mbox)) goto exit; @@ -163,8 +287,24 @@ int otx2_mcam_entry_init(struct otx2_nic *pfvf) struct npc_get_field_status_rsp *frsp; struct npc_mcam_alloc_entry_req *req; struct npc_mcam_alloc_entry_rsp *rsp; - int vf_vlan_max_flows; - int ent, count; + int vf_vlan_max_flows, count; + int rc, ref, prio, ent; + u16 dft_idx; + + ref = 0; + prio = 0; + if (is_cn20k(pfvf->pdev)) { + rc = otx2_get_dft_rl_idx(pfvf, &dft_idx); + if (rc) { + netdev_err(pfvf->netdev, + "Error to retrieve ucast mcam idx for pcifunc %#x\n", + pfvf->pcifunc); + return rc; + } + + ref = dft_idx; + prio = NPC_MCAM_HIGHER_PRIO; + } vf_vlan_max_flows = pfvf->total_vfs * OTX2_PER_VF_VLAN_FLOWS; count = flow_cfg->ucast_flt_cnt + @@ -183,8 +323,11 @@ int otx2_mcam_entry_init(struct otx2_nic *pfvf) return -ENOMEM; } + req->kw_type = NPC_MCAM_KEY_X2; req->contig = false; req->count = count; + req->ref_prio = prio; + req->ref_entry = ref; /* Send message to AF */ if (otx2_sync_mbox_msg(&pfvf->mbox)) { @@ -819,7 +962,7 @@ static int otx2_prepare_ipv6_flow(struct ethtool_rx_flow_spec *fsp, } static int otx2_prepare_flow_request(struct ethtool_rx_flow_spec *fsp, - struct npc_install_flow_req *req) + struct npc_install_flow_req *req) { struct ethhdr *eth_mask = &fsp->m_u.ether_spec; struct ethhdr *eth_hdr = &fsp->h_u.ether_spec; @@ -945,6 +1088,58 @@ static int otx2_prepare_flow_request(struct ethtool_rx_flow_spec *fsp, return 0; } +static int otx2_get_kw_type(struct otx2_nic *pfvf, + struct npc_install_flow_req *fl_req, + u8 *kw_type) +{ + struct npc_get_num_kws_req *req; + struct npc_get_num_kws_rsp *rsp; + u8 *src, *dst; + int off, err; + int kw_bits; + + off = offsetof(struct npc_install_flow_req, packet); + + mutex_lock(&pfvf->mbox.lock); + + req = otx2_mbox_alloc_msg_npc_get_num_kws(&pfvf->mbox); + if (!req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + dst = (u8 *)&req->fl + off; + src = (u8 *)fl_req + off; + + memcpy(dst, src, sizeof(struct npc_install_flow_req) - off); + + err = otx2_sync_mbox_msg(&pfvf->mbox); + if (err) { + mutex_unlock(&pfvf->mbox.lock); + netdev_err(pfvf->netdev, + "Error to get default number of keywords\n"); + return err; + } + + rsp = (struct npc_get_num_kws_rsp *)otx2_mbox_get_rsp + (&pfvf->mbox.mbox, 0, &req->hdr); + if (IS_ERR(rsp)) { + mutex_unlock(&pfvf->mbox.lock); + return -EFAULT; + } + + kw_bits = rsp->kws * 64; + + if (kw_bits <= 256) + *kw_type = NPC_MCAM_KEY_X2; + else + *kw_type = NPC_MCAM_KEY_X4; + + mutex_unlock(&pfvf->mbox.lock); + + return 0; +} + static int otx2_is_flow_rule_dmacfilter(struct otx2_nic *pfvf, struct ethtool_rx_flow_spec *fsp) { @@ -973,12 +1168,41 @@ static int otx2_is_flow_rule_dmacfilter(struct otx2_nic *pfvf, static int otx2_add_flow_msg(struct otx2_nic *pfvf, struct otx2_flow *flow) { + struct otx2_flow_config *flow_cfg = pfvf->flow_cfg; + struct npc_install_flow_req *req, treq = { 0 }; u64 ring_cookie = flow->flow_spec.ring_cookie; #ifdef CONFIG_DCB int vlan_prio, qidx, pfc_rule = 0; #endif - struct npc_install_flow_req *req; - int err, vf = 0; + int err, vf = 0, off, sz; + bool modify = false; + u8 kw_type = 0; + u8 *src, *dst; + u16 x4_slots; + bool is_x2; + + if (is_cn20k(pfvf->pdev)) { + err = otx2_mcam_pfl_info_get(pfvf, &is_x2, &x4_slots); + if (err) { + netdev_err(pfvf->netdev, + "Error to retrieve NPC profile info, pcifunc=%#x\n", + pfvf->pcifunc); + return -EFAULT; + } + + if (!is_x2) { + err = otx2_prepare_flow_request(&flow->flow_spec, + &treq); + if (err) + return err; + + err = otx2_get_kw_type(pfvf, &treq, &kw_type); + if (err) + return err; + + modify = true; + } + } mutex_lock(&pfvf->mbox.lock); req = otx2_mbox_alloc_msg_npc_install_flow(&pfvf->mbox); @@ -987,14 +1211,29 @@ static int otx2_add_flow_msg(struct otx2_nic *pfvf, struct otx2_flow *flow) return -ENOMEM; } - err = otx2_prepare_flow_request(&flow->flow_spec, req); - if (err) { - /* free the allocated msg above */ - otx2_mbox_reset(&pfvf->mbox.mbox, 0); - mutex_unlock(&pfvf->mbox.lock); - return err; + if (modify) { + off = offsetof(struct npc_install_flow_req, packet); + sz = sizeof(struct npc_install_flow_req) - off; + dst = (u8 *)req + off; + src = (u8 *)&treq + off; + + memcpy(dst, src, sz); + req->req_kw_type = kw_type; + } else { + err = otx2_prepare_flow_request(&flow->flow_spec, req); + if (err) { + /* free the allocated msg above */ + otx2_mbox_reset(&pfvf->mbox.mbox, 0); + mutex_unlock(&pfvf->mbox.lock); + return err; + } } + netdev_dbg(pfvf->netdev, + "flow entry (%u) installed at loc:%u kw_type=%u\n", + flow_cfg->flow_ent[flow->location], + flow->location, kw_type); + req->entry = flow->entry; req->intf = NIX_INTF_RX; req->set_cntr = 1; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c index 04fcfbdcf69b..40162b08014d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c @@ -31,30 +31,6 @@ #define MCAST_INVALID_GRP (-1U) -struct otx2_tc_flow_stats { - u64 bytes; - u64 pkts; - u64 used; -}; - -struct otx2_tc_flow { - struct list_head list; - unsigned long cookie; - struct rcu_head rcu; - struct otx2_tc_flow_stats stats; - spinlock_t lock; /* lock for stats */ - u16 rq; - u16 entry; - u16 leaf_profile; - bool is_act_police; - u32 prio; - struct npc_install_flow_req req; - u32 mcast_grp_idx; - u64 rate; - u32 burst; - bool is_pps; -}; - static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst, u32 *burst_exp, u32 *burst_mantissa) { @@ -971,8 +947,9 @@ static void otx2_destroy_tc_flow_list(struct otx2_nic *pfvf) } } -static struct otx2_tc_flow *otx2_tc_get_entry_by_cookie(struct otx2_flow_config *flow_cfg, - unsigned long cookie) +static struct otx2_tc_flow * +otx2_tc_get_entry_by_cookie(struct otx2_flow_config *flow_cfg, + unsigned long cookie) { struct otx2_tc_flow *tmp; @@ -984,8 +961,8 @@ static struct otx2_tc_flow *otx2_tc_get_entry_by_cookie(struct otx2_flow_config return NULL; } -static struct otx2_tc_flow *otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg, - int index) +struct otx2_tc_flow * +otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg, int index) { struct otx2_tc_flow *tmp; int i = 0; @@ -1014,8 +991,8 @@ static void otx2_tc_del_from_flow_list(struct otx2_flow_config *flow_cfg, } } -static int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg, - struct otx2_tc_flow *node) +int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) { struct list_head *pos, *n; struct otx2_tc_flow *tmp; @@ -1038,7 +1015,8 @@ static int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg, return index; } -static int otx2_add_mcam_flow_entry(struct otx2_nic *nic, struct npc_install_flow_req *req) +int otx2_add_mcam_flow_entry(struct otx2_nic *nic, + struct npc_install_flow_req *req) { struct npc_install_flow_req *tmp_req; int err; @@ -1064,7 +1042,7 @@ static int otx2_add_mcam_flow_entry(struct otx2_nic *nic, struct npc_install_flo return 0; } -static int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry, u16 *cntr_val) +int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry, u16 *cntr_val) { struct npc_delete_flow_rsp *rsp; struct npc_delete_flow_req *req; @@ -1114,6 +1092,11 @@ static int otx2_tc_update_mcam_table_del_req(struct otx2_nic *nic, int i = 0, index = 0; u16 cntr_val = 0; + if (is_cn20k(nic->pdev)) { + cn20k_tc_update_mcam_table_del_req(nic, flow_cfg, node); + return 0; + } + /* Find and delete the entry from the list and re-install * all the entries from beginning to the index of the * deleted entry to higher mcam indexes. @@ -1153,6 +1136,9 @@ static int otx2_tc_update_mcam_table_add_req(struct otx2_nic *nic, int list_idx, i; u16 cntr_val = 0; + if (is_cn20k(nic->pdev)) + return cn20k_tc_update_mcam_table_add_req(nic, flow_cfg, node); + /* Find the index of the entry(list_idx) whose priority * is greater than the new entry and re-install all * the entries from beginning to list_idx to higher @@ -1172,7 +1158,7 @@ static int otx2_tc_update_mcam_table_add_req(struct otx2_nic *nic, mcam_idx++; } - return mcam_idx; + return flow_cfg->flow_ent[mcam_idx]; } static int otx2_tc_update_mcam_table(struct otx2_nic *nic, @@ -1238,7 +1224,6 @@ static int otx2_tc_del_flow(struct otx2_nic *nic, mutex_unlock(&nic->mbox.lock); } - free_mcam_flow: otx2_del_mcam_flow_entry(nic, flow_node->entry, NULL); otx2_tc_update_mcam_table(nic, flow_cfg, flow_node, false); @@ -1254,7 +1239,7 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, struct otx2_flow_config *flow_cfg = nic->flow_cfg; struct otx2_tc_flow *new_node, *old_node; struct npc_install_flow_req *req, dummy; - int rc, err, mcam_idx; + int rc, err, entry; if (!(nic->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)) return -ENOMEM; @@ -1264,7 +1249,7 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, return -EINVAL; } - if (flow_cfg->nr_flows == flow_cfg->max_flows) { + if (!is_cn20k(nic->pdev) && flow_cfg->nr_flows == flow_cfg->max_flows) { NL_SET_ERR_MSG_MOD(extack, "Free MCAM entry not available to add the flow"); return -ENOMEM; @@ -1292,7 +1277,23 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, if (old_node) otx2_tc_del_flow(nic, tc_flow_cmd); - mcam_idx = otx2_tc_update_mcam_table(nic, flow_cfg, new_node, true); + if (is_cn20k(nic->pdev)) { + rc = cn20k_tc_alloc_entry(nic, tc_flow_cmd, new_node, &dummy); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, + "MCAM rule allocation failed"); + kfree_rcu(new_node, rcu); + return rc; + } + } + + entry = otx2_tc_update_mcam_table(nic, flow_cfg, new_node, true); + if (entry < 0) { + NL_SET_ERR_MSG_MOD(extack, "Adding rule failed"); + rc = entry; + goto free_leaf; + } + mutex_lock(&nic->mbox.lock); req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); if (!req) { @@ -1304,7 +1305,7 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, memcpy(&dummy.hdr, &req->hdr, sizeof(struct mbox_msghdr)); memcpy(req, &dummy, sizeof(struct npc_install_flow_req)); req->channel = nic->hw.rx_chan_base; - req->entry = flow_cfg->flow_ent[mcam_idx]; + req->entry = (u16)entry; req->intf = NIX_INTF_RX; req->vf = nic->pcifunc; req->set_cntr = 1; @@ -1325,6 +1326,8 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, return 0; free_leaf: + if (is_cn20k(nic->pdev)) + cn20k_tc_free_mcam_entry(nic, new_node->entry); otx2_tc_del_from_flow_list(flow_cfg, new_node); if (new_node->is_act_police) { mutex_lock(&nic->mbox.lock);