github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 09:36:22 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'pci/endpoint'

Browse Source 
- Add endpoint controller APIs for use by function drivers to discover
  auxiliary blocks like DMA engines (Koichiro Den)

- Remember DesignWare eDMA engine base/size and expose them via the EPC
  aux-resource API (Koichiro Den)

- Refactor endpoint doorbell allocation to allow non-MSI doorbells
  (Koichiro Den)

- Add endpoint embedded doorbell fallback, used if MSI allocation fails
  (Koichiro Den)

- Validate BAR index and remove dead BAR read in endpoint doorbell test
  (Carlos Bilbao)

- Unwind MSI/MSI-X vectors if NTB initialization fails part-way through
  (Koichiro Den)

- Cache sleepable pci_irq_vector() value at ISR setup to avoid calling it
  from hardirq context (Koichiro Den)

- Validate doorbell count when configuring NTB and vNTB doorbells
  (Manivannan Sadhasivam)

- Call sleepable pci_epc_raise_irq() from a work item instead of atomic
  context, e.g., when setting bits in NTB peer doorbells in the
  ntb_peer_db_set() path (Koichiro Den)

- Report 0-based vNTB doorbell vector to account for link event 0 and
  historically skipped slot 1 (Koichiro Den)

- Reject unusable vNTB doorbell counts, e.g., if they don't allow space for
  link event 0 and historically skipped slot 1 (Koichiro Den)

- Prevent configfs writes to vNTB db_count and other values that are
  already in use after EPC attach (Koichiro Den)

- Account for vNTB db_valid reserved slots (link event 0 and historically
  skipped slot 1) so they don't appear as valid doorbells (Koichiro Den)

- Implement vNTB .db_vector_count()/mask() for doorbells so clients can use
  multiple vectors and avoid thundering herds (Koichiro Den)

- Report 0-based NTB doorbell vector to account for link event 0 and
  historically skipped slot 1 (Koichiro Den)

- Fix doorbell bitmask and IRQ vector handling to clear only specified
  bits, use the correct vector for non-contiguous Linux IRQ numbers, and
  validate incoming vectors (Koichiro Den)

- Implement NTB .db_vector_count()/mask() for doorbells so clients can use
  multiple vectors (Koichiro Den)

* pci/endpoint:
  NTB: epf: Implement .db_vector_count()/mask() for doorbells
  NTB: epf: Fix doorbell bitmask and IRQ vector handling
  NTB: epf: Report 0-based doorbell vector via ntb_db_event()
  NTB: epf: Make db_valid_mask cover only real doorbell bits
  NTB: epf: Document legacy doorbell slot offset in ntb_epf_peer_db_set()
  PCI: endpoint: pci-epf-vntb: Implement .db_vector_count()/mask() for doorbells
  PCI: endpoint: pci-epf-vntb: Exclude reserved slots from db_valid_mask
  PCI: endpoint: pci-epf-vntb: Guard configfs writes after EPC attach
  PCI: endpoint: pci-epf-vntb: Reject unusable doorbell counts
  PCI: endpoint: pci-epf-vntb: Report 0-based doorbell vector via ntb_db_event()
  PCI: endpoint: pci-epf-vntb: Defer pci_epc_raise_irq() out of atomic context
  PCI: endpoint: pci-epf-vntb: Document legacy MSI doorbell offset
  PCI: endpoint: pci-epf-ntb: Add check to detect 'db_count' value of 0
  PCI: endpoint: pci-epf-vntb: Add check to detect 'db_count' value of 0
  NTB: epf: Avoid calling pci_irq_vector() from hardirq context
  NTB: epf: Fix request_irq() unwind in ntb_epf_init_isr()
  misc: pci_endpoint_test: Remove dead BAR read before doorbell trigger
  misc: pci_endpoint_test: Validate BAR index in doorbell test
  PCI: endpoint: pci-ep-msi: Add embedded doorbell fallback
  PCI: endpoint: pci-epf-test: Reuse pre-exposed doorbell targets
  PCI: endpoint: pci-epf-vntb: Reuse pre-exposed doorbells and IRQ flags
  PCI: endpoint: pci-ep-msi: Refactor doorbell allocation for new backends
  PCI: dwc: ep: Expose integrated eDMA resources via EPC aux-resource API
  PCI: dwc: Record integrated eDMA register window
  PCI: endpoint: Add auxiliary resource query API
This commit is contained in:
Bjorn Helgaas 2026-06-23 17:32:04 -05:00
commit ae385ca812
12 changed files with 883 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
drivers/misc/pci_endpoint_test.c
View File

@ -1100,7 +1100,6 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test)
data = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_DB_DATA);
addr = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_DB_OFFSET);
bar = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_DB_BAR);
pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
@ -1108,6 +1107,11 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test)
pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_STATUS, 0);
bar = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_DB_BAR);
if (bar < BAR_0 || bar >= PCI_STD_NUM_BARS) {
dev_err(dev, "BAR %d reported by endpoint out of range [0, %u]\n",
bar, PCI_STD_NUM_BARS - 1);
return -ERANGE;
}
writel(data, test->bar[bar] + addr);

141
drivers/ntb/hw/epf/ntb_hw_epf.c
View File

@ -6,6 +6,7 @@
* Author: Kishon Vijay Abraham I <kishon@ti.com>
*/
#include <linux/atomic.h>
#include <linux/delay.h>
#include <linux/module.h>
#include <linux/pci.h>
@ -43,6 +44,18 @@
#define NTB_EPF_DB_DATA(n) (0x34 + (n) * 4)
#define NTB_EPF_DB_OFFSET(n) (0xB4 + (n) * 4)
/*
* Legacy doorbell slot layout when paired with pci-epf-*ntb:
*
* slot 0 : reserved for link events
* slot 1 : unused (historical extra offset)
* slot 2 : DB#0
* slot 3 : DB#1
* ...
*
* Thus, NTB_EPF_MIN_DB_COUNT=3 means that we at least create vectors for
* doorbells DB#0 and DB#1.
*/
#define NTB_EPF_MIN_DB_COUNT 3
#define NTB_EPF_MAX_DB_COUNT 31
@ -69,8 +82,21 @@ enum epf_ntb_bar {
NTB_BAR_NUM,
};
enum epf_irq_slot {
EPF_IRQ_LINK = 0,
EPF_IRQ_RESERVED_DB, /* Historically skipped slot */
EPF_IRQ_DB_START,
};
#define NTB_EPF_MAX_MW_COUNT (NTB_BAR_NUM - BAR_MW1)
struct ntb_epf_dev;
struct ntb_epf_irq_ctx {
struct ntb_epf_dev *ndev;
unsigned int irq_no;
};
struct ntb_epf_dev {
struct ntb_dev ntb;
struct device *dev;
@ -90,8 +116,9 @@ struct ntb_epf_dev {
unsigned int self_spad;
unsigned int peer_spad;
int db_val;
atomic64_t db_val;
u64 db_valid_mask;
struct ntb_epf_irq_ctx irq_ctx[NTB_EPF_MAX_DB_COUNT + 1];
};
#define ntb_ndev(__ntb) container_of(__ntb, struct ntb_epf_dev, ntb)
@ -315,16 +342,29 @@ static int ntb_epf_link_disable(struct ntb_dev *ntb)
static irqreturn_t ntb_epf_vec_isr(int irq, void *dev)
{
struct ntb_epf_dev *ndev = dev;
int irq_no;
struct ntb_epf_irq_ctx *ctx = dev;
struct ntb_epf_dev *ndev = ctx->ndev;
unsigned int db_vector;
unsigned int irq_no = ctx->irq_no;
irq_no = irq - pci_irq_vector(ndev->ntb.pdev, 0);
ndev->db_val = irq_no + 1;
if (irq_no == 0)
if (irq_no == EPF_IRQ_LINK) {
ntb_link_event(&ndev->ntb);
else
ntb_db_event(&ndev->ntb, irq_no);
} else if (irq_no == EPF_IRQ_RESERVED_DB) {
dev_warn_ratelimited(ndev->dev,
"Unexpected reserved doorbell slot IRQ received\n");
} else {
db_vector = irq_no - EPF_IRQ_DB_START;
if (ndev->db_count < NTB_EPF_MIN_DB_COUNT ||
db_vector >= ndev->db_count - 1) {
dev_warn_ratelimited(ndev->dev,
"Unexpected doorbell vector %u (db_count %u)\n",
db_vector, ndev->db_count);
return IRQ_HANDLED;
}
atomic64_or(BIT_ULL(db_vector), &ndev->db_val);
ntb_db_event(&ndev->ntb, db_vector);
}
return IRQ_HANDLED;
}
@ -350,31 +390,30 @@ static int ntb_epf_init_isr(struct ntb_epf_dev *ndev, int msi_min, int msi_max)
argument &= ~MSIX_ENABLE;
}
ndev->db_count = irq - 1;
for (i = 0; i < irq; i++) {
ndev->irq_ctx[i].ndev = ndev;
ndev->irq_ctx[i].irq_no = i;
ret = request_irq(pci_irq_vector(pdev, i), ntb_epf_vec_isr,
0, "ntb_epf", ndev);
0, "ntb_epf", &ndev->irq_ctx[i]);
if (ret) {
dev_err(dev, "Failed to request irq\n");
goto err_request_irq;
goto err_free_irq;
}
}
ndev->db_count = irq - 1;
ret = ntb_epf_send_command(ndev, CMD_CONFIGURE_DOORBELL,
argument | irq);
if (ret) {
dev_err(dev, "Failed to configure doorbell\n");
goto err_configure_db;
goto err_free_irq;
}
return 0;
err_configure_db:
for (i = 0; i < ndev->db_count + 1; i++)
free_irq(pci_irq_vector(pdev, i), ndev);
err_request_irq:
err_free_irq:
while (i--)
free_irq(pci_irq_vector(pdev, i), &ndev->irq_ctx[i]);
pci_free_irq_vectors(pdev);
return ret;
@ -395,6 +434,36 @@ static u64 ntb_epf_db_valid_mask(struct ntb_dev *ntb)
return ntb_ndev(ntb)->db_valid_mask;
}
static int ntb_epf_db_vector_count(struct ntb_dev *ntb)
{
struct ntb_epf_dev *ndev = ntb_ndev(ntb);
unsigned int db_count = ndev->db_count;
/*
* db_count includes an extra skipped slot due to the legacy
* doorbell layout. Expose only the real doorbell vectors.
*/
if (db_count < NTB_EPF_MIN_DB_COUNT)
return 0;
return db_count - 1;
}
static u64 ntb_epf_db_vector_mask(struct ntb_dev *ntb, int db_vector)
{
int nr_vec;
/*
* db_count includes one skipped slot in the legacy layout. Valid
* doorbell vectors are therefore [0 .. (db_count - 2)].
*/
nr_vec = ntb_epf_db_vector_count(ntb);
if (db_vector < 0 || db_vector >= nr_vec)
return 0;
return BIT_ULL(db_vector);
}
static int ntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
{
return 0;
@ -473,6 +542,14 @@ static int ntb_epf_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
static int ntb_epf_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
{
struct ntb_epf_dev *ndev = ntb_ndev(ntb);
/*
* ffs() returns a 1-based bit index (bit 0 -> 1).
*
* With slot 0 reserved for link events, DB#0 would naturally map to
* slot 1. Historically an extra +1 offset was added, so DB#0 maps to
* slot 2 and slot 1 remains unused. Keep this mapping for
* backward-compatibility.
*/
u32 interrupt_num = ffs(db_bits) + 1;
struct device *dev = ndev->dev;
u32 db_entry_size;
@ -499,7 +576,7 @@ static u64 ntb_epf_db_read(struct ntb_dev *ntb)
{
struct ntb_epf_dev *ndev = ntb_ndev(ntb);
return ndev->db_val;
return atomic64_read(&ndev->db_val);
}
static int ntb_epf_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
@ -511,7 +588,7 @@ static int ntb_epf_db_clear(struct ntb_dev *ntb, u64 db_bits)
{
struct ntb_epf_dev *ndev = ntb_ndev(ntb);
ndev->db_val = 0;
atomic64_and(~db_bits, &ndev->db_val);
return 0;
}
@ -521,6 +598,8 @@ static const struct ntb_dev_ops ntb_epf_ops = {
.spad_count = ntb_epf_spad_count,
.peer_mw_count = ntb_epf_peer_mw_count,
.db_valid_mask = ntb_epf_db_valid_mask,
.db_vector_count = ntb_epf_db_vector_count,
.db_vector_mask = ntb_epf_db_vector_mask,
.db_set_mask = ntb_epf_db_set_mask,
.mw_set_trans = ntb_epf_mw_set_trans,
.mw_clear_trans = ntb_epf_mw_clear_trans,
@ -552,6 +631,12 @@ static int ntb_epf_init_dev(struct ntb_epf_dev *ndev)
struct device *dev = ndev->dev;
int ret;
ndev->mw_count = readl(ndev->ctrl_reg + NTB_EPF_MW_COUNT);
if (ndev->mw_count > NTB_EPF_MAX_MW_COUNT) {
dev_err(dev, "Unsupported MW count: %u\n", ndev->mw_count);
return -EINVAL;
}
/* One Link interrupt and rest doorbell interrupt */
ret = ntb_epf_init_isr(ndev, NTB_EPF_MIN_DB_COUNT + 1,
NTB_EPF_MAX_DB_COUNT + 1);
@ -560,15 +645,13 @@ static int ntb_epf_init_dev(struct ntb_epf_dev *ndev)
return ret;
}
ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
ndev->mw_count = readl(ndev->ctrl_reg + NTB_EPF_MW_COUNT);
/*
* ndev->db_count includes an extra skipped slot due to the legacy
* doorbell layout, hence -1.
*/
ndev->db_valid_mask = BIT_ULL(ndev->db_count - 1) - 1;
ndev->spad_count = readl(ndev->ctrl_reg + NTB_EPF_SPAD_COUNT);
if (ndev->mw_count > NTB_EPF_MAX_MW_COUNT) {
dev_err(dev, "Unsupported MW count: %u\n", ndev->mw_count);
return -EINVAL;
}
return 0;
}
@ -662,7 +745,7 @@ static void ntb_epf_cleanup_isr(struct ntb_epf_dev *ndev)
ntb_epf_send_command(ndev, CMD_TEARDOWN_DOORBELL, ndev->db_count + 1);
for (i = 0; i < ndev->db_count + 1; i++)
free_irq(pci_irq_vector(pdev, i), ndev);
free_irq(pci_irq_vector(pdev, i), &ndev->irq_ctx[i]);
pci_free_irq_vectors(pdev);
}

119
drivers/pci/controller/dwc/pcie-designware-ep.c
View File

@ -9,6 +9,7 @@
#include <linux/align.h>
#include <linux/bitfield.h>
#include <linux/of.h>
#include <linux/overflow.h>
#include <linux/platform_device.h>
#include "pcie-designware.h"
@ -817,6 +818,122 @@ dw_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
return ep->ops->get_features(ep);
}
static const struct pci_epc_bar_rsvd_region *
dw_pcie_ep_find_bar_rsvd_region(struct dw_pcie_ep *ep,
enum pci_epc_bar_rsvd_region_type type,
enum pci_barno *bar,
resource_size_t *bar_offset)
{
const struct pci_epc_features *features;
const struct pci_epc_bar_desc *bar_desc;
const struct pci_epc_bar_rsvd_region *r;
int i, j;
if (!ep->ops->get_features)
return NULL;
features = ep->ops->get_features(ep);
if (!features)
return NULL;
for (i = BAR_0; i <= BAR_5; i++) {
bar_desc = &features->bar[i];
if (!bar_desc->nr_rsvd_regions || !bar_desc->rsvd_regions)
continue;
for (j = 0; j < bar_desc->nr_rsvd_regions; j++) {
r = &bar_desc->rsvd_regions[j];
if (r->type != type)
continue;
if (bar)
*bar = i;
if (bar_offset)
*bar_offset = r->offset;
return r;
}
}
return NULL;
}
static int
dw_pcie_ep_get_aux_resources_count(struct pci_epc *epc, u8 func_no,
u8 vfunc_no)
{
struct dw_pcie_ep *ep = epc_get_drvdata(epc);
struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
struct dw_edma_chip *edma = &pci->edma;
if (!pci->edma_reg_size)
return 0;
if (edma->db_offset == ~0)
return 0;
return 1;
}
static int
dw_pcie_ep_get_aux_resources(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
struct pci_epc_aux_resource *resources,
int num_resources)
{
struct dw_pcie_ep *ep = epc_get_drvdata(epc);
struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
const struct pci_epc_bar_rsvd_region *rsvd;
struct dw_edma_chip *edma = &pci->edma;
enum pci_barno dma_ctrl_bar = NO_BAR;
resource_size_t db_offset = edma->db_offset;
resource_size_t dma_ctrl_bar_offset = 0;
resource_size_t dma_reg_size;
int count;
count = dw_pcie_ep_get_aux_resources_count(epc, func_no, vfunc_no);
if (count < 0)
return count;
if (num_resources < count)
return -ENOSPC;
if (!count)
return 0;
dma_reg_size = pci->edma_reg_size;
rsvd = dw_pcie_ep_find_bar_rsvd_region(ep,
PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO,
&dma_ctrl_bar,
&dma_ctrl_bar_offset);
if (rsvd && rsvd->size < dma_reg_size)
dma_reg_size = rsvd->size;
/*
* For interrupt-emulation doorbells, report a standalone resource
* instead of bundling it into the DMA controller MMIO resource.
*/
if (range_end_overflows_t(resource_size_t, db_offset,
sizeof(u32), dma_reg_size))
return -EINVAL;
resources[0] = (struct pci_epc_aux_resource) {
.type = PCI_EPC_AUX_DOORBELL_MMIO,
.phys_addr = pci->edma_reg_phys + db_offset,
.size = sizeof(u32),
.bar = dma_ctrl_bar,
.bar_offset = dma_ctrl_bar != NO_BAR ?
dma_ctrl_bar_offset + db_offset : 0,
.u.db_mmio = {
.irq = edma->db_irq,
.data = 0, /* write 0 to assert */
},
};
return 0;
}
static const struct pci_epc_ops epc_ops = {
.write_header = dw_pcie_ep_write_header,
.set_bar = dw_pcie_ep_set_bar,
@ -832,6 +949,8 @@ static const struct pci_epc_ops epc_ops = {
.start = dw_pcie_ep_start,
.stop = dw_pcie_ep_stop,
.get_features = dw_pcie_ep_get_features,
.get_aux_resources_count = dw_pcie_ep_get_aux_resources_count,
.get_aux_resources = dw_pcie_ep_get_aux_resources,
};
/**

4
drivers/pci/controller/dwc/pcie-designware.c
View File

@ -162,8 +162,12 @@ int dw_pcie_get_resources(struct dw_pcie *pci)
pci->edma.reg_base = devm_ioremap_resource(pci->dev, res);
if (IS_ERR(pci->edma.reg_base))
return PTR_ERR(pci->edma.reg_base);
pci->edma_reg_phys = res->start;
pci->edma_reg_size = resource_size(res);
} else if (pci->atu_size >= 2 * DEFAULT_DBI_DMA_OFFSET) {
pci->edma.reg_base = pci->atu_base + DEFAULT_DBI_DMA_OFFSET;
pci->edma_reg_phys = pci->atu_phys_addr + DEFAULT_DBI_DMA_OFFSET;
pci->edma_reg_size = pci->atu_size - DEFAULT_DBI_DMA_OFFSET;
}
}

2
drivers/pci/controller/dwc/pcie-designware.h
View File

@ -544,6 +544,8 @@ struct dw_pcie {
int max_link_speed;
u8 n_fts[2];
struct dw_edma_chip edma;
phys_addr_t edma_reg_phys;
resource_size_t edma_reg_size;
bool l1ss_support; /* L1 PM Substates support */
struct clk_bulk_data app_clks[DW_PCIE_NUM_APP_CLKS];
struct clk_bulk_data core_clks[DW_PCIE_NUM_CORE_CLKS];

21
drivers/pci/endpoint/functions/pci-epf-ntb.c
View File

@ -559,12 +559,15 @@ static int epf_ntb_configure_db(struct epf_ntb *ntb,
struct pci_epc *epc;
int ret;
if (db_count > MAX_DB_COUNT)
return -EINVAL;
ntb_epc = ntb->epc[type];
epc = ntb_epc->epc;
if (!db_count || db_count > MAX_DB_COUNT) {
dev_err(&epc->dev, "DB count %d out of range (1 - %d)\n",
db_count, MAX_DB_COUNT);
return -EINVAL;
}
if (msix)
ret = epf_ntb_configure_msix(ntb, type, db_count);
else
@ -1278,7 +1281,6 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb,
u8 func_no, vfunc_no;
struct pci_epc *epc;
struct device *dev;
u32 db_count;
int ret;
ntb_epc = ntb->epc[type];
@ -1296,17 +1298,16 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb,
func_no = ntb_epc->func_no;
vfunc_no = ntb_epc->vfunc_no;
db_count = ntb->db_count;
if (db_count > MAX_DB_COUNT) {
dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
if (!ntb->db_count || ntb->db_count > MAX_DB_COUNT) {
dev_err(dev, "DB count %d out of range (1 - %d)\n",
ntb->db_count, MAX_DB_COUNT);
return -EINVAL;
}
ntb->db_count = db_count;
epc = ntb_epc->epc;
if (msi_capable) {
ret = pci_epc_set_msi(epc, func_no, vfunc_no, db_count);
ret = pci_epc_set_msi(epc, func_no, vfunc_no, ntb->db_count);
if (ret) {
dev_err(dev, "%s intf: MSI configuration failed\n",
pci_epc_interface_string(type));
@ -1315,7 +1316,7 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb,
}
if (msix_capable) {
ret = pci_epc_set_msix(epc, func_no, vfunc_no, db_count,
ret = pci_epc_set_msix(epc, func_no, vfunc_no, ntb->db_count,
ntb_epc->msix_bar,
ntb_epc->msix_table_offset);
if (ret) {

86
drivers/pci/endpoint/functions/pci-epf-test.c
View File

@ -94,6 +94,7 @@ struct pci_epf_test {
bool dma_private;
const struct pci_epc_features *epc_features;
struct pci_epf_bar db_bar;
bool db_bar_programmed;
size_t bar_size[PCI_STD_NUM_BARS];
};
@ -733,7 +734,9 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test,
{
u32 status = le32_to_cpu(reg->status);
struct pci_epf *epf = epf_test->epf;
struct pci_epf_doorbell_msg *db;
struct pci_epc *epc = epf->epc;
unsigned long irq_flags;
struct msi_msg *msg;
enum pci_barno bar;
size_t offset;
@ -743,13 +746,28 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test,
if (ret)
goto set_status_err;
msg = &epf->db_msg[0].msg;
bar = pci_epc_get_next_free_bar(epf_test->epc_features, epf_test->test_reg_bar + 1);
if (bar < BAR_0)
goto err_doorbell_cleanup;
db = &epf->db_msg[0];
msg = &db->msg;
epf_test->db_bar_programmed = false;
if (db->bar != NO_BAR) {
/*
* The doorbell target is already exposed via a platform-owned
* fixed BAR
*/
bar = db->bar;
offset = db->offset;
} else {
bar = pci_epc_get_next_free_bar(epf_test->epc_features,
epf_test->test_reg_bar + 1);
if (bar < BAR_0)
goto err_doorbell_cleanup;
}
irq_flags = epf->db_msg[0].irq_flags | IRQF_ONESHOT;
ret = request_threaded_irq(epf->db_msg[0].virq, NULL,
pci_epf_test_doorbell_handler, IRQF_ONESHOT,
pci_epf_test_doorbell_handler, irq_flags,
"pci-ep-test-doorbell", epf_test);
if (ret) {
dev_err(&epf->dev,
@ -761,22 +779,30 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test,
reg->doorbell_data = cpu_to_le32(msg->data);
reg->doorbell_bar = cpu_to_le32(bar);
msg = &epf->db_msg[0].msg;
ret = pci_epf_align_inbound_addr(epf, bar, ((u64)msg->address_hi << 32) | msg->address_lo,
&epf_test->db_bar.phys_addr, &offset);
if (db->bar == NO_BAR) {
ret = pci_epf_align_inbound_addr(epf, bar,
((u64)msg->address_hi << 32) |
msg->address_lo,
&epf_test->db_bar.phys_addr,
&offset);
if (ret)
goto err_free_irq;
if (ret)
goto err_free_irq;
}
reg->doorbell_offset = cpu_to_le32(offset);
epf_test->db_bar.barno = bar;
epf_test->db_bar.size = epf->bar[bar].size;
epf_test->db_bar.flags = epf->bar[bar].flags;
if (db->bar == NO_BAR) {
epf_test->db_bar.barno = bar;
epf_test->db_bar.size = epf->bar[bar].size;
epf_test->db_bar.flags = epf->bar[bar].flags;
ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &epf_test->db_bar);
if (ret)
goto err_free_irq;
ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &epf_test->db_bar);
if (ret)
goto err_free_irq;
epf_test->db_bar_programmed = true;
}
status |= STATUS_DOORBELL_ENABLE_SUCCESS;
reg->status = cpu_to_le32(status);
@ -806,17 +832,23 @@ static void pci_epf_test_disable_doorbell(struct pci_epf_test *epf_test,
free_irq(epf->db_msg[0].virq, epf_test);
pci_epf_test_doorbell_cleanup(epf_test);
/*
* The doorbell feature temporarily overrides the inbound translation
* to point to the address stored in epf_test->db_bar.phys_addr, i.e.,
* it calls set_bar() twice without ever calling clear_bar(), as
* calling clear_bar() would clear the BAR's PCI address assigned by
* the host. Thus, when disabling the doorbell, restore the inbound
* translation to point to the memory allocated for the BAR.
*/
ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &epf->bar[bar]);
if (ret)
goto set_status_err;
if (epf_test->db_bar_programmed) {
/*
* The doorbell feature temporarily overrides the inbound
* translation to point to the address stored in
* epf_test->db_bar.phys_addr, i.e., it calls set_bar()
* twice without ever calling clear_bar(), as calling
* clear_bar() would clear the BAR's PCI address assigned
* by the host. Thus, when disabling the doorbell, restore
* the inbound translation to point to the memory allocated
* for the BAR.
*/
ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &epf->bar[bar]);
if (ret)
goto set_status_err;
epf_test->db_bar_programmed = false;
}
status |= STATUS_DOORBELL_DISABLE_SUCCESS;
reg->status = cpu_to_le32(status);

274
drivers/pci/endpoint/functions/pci-epf-vntb.c
View File

@ -37,6 +37,7 @@
*/
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/delay.h>
#include <linux/io.h>
#include <linux/module.h>
@ -66,9 +67,11 @@ static struct workqueue_struct *kpcintb_workqueue;
#define NTB_MW_OFFSET 2
#define DB_COUNT_MASK GENMASK(15, 0)
#define MSIX_ENABLE BIT(16)
#define MAX_DB_COUNT 32
#define MAX_MW 4
/* Limit per-work execution to avoid monopolizing kworker on doorbell storms. */
#define VNTB_PEER_DB_WORK_BUDGET 5
enum epf_ntb_bar {
BAR_CONFIG,
BAR_DB,
@ -79,6 +82,15 @@ enum epf_ntb_bar {
VNTB_BAR_NUM,
};
enum epf_irq_slot {
EPF_IRQ_LINK = 0,
EPF_IRQ_RESERVED_DB, /* Historically skipped slot */
EPF_IRQ_DB_START,
};
#define MIN_DB_COUNT (EPF_IRQ_DB_START + 1)
#define MAX_DB_COUNT 32
/*
* +--------------------------------------------------+ Base
* | |
@ -129,11 +141,18 @@ struct epf_ntb {
u32 spad_count;
u64 mws_size[MAX_MW];
atomic64_t db;
atomic64_t peer_db_pending;
struct work_struct peer_db_work;
u32 vbus_number;
u16 vntb_pid;
u16 vntb_vid;
bool linkup;
/*
* True when doorbells are interrupt-driven (MSI or embedded), false
* when polled.
*/
bool msi_doorbell;
u32 spad_size;
@ -261,10 +280,11 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
ntb = container_of(work, struct epf_ntb, cmd_handler.work);
for (i = 1; i < ntb->db_count && !ntb->msi_doorbell; i++) {
for (i = EPF_IRQ_DB_START; i < ntb->db_count && !ntb->msi_doorbell;
i++) {
if (ntb->epf_db[i]) {
atomic64_or(1 << (i - 1), &ntb->db);
ntb_db_event(&ntb->ntb, i);
atomic64_or(1 << (i - EPF_IRQ_DB_START), &ntb->db);
ntb_db_event(&ntb->ntb, i - EPF_IRQ_DB_START);
ntb->epf_db[i] = 0;
}
}
@ -330,10 +350,10 @@ static irqreturn_t epf_ntb_doorbell_handler(int irq, void *data)
struct epf_ntb *ntb = data;
int i;
for (i = 1; i < ntb->db_count; i++)
for (i = EPF_IRQ_DB_START; i < ntb->db_count; i++)
if (irq == ntb->epf->db_msg[i].virq) {
atomic64_or(1 << (i - 1), &ntb->db);
ntb_db_event(&ntb->ntb, i);
atomic64_or(1 << (i - EPF_IRQ_DB_START), &ntb->db);
ntb_db_event(&ntb->ntb, i - EPF_IRQ_DB_START);
}
return IRQ_HANDLED;
@ -483,7 +503,6 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
{
const struct pci_epc_features *epc_features;
struct device *dev;
u32 db_count;
int ret;
dev = &ntb->epf->dev;
@ -495,14 +514,12 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
return -EINVAL;
}
db_count = ntb->db_count;
if (db_count > MAX_DB_COUNT) {
dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
if (ntb->db_count < MIN_DB_COUNT || ntb->db_count > MAX_DB_COUNT) {
dev_err(dev, "DB count %d out of range (%d - %d)\n",
ntb->db_count, MIN_DB_COUNT, MAX_DB_COUNT);
return -EINVAL;
}
ntb->db_count = db_count;
if (epc_features->msi_capable) {
ret = pci_epc_set_msi(ntb->epf->epc,
ntb->epf->func_no,
@ -517,6 +534,17 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
return 0;
}
static bool epf_ntb_db_irq_is_duplicated(const struct pci_epf *epf, unsigned int idx)
{
unsigned int i;
for (i = 0; i < idx; i++)
if (epf->db_msg[i].virq == epf->db_msg[idx].virq)
return true;
return false;
}
static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
struct pci_epf_bar *db_bar,
const struct pci_epc_features *epc_features,
@ -533,9 +561,24 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
if (ret)
return ret;
/*
* The doorbell target may already be exposed by a platform-owned fixed
* BAR. In that case, we must reuse it and the requested db_bar must
* match.
*/
if (epf->db_msg[0].bar != NO_BAR && epf->db_msg[0].bar != barno) {
ret = -EINVAL;
goto err_free_doorbell;
}
for (req = 0; req < ntb->db_count; req++) {
/* Avoid requesting duplicate handlers */
if (epf_ntb_db_irq_is_duplicated(epf, req))
continue;
ret = request_irq(epf->db_msg[req].virq, epf_ntb_doorbell_handler,
0, "pci_epf_vntb_db", ntb);
epf->db_msg[req].irq_flags, "pci_epf_vntb_db",
ntb);
if (ret) {
dev_err(&epf->dev,
@ -545,6 +588,22 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
}
}
if (epf->db_msg[0].bar != NO_BAR) {
for (i = 0; i < ntb->db_count; i++) {
msg = &epf->db_msg[i].msg;
if (epf->db_msg[i].bar != barno) {
ret = -EINVAL;
goto err_free_irq;
}
ntb->reg->db_data[i] = msg->data;
ntb->reg->db_offset[i] = epf->db_msg[i].offset;
}
goto out;
}
/* Program inbound mapping for the doorbell */
msg = &epf->db_msg[0].msg;
high = 0;
@ -591,6 +650,7 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
ntb->reg->db_offset[i] = offset;
}
out:
ntb->reg->db_entry_size = 0;
ntb->msi_doorbell = true;
@ -598,9 +658,13 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
return 0;
err_free_irq:
for (req--; req >= 0; req--)
for (req--; req >= 0; req--) {
if (epf_ntb_db_irq_is_duplicated(epf, req))
continue;
free_irq(epf->db_msg[req].virq, ntb);
}
err_free_doorbell:
pci_epf_free_doorbell(ntb->epf);
return ret;
}
@ -666,8 +730,11 @@ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
if (ntb->msi_doorbell) {
int i;
for (i = 0; i < ntb->db_count; i++)
for (i = 0; i < ntb->db_count; i++) {
if (epf_ntb_db_irq_is_duplicated(ntb->epf, i))
continue;
free_irq(ntb->epf->db_msg[i].virq, ntb);
}
}
if (ntb->epf->db_msg)
@ -920,6 +987,9 @@ static int epf_ntb_epc_init(struct epf_ntb *ntb)
INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler);
queue_work(kpcintb_workqueue, &ntb->cmd_handler.work);
atomic64_set(&ntb->peer_db_pending, 0);
enable_work(&ntb->peer_db_work);
return 0;
err_write_header:
@ -943,11 +1013,18 @@ static int epf_ntb_epc_init(struct epf_ntb *ntb)
static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
{
disable_delayed_work_sync(&ntb->cmd_handler);
disable_work_sync(&ntb->peer_db_work);
atomic64_set(&ntb->peer_db_pending, 0);
epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
epf_ntb_db_bar_clear(ntb);
epf_ntb_config_sspad_bar_clear(ntb);
}
static bool epf_ntb_epc_attached(struct epf_ntb *ntb)
{
return ntb->epf->epc || ntb->epf->sec_epc;
}
#define EPF_NTB_R(_name) \
static ssize_t epf_ntb_##_name##_show(struct config_item *item, \
char *page) \
@ -967,6 +1044,9 @@ static ssize_t epf_ntb_##_name##_store(struct config_item *item, \
u32 val; \
int ret; \
\
if (epf_ntb_epc_attached(ntb)) \
return -EOPNOTSUPP; \
\
ret = kstrtou32(page, 0, &val); \
if (ret) \
return ret; \
@ -1009,6 +1089,9 @@ static ssize_t epf_ntb_##_name##_store(struct config_item *item, \
u64 val; \
int ret; \
\
if (epf_ntb_epc_attached(ntb)) \
return -EOPNOTSUPP; \
\
ret = kstrtou64(page, 0, &val); \
if (ret) \
return ret; \
@ -1047,6 +1130,9 @@ static ssize_t epf_ntb_##_name##_store(struct config_item *item, \
int val; \
int ret; \
\
if (epf_ntb_epc_attached(ntb)) \
return -EOPNOTSUPP; \
\
ret = kstrtoint(page, 0, &val); \
if (ret) \
return ret; \
@ -1067,6 +1153,9 @@ static ssize_t epf_ntb_num_mws_store(struct config_item *item,
u32 val;
int ret;
if (epf_ntb_epc_attached(ntb))
return -EOPNOTSUPP;
ret = kstrtou32(page, 0, &val);
if (ret)
return ret;
@ -1079,10 +1168,32 @@ static ssize_t epf_ntb_num_mws_store(struct config_item *item,
return len;
}
static ssize_t epf_ntb_db_count_store(struct config_item *item,
const char *page, size_t len)
{
struct config_group *group = to_config_group(item);
struct epf_ntb *ntb = to_epf_ntb(group);
u32 val;
int ret;
if (epf_ntb_epc_attached(ntb))
return -EOPNOTSUPP;
ret = kstrtou32(page, 0, &val);
if (ret)
return ret;
if (val < MIN_DB_COUNT || val > MAX_DB_COUNT)
return -EINVAL;
WRITE_ONCE(ntb->db_count, val);
return len;
}
EPF_NTB_R(spad_count)
EPF_NTB_W(spad_count)
EPF_NTB_R(db_count)
EPF_NTB_W(db_count)
EPF_NTB_R(num_mws)
EPF_NTB_R(vbus_number)
EPF_NTB_W(vbus_number)
@ -1251,9 +1362,48 @@ static int vntb_epf_peer_mw_count(struct ntb_dev *ntb)
return ntb_ndev(ntb)->num_mws;
}
static int vntb_epf_db_vector_count(struct ntb_dev *ntb)
{
struct epf_ntb *ndev = ntb_ndev(ntb);
u32 db_count = READ_ONCE(ndev->db_count);
/*
* db_count is the total number of doorbell slots exposed to
* the peer, including:
* - slot #0 reserved for link events
* - slot #1 historically unused (kept for protocol compatibility)
*
* Report only usable per-vector doorbell interrupts.
*/
if (db_count < MIN_DB_COUNT || db_count > MAX_DB_COUNT)
return 0;
return db_count - EPF_IRQ_DB_START;
}
static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb)
{
return BIT_ULL(ntb_ndev(ntb)->db_count) - 1;
int nr_vec = vntb_epf_db_vector_count(ntb);
if (!nr_vec)
return 0;
return GENMASK_ULL(nr_vec - 1, 0);
}
static u64 vntb_epf_db_vector_mask(struct ntb_dev *ntb, int db_vector)
{
int nr_vec;
/*
* Doorbell vectors are numbered [0 .. nr_vec - 1], where nr_vec
* excludes the two reserved slots described above.
*/
nr_vec = vntb_epf_db_vector_count(ntb);
if (db_vector < 0 || db_vector >= nr_vec)
return 0;
return BIT_ULL(db_vector);
}
static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
@ -1357,22 +1507,84 @@ static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32
return 0;
}
static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits)
static void vntb_epf_peer_db_work(struct work_struct *work)
{
u32 interrupt_num = ffs(db_bits) + 1;
struct epf_ntb *ntb = ntb_ndev(ndev);
struct epf_ntb *ntb = container_of(work, struct epf_ntb, peer_db_work);
struct pci_epf *epf = ntb->epf;
unsigned int budget = VNTB_PEER_DB_WORK_BUDGET;
u8 func_no, vfunc_no;
unsigned int db_bit;
u32 interrupt_num;
u64 db_bits;
int ret;
func_no = ntb->epf->func_no;
vfunc_no = ntb->epf->vfunc_no;
if (!epf || !epf->epc)
return;
ret = pci_epc_raise_irq(ntb->epf->epc, func_no, vfunc_no,
PCI_IRQ_MSI, interrupt_num + 1);
if (ret)
dev_err(&ntb->ntb.dev, "Failed to raise IRQ\n");
func_no = epf->func_no;
vfunc_no = epf->vfunc_no;
return ret;
/*
* Drain doorbells from peer_db_pending in snapshots (atomic64_xchg()).
* Limit the number of snapshots handled per run so we don't monopolize
* the workqueue under a doorbell storm.
*/
while (budget--) {
db_bits = atomic64_xchg(&ntb->peer_db_pending, 0);
if (!db_bits)
return;
while (db_bits) {
/*
* pci_epc_raise_irq() for MSI expects a 1-based
* interrupt number. The first usable doorbell starts
* at EPF_IRQ_DB_START in the legacy slot layout.
*
* Legacy mapping (kept for compatibility):
*
* MSI #1 : link event (reserved)
* MSI #2 : unused (historical offset)
* MSI #3 : doorbell bit 0 (DB#0)
* MSI #4 : doorbell bit 1 (DB#1)
* ...
*
* Do not change this mapping to avoid breaking
* interoperability with older peers.
*/
db_bit = __ffs64(db_bits);
interrupt_num = db_bit + EPF_IRQ_DB_START + 1;
db_bits &= ~BIT_ULL(db_bit);
ret = pci_epc_raise_irq(epf->epc, func_no, vfunc_no,
PCI_IRQ_MSI, interrupt_num);
if (ret)
dev_err(&ntb->ntb.dev,
"Failed to raise IRQ for interrupt_num %u: %d\n",
interrupt_num, ret);
}
}
if (atomic64_read(&ntb->peer_db_pending))
queue_work(kpcintb_workqueue, &ntb->peer_db_work);
}
static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits)
{
struct epf_ntb *ntb = ntb_ndev(ndev);
db_bits &= vntb_epf_db_valid_mask(ndev);
if (!db_bits)
return 0;
/*
* .peer_db_set() may be called from atomic context. pci_epc_raise_irq()
* can sleep (it takes epc->lock), so defer MSI raising to process
* context. Doorbell requests are coalesced in peer_db_pending.
*/
atomic64_or(db_bits, &ntb->peer_db_pending);
queue_work(kpcintb_workqueue, &ntb->peer_db_work);
return 0;
}
static u64 vntb_epf_db_read(struct ntb_dev *ndev)
@ -1441,6 +1653,8 @@ static const struct ntb_dev_ops vntb_epf_ops = {
.spad_count = vntb_epf_spad_count,
.peer_mw_count = vntb_epf_peer_mw_count,
.db_valid_mask = vntb_epf_db_valid_mask,
.db_vector_count = vntb_epf_db_vector_count,
.db_vector_mask = vntb_epf_db_vector_mask,
.db_set_mask = vntb_epf_db_set_mask,
.mw_set_trans = vntb_epf_mw_set_trans,
.mw_clear_trans = vntb_epf_mw_clear_trans,
@ -1619,6 +1833,10 @@ static int epf_ntb_probe(struct pci_epf *epf,
ntb->epf = epf;
ntb->vbus_number = 0xff;
INIT_WORK(&ntb->peer_db_work, vntb_epf_peer_db_work);
disable_work(&ntb->peer_db_work);
atomic64_set(&ntb->peer_db_pending, 0);
/* Initially, no bar is assigned */
for (i = 0; i < VNTB_BAR_NUM; i++)
ntb->epf_ntb_bar[i] = NO_BAR;

175
drivers/pci/endpoint/pci-ep-msi.c
View File

@ -6,8 +6,11 @@
* Author: Frank Li <Frank.Li@nxp.com>
*/
#include <linux/align.h>
#include <linux/cleanup.h>
#include <linux/device.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/irqdomain.h>
#include <linux/module.h>
#include <linux/msi.h>
@ -35,23 +38,116 @@ static void pci_epf_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
pci_epc_put(epc);
}
int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db)
static int pci_epf_alloc_doorbell_embedded(struct pci_epf *epf, u16 num_db)
{
const struct pci_epc_aux_resource *doorbell = NULL;
struct pci_epf_doorbell_msg *msg;
struct pci_epc *epc = epf->epc;
struct device *dev = &epf->dev;
struct irq_domain *domain;
void *msg;
int ret;
int i;
size_t map_size = 0, off = 0;
dma_addr_t iova_base = 0;
phys_addr_t phys_base;
int count, ret, i;
u64 addr;
/* TODO: Multi-EPF support */
if (list_first_entry_or_null(&epc->pci_epf, struct pci_epf, list) != epf) {
dev_err(dev, "MSI doorbell doesn't support multiple EPF\n");
count = pci_epc_get_aux_resources_count(epc, epf->func_no,
epf->vfunc_no);
if (count < 0)
return count;
if (!count)
return -ENODEV;
struct pci_epc_aux_resource *res __free(kfree) =
kcalloc(count, sizeof(*res), GFP_KERNEL);
if (!res)
return -ENOMEM;
ret = pci_epc_get_aux_resources(epc, epf->func_no, epf->vfunc_no,
res, count);
if (ret)
return ret;
/* TODO: Support multiple DOORBELL_MMIO resources per EPC. */
for (i = 0; i < count; i++) {
if (res[i].type != PCI_EPC_AUX_DOORBELL_MMIO)
continue;
doorbell = &res[i];
break;
}
if (!doorbell)
return -ENODEV;
addr = doorbell->phys_addr;
if (!IS_ALIGNED(addr, sizeof(u32)))
return -EINVAL;
/*
* Reuse the pre-exposed BAR window if available. Otherwise map the MMIO
* doorbell resource here. Any required IOMMU mapping is handled
* internally, matching the MSI doorbell semantics.
*/
if (doorbell->bar == NO_BAR) {
phys_base = addr & PAGE_MASK;
off = addr - phys_base;
map_size = PAGE_ALIGN(off + sizeof(u32));
iova_base = dma_map_resource(epc->dev.parent, phys_base,
map_size, DMA_FROM_DEVICE, 0);
if (dma_mapping_error(epc->dev.parent, iova_base))
return -EIO;
addr = iova_base + off;
}
if (epf->db_msg)
return -EBUSY;
msg = kcalloc(num_db, sizeof(*msg), GFP_KERNEL);
if (!msg) {
ret = -ENOMEM;
goto err_unmap;
}
/*
* Embedded doorbell backends (e.g. DesignWare eDMA interrupt emulation)
* typically provide a single IRQ and do not offer per-doorbell
* distinguishable address/data pairs. The EPC aux resource therefore
* exposes one DOORBELL_MMIO entry (u.db_mmio.irq).
*
* Still, pci_epf_alloc_doorbell() allows requesting multiple doorbells.
* For such backends we replicate the same address/data for each entry
* and mark the IRQ as shared (IRQF_SHARED). Consumers must treat them
* as equivalent "kick" doorbells.
*/
for (i = 0; i < num_db; i++)
msg[i] = (struct pci_epf_doorbell_msg) {
.msg.address_lo = (u32)addr,
.msg.address_hi = (u32)(addr >> 32),
.msg.data = doorbell->u.db_mmio.data,
.virq = doorbell->u.db_mmio.irq,
.irq_flags = IRQF_SHARED,
.type = PCI_EPF_DOORBELL_EMBEDDED,
.bar = doorbell->bar,
.offset = (doorbell->bar == NO_BAR) ? 0 :
doorbell->bar_offset,
.iova_base = iova_base,
.iova_size = map_size,
};
epf->num_db = num_db;
epf->db_msg = msg;
return 0;
err_unmap:
if (map_size)
dma_unmap_resource(epc->dev.parent, iova_base, map_size,
DMA_FROM_DEVICE, 0);
return ret;
}
static int pci_epf_alloc_doorbell_msi(struct pci_epf *epf, u16 num_db)
{
struct pci_epf_doorbell_msg *msg;
struct device *dev = &epf->dev;
struct pci_epc *epc = epf->epc;
struct irq_domain *domain;
int ret, i;
domain = of_msi_map_get_device_domain(epc->dev.parent, 0,
DOMAIN_BUS_PLATFORM_MSI);
@ -74,6 +170,12 @@ int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db)
if (!msg)
return -ENOMEM;
for (i = 0; i < num_db; i++)
msg[i] = (struct pci_epf_doorbell_msg) {
.type = PCI_EPF_DOORBELL_MSI,
.bar = NO_BAR,
};
epf->num_db = num_db;
epf->db_msg = msg;
@ -90,13 +192,60 @@ int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db)
for (i = 0; i < num_db; i++)
epf->db_msg[i].virq = msi_get_virq(epc->dev.parent, i);
return ret;
return 0;
}
int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db)
{
struct pci_epc *epc = epf->epc;
struct device *dev = &epf->dev;
int ret;
/* TODO: Multi-EPF support */
if (list_first_entry_or_null(&epc->pci_epf, struct pci_epf, list) != epf) {
dev_err(dev, "Doorbell doesn't support multiple EPF\n");
return -EINVAL;
}
if (epf->db_msg)
return -EBUSY;
ret = pci_epf_alloc_doorbell_msi(epf, num_db);
if (!ret)
return 0;
/*
* Fall back to embedded doorbell only when platform MSI is unavailable
* for this EPC.
*/
if (ret != -ENODEV)
return ret;
ret = pci_epf_alloc_doorbell_embedded(epf, num_db);
if (ret) {
dev_err(dev, "Failed to allocate doorbell: %d\n", ret);
return ret;
}
dev_info(dev, "Using embedded (DMA) doorbell fallback\n");
return 0;
}
EXPORT_SYMBOL_GPL(pci_epf_alloc_doorbell);
void pci_epf_free_doorbell(struct pci_epf *epf)
{
platform_device_msi_free_irqs_all(epf->epc->dev.parent);
struct pci_epf_doorbell_msg *msg0;
struct pci_epc *epc = epf->epc;
if (!epf->db_msg)
return;
msg0 = &epf->db_msg[0];
if (msg0->type == PCI_EPF_DOORBELL_MSI)
platform_device_msi_free_irqs_all(epf->epc->dev.parent);
else if (msg0->type == PCI_EPF_DOORBELL_EMBEDDED && msg0->iova_size)
dma_unmap_resource(epc->dev.parent, msg0->iova_base,
msg0->iova_size, DMA_FROM_DEVICE, 0);
kfree(epf->db_msg);
epf->db_msg = NULL;

80
drivers/pci/endpoint/pci-epc-core.c
View File

@ -156,6 +156,86 @@ const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc,
}
EXPORT_SYMBOL_GPL(pci_epc_get_features);
/**
* pci_epc_get_aux_resources_count() - get the number of EPC-provided auxiliary resources
* @epc: EPC device
* @func_no: function number
* @vfunc_no: virtual function number
*
* Some EPC backends integrate auxiliary blocks (e.g. DMA engines) whose control
* registers and/or descriptor memories can be exposed to the host by mapping
* them into BAR space. This helper queries how many such resources the backend
* provides.
*
* Return: the number of available resources on success, -EOPNOTSUPP if the
* backend does not support auxiliary resource queries, or another -errno on
* failure.
*/
int pci_epc_get_aux_resources_count(struct pci_epc *epc, u8 func_no,
u8 vfunc_no)
{
int count;
if (!epc || !epc->ops)
return -EINVAL;
if (!pci_epc_function_is_valid(epc, func_no, vfunc_no))
return -EINVAL;
if (!epc->ops->get_aux_resources_count)
return -EOPNOTSUPP;
mutex_lock(&epc->lock);
count = epc->ops->get_aux_resources_count(epc, func_no,
vfunc_no);
mutex_unlock(&epc->lock);
return count;
}
EXPORT_SYMBOL_GPL(pci_epc_get_aux_resources_count);
/**
* pci_epc_get_aux_resources() - query EPC-provided auxiliary resources
* @epc: EPC device
* @func_no: function number
* @vfunc_no: virtual function number
* @resources: output array
* @num_resources: size of @resources array in entries
*
* Some EPC backends integrate auxiliary blocks (e.g. DMA engines) whose control
* registers and/or descriptor memories can be exposed to the host by mapping
* them into BAR space. This helper queries the backend for such resources.
*
* Return: 0 on success, -EOPNOTSUPP if the backend does not support auxiliary
* resource queries, or another -errno on failure.
*/
int pci_epc_get_aux_resources(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
struct pci_epc_aux_resource *resources,
int num_resources)
{
int ret;
if (!resources || num_resources <= 0)
return -EINVAL;
if (!epc || !epc->ops)
return -EINVAL;
if (!pci_epc_function_is_valid(epc, func_no, vfunc_no))
return -EINVAL;
if (!epc->ops->get_aux_resources)
return -EOPNOTSUPP;
mutex_lock(&epc->lock);
ret = epc->ops->get_aux_resources(epc, func_no, vfunc_no, resources,
num_resources);
mutex_unlock(&epc->lock);
return ret;
}
EXPORT_SYMBOL_GPL(pci_epc_get_aux_resources);
/**
* pci_epc_stop() - stop the PCI link
* @epc: the link of the EPC device that has to be stopped

54
include/linux/pci-epc.h
View File

@ -61,6 +61,47 @@ struct pci_epc_map {
void __iomem *virt_addr;
};
/**
* enum pci_epc_aux_resource_type - auxiliary resource type identifiers
* @PCI_EPC_AUX_DOORBELL_MMIO: Doorbell MMIO, that might be outside the DMA
* controller register window
*
* EPC backends may expose auxiliary blocks (e.g. DMA engines) by mapping their
* register windows and descriptor memories into BAR space. This enum
* identifies the type of each exposable resource.
*/
enum pci_epc_aux_resource_type {
PCI_EPC_AUX_DOORBELL_MMIO,
};
/**
* struct pci_epc_aux_resource - a physical auxiliary resource that may be
* exposed for peer use
* @type: resource type, see enum pci_epc_aux_resource_type
* @phys_addr: physical base address of the resource
* @size: size of the resource in bytes
* @bar: BAR number where this resource is already exposed to the RC
* (NO_BAR if not)
* @bar_offset: offset within @bar where the resource starts (valid iff
* @bar != NO_BAR)
* @u: type-specific metadata
*/
struct pci_epc_aux_resource {
enum pci_epc_aux_resource_type type;
phys_addr_t phys_addr;
resource_size_t size;
enum pci_barno bar;
resource_size_t bar_offset;
union {
/* PCI_EPC_AUX_DOORBELL_MMIO */
struct {
int irq; /* IRQ number for the doorbell handler */
u32 data; /* write value to ring the doorbell */
} db_mmio;
} u;
};
/**
* struct pci_epc_ops - set of function pointers for performing EPC operations
* @write_header: ops to populate configuration space header
@ -84,6 +125,9 @@ struct pci_epc_map {
* @start: ops to start the PCI link
* @stop: ops to stop the PCI link
* @get_features: ops to get the features supported by the EPC
* @get_aux_resources_count: ops to get the number of controller-owned
* auxiliary resources
* @get_aux_resources: ops to retrieve controller-owned auxiliary resources
* @owner: the module owner containing the ops
*/
struct pci_epc_ops {
@ -115,6 +159,11 @@ struct pci_epc_ops {
void (*stop)(struct pci_epc *epc);
const struct pci_epc_features* (*get_features)(struct pci_epc *epc,
u8 func_no, u8 vfunc_no);
int (*get_aux_resources_count)(struct pci_epc *epc, u8 func_no,
u8 vfunc_no);
int (*get_aux_resources)(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
struct pci_epc_aux_resource *resources,
int num_resources);
struct module *owner;
};
@ -343,6 +392,11 @@ int pci_epc_start(struct pci_epc *epc);
void pci_epc_stop(struct pci_epc *epc);
const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc,
u8 func_no, u8 vfunc_no);
int pci_epc_get_aux_resources_count(struct pci_epc *epc, u8 func_no,
u8 vfunc_no);
int pci_epc_get_aux_resources(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
struct pci_epc_aux_resource *resources,
int num_resources);
enum pci_barno
pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features);
enum pci_barno pci_epc_get_next_free_bar(const struct pci_epc_features

31
include/linux/pci-epf.h
View File

@ -152,14 +152,41 @@ struct pci_epf_bar {
struct pci_epf_bar_submap *submap;
};
enum pci_epf_doorbell_type {
PCI_EPF_DOORBELL_MSI = 0,
PCI_EPF_DOORBELL_EMBEDDED,
};
/**
* struct pci_epf_doorbell_msg - represents doorbell message
* @msg: MSI message
* @virq: IRQ number of this doorbell MSI message
* @msg: Doorbell address/data pair to be mapped into BAR space.
* For MSI-backed doorbells this is the MSI message, while for
* "embedded" doorbells this represents an MMIO write that asserts
* an interrupt on the EP side.
* @virq: IRQ number of this doorbell message
* @irq_flags: Required flags for request_irq()/request_threaded_irq().
* Callers may OR-in additional flags (e.g. IRQF_ONESHOT).
* @type: Doorbell type.
* @bar: BAR number where the doorbell target is already exposed to the RC
* (NO_BAR if not)
* @offset: offset within @bar for the doorbell target (valid iff
* @bar != NO_BAR)
* @iova_base: Internal: base DMA address returned by dma_map_resource() for the
* embedded doorbell MMIO window (used only for unmapping). Valid
* when @type is PCI_EPF_DOORBELL_EMBEDDED and @iova_size is
* non-zero.
* @iova_size: Internal: size of the dma_map_resource() mapping at @iova_base.
* Zero when no mapping was created (e.g. pre-exposed fixed BAR).
*/
struct pci_epf_doorbell_msg {
struct msi_msg msg;
int virq;
unsigned long irq_flags;
enum pci_epf_doorbell_type type;
enum pci_barno bar;
resource_size_t offset;
dma_addr_t iova_base;
size_t iova_size;
};
/**