mirror of
https://github.com/torvalds/linux.git
synced 2026-05-27 00:22:00 +02:00
Merge branch 'net-ntb_netdev-add-multi-queue-support'
Koichiro Den says:
====================
net: ntb_netdev: Add Multi-queue support
ntb_netdev currently hard-codes a single NTB transport queue pair, which
means the datapath effectively runs as a single-queue netdev regardless
of available CPUs / parallel flows.
The longer-term motivation here is throughput scale-out: allow
ntb_netdev to grow beyond the single-QP bottleneck and make it possible
to spread TX/RX work across multiple queue pairs as link speeds and core
counts keep increasing.
Multi-queue also unlocks the standard networking knobs on top of it. In
particular, once the device exposes multiple TX queues, qdisc/tc can
steer flows/traffic classes into different queues (via
skb->queue_mapping), enabling per-flow/per-class scheduling and QoS in a
familiar way.
Usage
=====
1. Ensure the NTB device you want to use has multiple Memory Windows.
2. modprobe ntb_transport on both sides, if it's not built-in.
3. modprobe ntb_netdev on both sides, if it's not built-in.
4. Use ethtool -L to configure the desired number of queues.
The default number of real (combined) queues is 1.
e.g. ethtool -L eth0 combined 2 # to increase
ethtool -L eth0 combined 1 # to reduce back to 1
Note:
* If the NTB device has only a single Memory Window, ethtool -L eth0
combined N (N > 1) fails with:
"netlink error: No space left on device".
* ethtool -L can be executed while the net_device is up.
Compatibility
=============
The default remains a single queue, so behavior is unchanged unless
the user explicitly increases the number of queues.
Kernel base
===========
ntb-next (latest as of 2026-03-06):
commit 7b3302c687ca ("ntb_hw_amd: Fix incorrect debug message in link
disable path")
Testing / Results
=================
Environment / command line:
- 2x R-Car S4 Spider boards
"Kernel base" (see above) + this series
TCP:
[RC] $ sudo iperf3 -s
[EP] $ sudo iperf3 -Z -c ${SERVER_IP} -l 65480 -w 512M -P 4
UDP:
[RC] $ sudo iperf3 -s
[EP] $ sudo iperf3 -ub0 -c ${SERVER_IP} -l 65480 -w 512M -P 4
Without this series:
TCP / UDP : 589 Mbps / 580 Mbps
With this series (default single queue):
TCP / UDP : 583 Mbps / 583 Mbps
With this series + `ethtool -L eth0 combined 2`:
TCP / UDP : 576 Mbps / 584 Mbps
With this series + `ethtool -L eth0 combined 2` + [1], where flows are
properly distributed across queues:
TCP / UDP : 1.13 Gbps / 1.16 Gbps (re-measured with v3)
The 575~590 Mbps variation is run-to-run variance i.e. no measurable
regression or improvement is observed with a single queue. The key
point is scaling from ~600 Mbps to ~1.20 Gbps once flows are
distributed across multiple queues.
Note: On R-Car S4 Spider, only BAR2 is usable for ntb_transport MW.
For testing, BAR2 was expanded from 1 MiB to 2 MiB and split into two
Memory Windows. A follow-up series is planned to add split BAR support
for vNTB. On platforms where multiple BARs can be used for the
datapath, this series should allow >=2 queues without additional
changes.
[1] [PATCH v2 00/10] NTB: epf: Enable per-doorbell bit handling while keeping legacy offset
https://lore.kernel.org/linux-pci/20260227084955.3184017-1-den@valinux.co.jp/
(subject was accidentally incorrect in the original posting)
====================
Link: https://patch.msgid.link/20260305155639.1885517-1-den@valinux.co.jp
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski
commit
f67ab9d810
|
|
@ -53,6 +53,7 @@
|
|||
#include <linux/pci.h>
|
||||
#include <linux/ntb.h>
|
||||
#include <linux/ntb_transport.h>
|
||||
#include <linux/slab.h>
|
||||
|
||||
#define NTB_NETDEV_VER "0.7"
|
||||
|
||||
|
|
@ -70,39 +71,112 @@ static unsigned int tx_start = 10;
|
|||
/* Number of descriptors still available before stop upper layer tx */
|
||||
static unsigned int tx_stop = 5;
|
||||
|
||||
struct ntb_netdev {
|
||||
struct pci_dev *pdev;
|
||||
struct net_device *ndev;
|
||||
#define NTB_NETDEV_MAX_QUEUES 64
|
||||
#define NTB_NETDEV_DEFAULT_QUEUES 1
|
||||
|
||||
struct ntb_netdev;
|
||||
|
||||
struct ntb_netdev_queue {
|
||||
struct ntb_netdev *ntdev;
|
||||
struct ntb_transport_qp *qp;
|
||||
struct timer_list tx_timer;
|
||||
u16 qid;
|
||||
};
|
||||
|
||||
struct ntb_netdev {
|
||||
struct pci_dev *pdev;
|
||||
struct device *client_dev;
|
||||
struct net_device *ndev;
|
||||
unsigned int num_queues;
|
||||
struct ntb_netdev_queue *queues;
|
||||
};
|
||||
|
||||
#define NTB_TX_TIMEOUT_MS 1000
|
||||
#define NTB_RXQ_SIZE 100
|
||||
|
||||
static void ntb_netdev_update_carrier(struct ntb_netdev *dev)
|
||||
{
|
||||
struct net_device *ndev;
|
||||
bool any_up = false;
|
||||
unsigned int i;
|
||||
|
||||
ndev = dev->ndev;
|
||||
|
||||
for (i = 0; i < dev->num_queues; i++) {
|
||||
if (ntb_transport_link_query(dev->queues[i].qp)) {
|
||||
any_up = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (any_up)
|
||||
netif_carrier_on(ndev);
|
||||
else
|
||||
netif_carrier_off(ndev);
|
||||
}
|
||||
|
||||
static void ntb_netdev_queue_rx_drain(struct ntb_netdev_queue *queue)
|
||||
{
|
||||
struct sk_buff *skb;
|
||||
int len;
|
||||
|
||||
while ((skb = ntb_transport_rx_remove(queue->qp, &len)))
|
||||
dev_kfree_skb(skb);
|
||||
}
|
||||
|
||||
static int ntb_netdev_queue_rx_fill(struct net_device *ndev,
|
||||
struct ntb_netdev_queue *queue)
|
||||
{
|
||||
struct sk_buff *skb;
|
||||
int rc, i;
|
||||
|
||||
for (i = 0; i < NTB_RXQ_SIZE; i++) {
|
||||
skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN);
|
||||
if (!skb)
|
||||
return -ENOMEM;
|
||||
|
||||
rc = ntb_transport_rx_enqueue(queue->qp, skb, skb->data,
|
||||
ndev->mtu + ETH_HLEN);
|
||||
if (rc) {
|
||||
dev_kfree_skb(skb);
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void ntb_netdev_event_handler(void *data, int link_is_up)
|
||||
{
|
||||
struct net_device *ndev = data;
|
||||
struct ntb_netdev *dev = netdev_priv(ndev);
|
||||
struct ntb_netdev_queue *q = data;
|
||||
struct ntb_netdev *dev = q->ntdev;
|
||||
struct net_device *ndev;
|
||||
|
||||
netdev_dbg(ndev, "Event %x, Link %x\n", link_is_up,
|
||||
ntb_transport_link_query(dev->qp));
|
||||
ndev = dev->ndev;
|
||||
|
||||
if (link_is_up) {
|
||||
if (ntb_transport_link_query(dev->qp))
|
||||
netif_carrier_on(ndev);
|
||||
} else {
|
||||
netif_carrier_off(ndev);
|
||||
netdev_dbg(ndev, "Event %x, Link %x, qp %u\n", link_is_up,
|
||||
ntb_transport_link_query(q->qp), q->qid);
|
||||
|
||||
if (netif_running(ndev)) {
|
||||
if (link_is_up)
|
||||
netif_wake_subqueue(ndev, q->qid);
|
||||
else
|
||||
netif_stop_subqueue(ndev, q->qid);
|
||||
}
|
||||
|
||||
ntb_netdev_update_carrier(dev);
|
||||
}
|
||||
|
||||
static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
|
||||
void *data, int len)
|
||||
{
|
||||
struct net_device *ndev = qp_data;
|
||||
struct ntb_netdev_queue *q = qp_data;
|
||||
struct ntb_netdev *dev = q->ntdev;
|
||||
struct net_device *ndev;
|
||||
struct sk_buff *skb;
|
||||
int rc;
|
||||
|
||||
ndev = dev->ndev;
|
||||
skb = data;
|
||||
if (!skb)
|
||||
return;
|
||||
|
|
@ -118,6 +192,7 @@ static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
|
|||
skb_put(skb, len);
|
||||
skb->protocol = eth_type_trans(skb, ndev);
|
||||
skb->ip_summed = CHECKSUM_NONE;
|
||||
skb_record_rx_queue(skb, q->qid);
|
||||
|
||||
if (netif_rx(skb) == NET_RX_DROP) {
|
||||
ndev->stats.rx_errors++;
|
||||
|
|
@ -144,42 +219,46 @@ static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
|
|||
}
|
||||
|
||||
static int __ntb_netdev_maybe_stop_tx(struct net_device *netdev,
|
||||
struct ntb_transport_qp *qp, int size)
|
||||
struct ntb_netdev_queue *q, int size)
|
||||
{
|
||||
struct ntb_netdev *dev = netdev_priv(netdev);
|
||||
netif_stop_subqueue(netdev, q->qid);
|
||||
|
||||
netif_stop_queue(netdev);
|
||||
/* Make sure to see the latest value of ntb_transport_tx_free_entry()
|
||||
* since the queue was last started.
|
||||
*/
|
||||
smp_mb();
|
||||
|
||||
if (likely(ntb_transport_tx_free_entry(qp) < size)) {
|
||||
mod_timer(&dev->tx_timer, jiffies + usecs_to_jiffies(tx_time));
|
||||
if (likely(ntb_transport_tx_free_entry(q->qp) < size)) {
|
||||
mod_timer(&q->tx_timer, jiffies + usecs_to_jiffies(tx_time));
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
netif_start_queue(netdev);
|
||||
/* The subqueue must be kept stopped if the link is down */
|
||||
if (ntb_transport_link_query(q->qp))
|
||||
netif_start_subqueue(netdev, q->qid);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int ntb_netdev_maybe_stop_tx(struct net_device *ndev,
|
||||
struct ntb_transport_qp *qp, int size)
|
||||
struct ntb_netdev_queue *q, int size)
|
||||
{
|
||||
if (netif_queue_stopped(ndev) ||
|
||||
(ntb_transport_tx_free_entry(qp) >= size))
|
||||
if (__netif_subqueue_stopped(ndev, q->qid) ||
|
||||
(ntb_transport_tx_free_entry(q->qp) >= size))
|
||||
return 0;
|
||||
|
||||
return __ntb_netdev_maybe_stop_tx(ndev, qp, size);
|
||||
return __ntb_netdev_maybe_stop_tx(ndev, q, size);
|
||||
}
|
||||
|
||||
static void ntb_netdev_tx_handler(struct ntb_transport_qp *qp, void *qp_data,
|
||||
void *data, int len)
|
||||
{
|
||||
struct net_device *ndev = qp_data;
|
||||
struct ntb_netdev_queue *q = qp_data;
|
||||
struct ntb_netdev *dev = q->ntdev;
|
||||
struct net_device *ndev;
|
||||
struct sk_buff *skb;
|
||||
struct ntb_netdev *dev = netdev_priv(ndev);
|
||||
|
||||
ndev = dev->ndev;
|
||||
skb = data;
|
||||
if (!skb || !ndev)
|
||||
return;
|
||||
|
|
@ -194,30 +273,41 @@ static void ntb_netdev_tx_handler(struct ntb_transport_qp *qp, void *qp_data,
|
|||
|
||||
dev_kfree_skb_any(skb);
|
||||
|
||||
if (ntb_transport_tx_free_entry(dev->qp) >= tx_start) {
|
||||
if (ntb_transport_tx_free_entry(qp) >= tx_start) {
|
||||
/* Make sure anybody stopping the queue after this sees the new
|
||||
* value of ntb_transport_tx_free_entry()
|
||||
*/
|
||||
smp_mb();
|
||||
if (netif_queue_stopped(ndev))
|
||||
netif_wake_queue(ndev);
|
||||
if (__netif_subqueue_stopped(ndev, q->qid) &&
|
||||
ntb_transport_link_query(q->qp))
|
||||
netif_wake_subqueue(ndev, q->qid);
|
||||
}
|
||||
}
|
||||
|
||||
static const struct ntb_queue_handlers ntb_netdev_handlers = {
|
||||
.tx_handler = ntb_netdev_tx_handler,
|
||||
.rx_handler = ntb_netdev_rx_handler,
|
||||
.event_handler = ntb_netdev_event_handler,
|
||||
};
|
||||
|
||||
static netdev_tx_t ntb_netdev_start_xmit(struct sk_buff *skb,
|
||||
struct net_device *ndev)
|
||||
{
|
||||
struct ntb_netdev *dev = netdev_priv(ndev);
|
||||
u16 qid = skb_get_queue_mapping(skb);
|
||||
struct ntb_netdev_queue *q;
|
||||
int rc;
|
||||
|
||||
ntb_netdev_maybe_stop_tx(ndev, dev->qp, tx_stop);
|
||||
q = &dev->queues[qid];
|
||||
|
||||
rc = ntb_transport_tx_enqueue(dev->qp, skb, skb->data, skb->len);
|
||||
ntb_netdev_maybe_stop_tx(ndev, q, tx_stop);
|
||||
|
||||
rc = ntb_transport_tx_enqueue(q->qp, skb, skb->data, skb->len);
|
||||
if (rc)
|
||||
goto err;
|
||||
|
||||
/* check for next submit */
|
||||
ntb_netdev_maybe_stop_tx(ndev, dev->qp, tx_stop);
|
||||
ntb_netdev_maybe_stop_tx(ndev, q, tx_stop);
|
||||
|
||||
return NETDEV_TX_OK;
|
||||
|
||||
|
|
@ -229,69 +319,77 @@ static netdev_tx_t ntb_netdev_start_xmit(struct sk_buff *skb,
|
|||
|
||||
static void ntb_netdev_tx_timer(struct timer_list *t)
|
||||
{
|
||||
struct ntb_netdev *dev = timer_container_of(dev, t, tx_timer);
|
||||
struct net_device *ndev = dev->ndev;
|
||||
struct ntb_netdev_queue *q = timer_container_of(q, t, tx_timer);
|
||||
struct ntb_netdev *dev = q->ntdev;
|
||||
struct net_device *ndev;
|
||||
|
||||
if (ntb_transport_tx_free_entry(dev->qp) < tx_stop) {
|
||||
mod_timer(&dev->tx_timer, jiffies + usecs_to_jiffies(tx_time));
|
||||
ndev = dev->ndev;
|
||||
|
||||
if (ntb_transport_tx_free_entry(q->qp) < tx_stop) {
|
||||
mod_timer(&q->tx_timer, jiffies + usecs_to_jiffies(tx_time));
|
||||
} else {
|
||||
/* Make sure anybody stopping the queue after this sees the new
|
||||
* value of ntb_transport_tx_free_entry()
|
||||
*/
|
||||
smp_mb();
|
||||
if (netif_queue_stopped(ndev))
|
||||
netif_wake_queue(ndev);
|
||||
|
||||
/* The subqueue must be kept stopped if the link is down */
|
||||
if (__netif_subqueue_stopped(ndev, q->qid) &&
|
||||
ntb_transport_link_query(q->qp))
|
||||
netif_wake_subqueue(ndev, q->qid);
|
||||
}
|
||||
}
|
||||
|
||||
static int ntb_netdev_open(struct net_device *ndev)
|
||||
{
|
||||
struct ntb_netdev *dev = netdev_priv(ndev);
|
||||
struct sk_buff *skb;
|
||||
int rc, i, len;
|
||||
struct ntb_netdev_queue *queue;
|
||||
unsigned int q;
|
||||
int rc = 0;
|
||||
|
||||
/* Add some empty rx bufs */
|
||||
for (i = 0; i < NTB_RXQ_SIZE; i++) {
|
||||
skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN);
|
||||
if (!skb) {
|
||||
rc = -ENOMEM;
|
||||
goto err;
|
||||
}
|
||||
/* Add some empty rx bufs for each queue */
|
||||
for (q = 0; q < dev->num_queues; q++) {
|
||||
queue = &dev->queues[q];
|
||||
|
||||
rc = ntb_transport_rx_enqueue(dev->qp, skb, skb->data,
|
||||
ndev->mtu + ETH_HLEN);
|
||||
if (rc) {
|
||||
dev_kfree_skb(skb);
|
||||
rc = ntb_netdev_queue_rx_fill(ndev, queue);
|
||||
if (rc)
|
||||
goto err;
|
||||
}
|
||||
|
||||
timer_setup(&queue->tx_timer, ntb_netdev_tx_timer, 0);
|
||||
}
|
||||
|
||||
timer_setup(&dev->tx_timer, ntb_netdev_tx_timer, 0);
|
||||
|
||||
netif_carrier_off(ndev);
|
||||
ntb_transport_link_up(dev->qp);
|
||||
netif_start_queue(ndev);
|
||||
netif_tx_stop_all_queues(ndev);
|
||||
|
||||
for (q = 0; q < dev->num_queues; q++)
|
||||
ntb_transport_link_up(dev->queues[q].qp);
|
||||
|
||||
return 0;
|
||||
|
||||
err:
|
||||
while ((skb = ntb_transport_rx_remove(dev->qp, &len)))
|
||||
dev_kfree_skb(skb);
|
||||
for (q = 0; q < dev->num_queues; q++) {
|
||||
queue = &dev->queues[q];
|
||||
ntb_netdev_queue_rx_drain(queue);
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int ntb_netdev_close(struct net_device *ndev)
|
||||
{
|
||||
struct ntb_netdev *dev = netdev_priv(ndev);
|
||||
struct sk_buff *skb;
|
||||
int len;
|
||||
struct ntb_netdev_queue *queue;
|
||||
unsigned int q;
|
||||
|
||||
ntb_transport_link_down(dev->qp);
|
||||
netif_tx_stop_all_queues(ndev);
|
||||
netif_carrier_off(ndev);
|
||||
|
||||
while ((skb = ntb_transport_rx_remove(dev->qp, &len)))
|
||||
dev_kfree_skb(skb);
|
||||
for (q = 0; q < dev->num_queues; q++) {
|
||||
queue = &dev->queues[q];
|
||||
|
||||
timer_delete_sync(&dev->tx_timer);
|
||||
ntb_transport_link_down(queue->qp);
|
||||
ntb_netdev_queue_rx_drain(queue);
|
||||
timer_delete_sync(&queue->tx_timer);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -299,10 +397,12 @@ static int ntb_netdev_close(struct net_device *ndev)
|
|||
static int ntb_netdev_change_mtu(struct net_device *ndev, int new_mtu)
|
||||
{
|
||||
struct ntb_netdev *dev = netdev_priv(ndev);
|
||||
struct ntb_netdev_queue *queue;
|
||||
struct sk_buff *skb;
|
||||
int len, rc;
|
||||
unsigned int q, i;
|
||||
int len, rc = 0;
|
||||
|
||||
if (new_mtu > ntb_transport_max_size(dev->qp) - ETH_HLEN)
|
||||
if (new_mtu > ntb_transport_max_size(dev->queues[0].qp) - ETH_HLEN)
|
||||
return -EINVAL;
|
||||
|
||||
if (!netif_running(ndev)) {
|
||||
|
|
@ -311,41 +411,53 @@ static int ntb_netdev_change_mtu(struct net_device *ndev, int new_mtu)
|
|||
}
|
||||
|
||||
/* Bring down the link and dispose of posted rx entries */
|
||||
ntb_transport_link_down(dev->qp);
|
||||
for (q = 0; q < dev->num_queues; q++)
|
||||
ntb_transport_link_down(dev->queues[q].qp);
|
||||
|
||||
if (ndev->mtu < new_mtu) {
|
||||
int i;
|
||||
for (q = 0; q < dev->num_queues; q++) {
|
||||
queue = &dev->queues[q];
|
||||
|
||||
for (i = 0; (skb = ntb_transport_rx_remove(dev->qp, &len)); i++)
|
||||
dev_kfree_skb(skb);
|
||||
|
||||
for (; i; i--) {
|
||||
skb = netdev_alloc_skb(ndev, new_mtu + ETH_HLEN);
|
||||
if (!skb) {
|
||||
rc = -ENOMEM;
|
||||
goto err;
|
||||
}
|
||||
|
||||
rc = ntb_transport_rx_enqueue(dev->qp, skb, skb->data,
|
||||
new_mtu + ETH_HLEN);
|
||||
if (rc) {
|
||||
for (i = 0;
|
||||
(skb = ntb_transport_rx_remove(queue->qp, &len));
|
||||
i++)
|
||||
dev_kfree_skb(skb);
|
||||
goto err;
|
||||
|
||||
for (; i; i--) {
|
||||
skb = netdev_alloc_skb(ndev,
|
||||
new_mtu + ETH_HLEN);
|
||||
if (!skb) {
|
||||
rc = -ENOMEM;
|
||||
goto err;
|
||||
}
|
||||
|
||||
rc = ntb_transport_rx_enqueue(queue->qp, skb,
|
||||
skb->data,
|
||||
new_mtu +
|
||||
ETH_HLEN);
|
||||
if (rc) {
|
||||
dev_kfree_skb(skb);
|
||||
goto err;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
WRITE_ONCE(ndev->mtu, new_mtu);
|
||||
|
||||
ntb_transport_link_up(dev->qp);
|
||||
for (q = 0; q < dev->num_queues; q++)
|
||||
ntb_transport_link_up(dev->queues[q].qp);
|
||||
|
||||
return 0;
|
||||
|
||||
err:
|
||||
ntb_transport_link_down(dev->qp);
|
||||
for (q = 0; q < dev->num_queues; q++) {
|
||||
struct ntb_netdev_queue *queue = &dev->queues[q];
|
||||
|
||||
while ((skb = ntb_transport_rx_remove(dev->qp, &len)))
|
||||
dev_kfree_skb(skb);
|
||||
ntb_transport_link_down(queue->qp);
|
||||
|
||||
ntb_netdev_queue_rx_drain(queue);
|
||||
}
|
||||
|
||||
netdev_err(ndev, "Error changing MTU, device inoperable\n");
|
||||
return rc;
|
||||
|
|
@ -386,16 +498,155 @@ static int ntb_get_link_ksettings(struct net_device *dev,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void ntb_get_channels(struct net_device *ndev,
|
||||
struct ethtool_channels *channels)
|
||||
{
|
||||
struct ntb_netdev *dev = netdev_priv(ndev);
|
||||
|
||||
channels->combined_count = dev->num_queues;
|
||||
channels->max_combined = ndev->num_tx_queues;
|
||||
}
|
||||
|
||||
static int ntb_inc_channels(struct net_device *ndev,
|
||||
unsigned int old, unsigned int new)
|
||||
{
|
||||
struct ntb_netdev *dev = netdev_priv(ndev);
|
||||
bool running = netif_running(ndev);
|
||||
struct ntb_netdev_queue *queue;
|
||||
unsigned int q, created;
|
||||
int rc;
|
||||
|
||||
created = old;
|
||||
for (q = old; q < new; q++) {
|
||||
queue = &dev->queues[q];
|
||||
|
||||
queue->ntdev = dev;
|
||||
queue->qid = q;
|
||||
queue->qp = ntb_transport_create_queue(queue, dev->client_dev,
|
||||
&ntb_netdev_handlers);
|
||||
if (!queue->qp) {
|
||||
rc = -ENOSPC;
|
||||
goto err_new;
|
||||
}
|
||||
created++;
|
||||
|
||||
if (!running)
|
||||
continue;
|
||||
|
||||
timer_setup(&queue->tx_timer, ntb_netdev_tx_timer, 0);
|
||||
|
||||
rc = ntb_netdev_queue_rx_fill(ndev, queue);
|
||||
if (rc)
|
||||
goto err_new;
|
||||
|
||||
/*
|
||||
* Carrier may already be on due to other QPs. Keep the new
|
||||
* subqueue stopped until we get a Link Up event for this QP.
|
||||
*/
|
||||
netif_stop_subqueue(ndev, q);
|
||||
}
|
||||
|
||||
rc = netif_set_real_num_queues(ndev, new, new);
|
||||
if (rc)
|
||||
goto err_new;
|
||||
|
||||
dev->num_queues = new;
|
||||
|
||||
if (running)
|
||||
for (q = old; q < new; q++)
|
||||
ntb_transport_link_up(dev->queues[q].qp);
|
||||
|
||||
return 0;
|
||||
|
||||
err_new:
|
||||
if (running) {
|
||||
unsigned int rollback = created;
|
||||
|
||||
while (rollback-- > old) {
|
||||
queue = &dev->queues[rollback];
|
||||
ntb_transport_link_down(queue->qp);
|
||||
ntb_netdev_queue_rx_drain(queue);
|
||||
timer_delete_sync(&queue->tx_timer);
|
||||
}
|
||||
}
|
||||
while (created-- > old) {
|
||||
queue = &dev->queues[created];
|
||||
ntb_transport_free_queue(queue->qp);
|
||||
queue->qp = NULL;
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int ntb_dec_channels(struct net_device *ndev,
|
||||
unsigned int old, unsigned int new)
|
||||
{
|
||||
struct ntb_netdev *dev = netdev_priv(ndev);
|
||||
bool running = netif_running(ndev);
|
||||
struct ntb_netdev_queue *queue;
|
||||
unsigned int q;
|
||||
int rc;
|
||||
|
||||
if (running)
|
||||
for (q = new; q < old; q++)
|
||||
netif_stop_subqueue(ndev, q);
|
||||
|
||||
rc = netif_set_real_num_queues(ndev, new, new);
|
||||
if (rc)
|
||||
goto err;
|
||||
|
||||
/* Publish new queue count before invalidating QP pointers */
|
||||
dev->num_queues = new;
|
||||
|
||||
for (q = new; q < old; q++) {
|
||||
queue = &dev->queues[q];
|
||||
|
||||
if (running) {
|
||||
ntb_transport_link_down(queue->qp);
|
||||
ntb_netdev_queue_rx_drain(queue);
|
||||
timer_delete_sync(&queue->tx_timer);
|
||||
}
|
||||
|
||||
ntb_transport_free_queue(queue->qp);
|
||||
queue->qp = NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* It might be the case that the removed queues are the only queues that
|
||||
* were up, so see if the global carrier needs to change.
|
||||
*/
|
||||
ntb_netdev_update_carrier(dev);
|
||||
return 0;
|
||||
|
||||
err:
|
||||
if (running) {
|
||||
for (q = new; q < old; q++)
|
||||
netif_wake_subqueue(ndev, q);
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int ntb_set_channels(struct net_device *ndev,
|
||||
struct ethtool_channels *channels)
|
||||
{
|
||||
struct ntb_netdev *dev = netdev_priv(ndev);
|
||||
unsigned int new = channels->combined_count;
|
||||
unsigned int old = dev->num_queues;
|
||||
|
||||
if (new == old)
|
||||
return 0;
|
||||
|
||||
if (new < old)
|
||||
return ntb_dec_channels(ndev, old, new);
|
||||
else
|
||||
return ntb_inc_channels(ndev, old, new);
|
||||
}
|
||||
|
||||
static const struct ethtool_ops ntb_ethtool_ops = {
|
||||
.get_drvinfo = ntb_get_drvinfo,
|
||||
.get_link = ethtool_op_get_link,
|
||||
.get_link_ksettings = ntb_get_link_ksettings,
|
||||
};
|
||||
|
||||
static const struct ntb_queue_handlers ntb_netdev_handlers = {
|
||||
.tx_handler = ntb_netdev_tx_handler,
|
||||
.rx_handler = ntb_netdev_rx_handler,
|
||||
.event_handler = ntb_netdev_event_handler,
|
||||
.get_channels = ntb_get_channels,
|
||||
.set_channels = ntb_set_channels,
|
||||
};
|
||||
|
||||
static int ntb_netdev_probe(struct device *client_dev)
|
||||
|
|
@ -404,6 +655,7 @@ static int ntb_netdev_probe(struct device *client_dev)
|
|||
struct net_device *ndev;
|
||||
struct pci_dev *pdev;
|
||||
struct ntb_netdev *dev;
|
||||
unsigned int q;
|
||||
int rc;
|
||||
|
||||
ntb = dev_ntb(client_dev->parent);
|
||||
|
|
@ -411,7 +663,7 @@ static int ntb_netdev_probe(struct device *client_dev)
|
|||
if (!pdev)
|
||||
return -ENODEV;
|
||||
|
||||
ndev = alloc_etherdev(sizeof(*dev));
|
||||
ndev = alloc_etherdev_mq(sizeof(*dev), NTB_NETDEV_MAX_QUEUES);
|
||||
if (!ndev)
|
||||
return -ENOMEM;
|
||||
|
||||
|
|
@ -420,6 +672,16 @@ static int ntb_netdev_probe(struct device *client_dev)
|
|||
dev = netdev_priv(ndev);
|
||||
dev->ndev = ndev;
|
||||
dev->pdev = pdev;
|
||||
dev->client_dev = client_dev;
|
||||
dev->num_queues = 0;
|
||||
|
||||
dev->queues = kzalloc_objs(*dev->queues, NTB_NETDEV_MAX_QUEUES,
|
||||
GFP_KERNEL);
|
||||
if (!dev->queues) {
|
||||
rc = -ENOMEM;
|
||||
goto err_free_netdev;
|
||||
}
|
||||
|
||||
ndev->features = NETIF_F_HIGHDMA;
|
||||
|
||||
ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
|
||||
|
|
@ -436,26 +698,47 @@ static int ntb_netdev_probe(struct device *client_dev)
|
|||
ndev->min_mtu = 0;
|
||||
ndev->max_mtu = ETH_MAX_MTU;
|
||||
|
||||
dev->qp = ntb_transport_create_queue(ndev, client_dev,
|
||||
&ntb_netdev_handlers);
|
||||
if (!dev->qp) {
|
||||
rc = -EIO;
|
||||
goto err;
|
||||
for (q = 0; q < NTB_NETDEV_DEFAULT_QUEUES; q++) {
|
||||
struct ntb_netdev_queue *queue = &dev->queues[q];
|
||||
|
||||
queue->ntdev = dev;
|
||||
queue->qid = q;
|
||||
queue->qp = ntb_transport_create_queue(queue, client_dev,
|
||||
&ntb_netdev_handlers);
|
||||
if (!queue->qp)
|
||||
break;
|
||||
|
||||
dev->num_queues++;
|
||||
}
|
||||
|
||||
ndev->mtu = ntb_transport_max_size(dev->qp) - ETH_HLEN;
|
||||
if (!dev->num_queues) {
|
||||
rc = -EIO;
|
||||
goto err_free_queues;
|
||||
}
|
||||
|
||||
rc = netif_set_real_num_queues(ndev, dev->num_queues, dev->num_queues);
|
||||
if (rc)
|
||||
goto err_free_qps;
|
||||
|
||||
ndev->mtu = ntb_transport_max_size(dev->queues[0].qp) - ETH_HLEN;
|
||||
|
||||
rc = register_netdev(ndev);
|
||||
if (rc)
|
||||
goto err1;
|
||||
goto err_free_qps;
|
||||
|
||||
dev_set_drvdata(client_dev, ndev);
|
||||
dev_info(&pdev->dev, "%s created\n", ndev->name);
|
||||
dev_info(&pdev->dev, "%s created with %u queue pairs\n",
|
||||
ndev->name, dev->num_queues);
|
||||
return 0;
|
||||
|
||||
err1:
|
||||
ntb_transport_free_queue(dev->qp);
|
||||
err:
|
||||
err_free_qps:
|
||||
for (q = 0; q < dev->num_queues; q++)
|
||||
ntb_transport_free_queue(dev->queues[q].qp);
|
||||
|
||||
err_free_queues:
|
||||
kfree(dev->queues);
|
||||
|
||||
err_free_netdev:
|
||||
free_netdev(ndev);
|
||||
return rc;
|
||||
}
|
||||
|
|
@ -464,9 +747,13 @@ static void ntb_netdev_remove(struct device *client_dev)
|
|||
{
|
||||
struct net_device *ndev = dev_get_drvdata(client_dev);
|
||||
struct ntb_netdev *dev = netdev_priv(ndev);
|
||||
unsigned int q;
|
||||
|
||||
unregister_netdev(ndev);
|
||||
ntb_transport_free_queue(dev->qp);
|
||||
for (q = 0; q < dev->num_queues; q++)
|
||||
ntb_transport_free_queue(dev->queues[q].qp);
|
||||
|
||||
kfree(dev->queues);
|
||||
free_netdev(ndev);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user