github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-30 18:13:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

RDMA/mana_ib: add support of multiple ports

Browse Source 
If the HW indicates support of multiple ports for rdma, create an IB device
with a port per netdev in the ethernet mana driver.

CM is only available on port 1, but RC QPs are supported on all
ports.

Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
Link: https://patch.msgid.link/1753174515-23634-1-git-send-email-kotaranov@linux.microsoft.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>
This commit is contained in:
Konstantin Taranov 2025-07-22 01:55:15 -07:00 committed by Leon Romanovsky
parent 10d4de4189
commit 60c9a34df2
3 changed files with 69 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
drivers/infiniband/hw/mana/device.c
View File

@ -77,28 +77,31 @@ static int mana_ib_netdev_event(struct notifier_block *this,
struct gdma_context *gc = dev->gdma_dev->gdma_context;
struct mana_context *mc = gc->mana.driver_data;
struct net_device *ndev;
int i;
/* Only process events from our parent device */
if (event_dev != mc->ports[0])
return NOTIFY_DONE;
for (i = 0; i < dev->ib_dev.phys_port_cnt; i++)
if (event_dev == mc->ports[i]) {
switch (event) {
case NETDEV_CHANGEUPPER:
ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
/*
* RDMA core will setup GID based on updated netdev.
* It's not possible to race with the core as rtnl lock is being
* held.
*/
ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
switch (event) {
case NETDEV_CHANGEUPPER:
ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
/*
* RDMA core will setup GID based on updated netdev.
* It's not possible to race with the core as rtnl lock is being
* held.
*/
ib_device_set_netdev(&dev->ib_dev, ndev, 1);
/* mana_get_primary_netdev() returns ndev with refcount held */
if (ndev)
netdev_put(ndev, &dev->dev_tracker);
/* mana_get_primary_netdev() returns ndev with refcount held */
netdev_put(ndev, &dev->dev_tracker);
return NOTIFY_OK;
default:
return NOTIFY_DONE;
}
return NOTIFY_OK;
default:
return NOTIFY_DONE;
}
}
return NOTIFY_DONE;
}
static int mana_ib_probe(struct auxiliary_device *adev,
@ -111,7 +114,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
struct net_device *ndev;
struct mana_ib_dev *dev;
u8 mac_addr[ETH_ALEN];
int ret;
int ret, i;
dev = ib_alloc_device(mana_ib_dev, ib_dev);
if (!dev)
@ -126,34 +129,11 @@ static int mana_ib_probe(struct auxiliary_device *adev,
if (mana_ib_is_rnic(dev)) {
dev->ib_dev.phys_port_cnt = 1;
ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
if (!ndev) {
ret = -ENODEV;
ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1");
goto free_ib_device;
}
ether_addr_copy(mac_addr, ndev->dev_addr);
addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr);
ret = ib_device_set_netdev(&dev->ib_dev, ndev, 1);
/* mana_get_primary_netdev() returns ndev with refcount held */
netdev_put(ndev, &dev->dev_tracker);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
goto free_ib_device;
}
dev->nb.notifier_call = mana_ib_netdev_event;
ret = register_netdevice_notifier(&dev->nb);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d",
ret);
goto free_ib_device;
}
addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr);
ret = mana_ib_gd_query_adapter_caps(dev);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret);
goto deregister_net_notifier;
goto free_ib_device;
}
ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops);
@ -163,16 +143,42 @@ static int mana_ib_probe(struct auxiliary_device *adev,
ret = mana_ib_create_eqs(dev);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret);
goto deregister_net_notifier;
goto free_ib_device;
}
ret = mana_ib_gd_create_rnic_adapter(dev);
if (ret)
goto destroy_eqs;
ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT)
dev->ib_dev.phys_port_cnt = mc->num_ports;
for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) {
ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
if (!ndev) {
ret = -ENODEV;
ibdev_err(&dev->ib_dev,
"Failed to get netdev for IB port %d", i + 1);
goto destroy_rnic;
}
ether_addr_copy(mac_addr, ndev->dev_addr);
ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
/* mana_get_primary_netdev() returns ndev with refcount held */
netdev_put(ndev, &dev->dev_tracker);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
goto destroy_rnic;
}
ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
goto destroy_rnic;
}
}
dev->nb.notifier_call = mana_ib_netdev_event;
ret = register_netdevice_notifier(&dev->nb);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret);
goto destroy_rnic;
}
} else {
@ -188,7 +194,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
MANA_AV_BUFFER_SIZE, 0);
if (!dev->av_pool) {
ret = -ENOMEM;
goto destroy_rnic;
goto deregister_net_notifier;
}
ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
@ -205,15 +211,15 @@ static int mana_ib_probe(struct auxiliary_device *adev,
deallocate_pool:
dma_pool_destroy(dev->av_pool);
deregister_net_notifier:
if (mana_ib_is_rnic(dev))
unregister_netdevice_notifier(&dev->nb);
destroy_rnic:
if (mana_ib_is_rnic(dev))
mana_ib_gd_destroy_rnic_adapter(dev);
destroy_eqs:
if (mana_ib_is_rnic(dev))
mana_ib_destroy_eqs(dev);
deregister_net_notifier:
if (mana_ib_is_rnic(dev))
unregister_netdevice_notifier(&dev->nb);
free_ib_device:
xa_destroy(&dev->qp_table_wq);
ib_dealloc_device(&dev->ib_dev);
@ -227,9 +233,9 @@ static void mana_ib_remove(struct auxiliary_device *adev)
ib_unregister_device(&dev->ib_dev);
dma_pool_destroy(dev->av_pool);
if (mana_ib_is_rnic(dev)) {
unregister_netdevice_notifier(&dev->nb);
mana_ib_gd_destroy_rnic_adapter(dev);
mana_ib_destroy_eqs(dev);
unregister_netdevice_notifier(&dev->nb);
}
xa_destroy(&dev->qp_table_wq);
ib_dealloc_device(&dev->ib_dev);

13
drivers/infiniband/hw/mana/main.c
View File

@ -563,8 +563,14 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
immutable->gid_tbl_len = attr.gid_tbl_len;
if (mana_ib_is_rnic(dev)) {
immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
if (port_num == 1) {
immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
} else {
immutable->core_cap_flags = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP
| RDMA_CORE_CAP_ETH_AH;
immutable->max_mad_size = 0;
}
} else {
immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
}
@ -633,8 +639,9 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
props->pkey_tbl_len = 1;
if (mana_ib_is_rnic(dev)) {
props->gid_tbl_len = 16;
props->port_cap_flags = IB_PORT_CM_SUP;
props->ip_gids = true;
if (port == 1)
props->port_cap_flags = IB_PORT_CM_SUP;
}
return 0;

1
drivers/infiniband/hw/mana/mana_ib.h
View File

@ -220,6 +220,7 @@ struct mana_ib_query_adapter_caps_req {
enum mana_ib_adapter_features {
MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
};
struct mana_ib_query_adapter_caps_resp {