mirror of
https://github.com/torvalds/linux.git
synced 2026-10-07 19:16:02 +02:00
b27a8560ee
Every devmem dmabuf binding today hands the page_pool PAGE_SIZE niovs.
This caps a single RX descriptor at PAGE_SIZE, burning CPU on buffer
churn for large flows.
Add a bind-time netlink attribute, NETDEV_A_DMABUF_RX_PAGE_SIZE, that
lets userspace request a larger niov size. The value must be a power of
two >= PAGE_SIZE.
The TX path is changed to always pass PAGE_SIZE.
Measurements:
Setup: kperf in devmem RX/TX cuda mode, 4 flows, 64 MB messages, 60s,
dctcp, num-rx-queues=4, dmabuf-rx/tx-size-mb=2048, 10 runs per niov
size, mlx5.
CPU Util:
niov net sirq % net idle % app sys % app idle %
----- ---------------- ---------------- ---------------- ----------------
4K 62.38 +/- 8.27 33.40 +/- 7.51 54.15 +/- 10.23 43.67 +/- 10.53
16K 58.91 +/- 5.35 35.23 +/- 5.88 41.05 +/- 8.87 56.42 +/- 9.24
32K 64.12 +/- 0.68 31.09 +/- 1.48 44.54 +/- 3.51 52.63 +/- 3.65
64K 54.69 +/- 5.54 39.67 +/- 5.81 35.47 +/- 3.11 61.97 +/- 3.27
RX app sys % drops ~19% from 4K to 64K.
Throughput:
niov RX dev Gbps RX flow avg Gbps
----- ---------------- -----------------
4K 300.63 +/- 53.21 75.16 +/- 13.30
16K 321.35 +/- 28.20 80.34 +/- 7.05
32K 347.63 +/- 2.20 86.91 +/- 0.55
64K 332.11 +/- 14.26 83.03 +/- 3.56
Throughput seems to increase, but the stdev is pretty wide so could just
be noise.
kperf support (not yet merged):
8837577f92
Acked-by: Stanislav Fomichev <sdf@fomichev.me>
Reviewed-by: Mina Almasry <almasrymina@google.com>
Reviewed-by: Nikolay Aleksandrov <razor@blackwall.org>
Signed-off-by: Bobby Eshleman <bobbyeshleman@meta.com>
Link: https://patch.msgid.link/20260805-tcpdm-large-niovs-v8-1-3e0225e2808c@meta.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
284 lines
9.2 KiB
C
284 lines
9.2 KiB
C
// SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
|
|
/* Do not edit directly, auto-generated from: */
|
|
/* Documentation/netlink/specs/netdev.yaml */
|
|
/* YNL-GEN kernel source */
|
|
/* To regenerate run: tools/net/ynl/ynl-regen.sh */
|
|
|
|
#include <net/netlink.h>
|
|
#include <net/genetlink.h>
|
|
|
|
#include "netdev-genl-gen.h"
|
|
|
|
#include <uapi/linux/netdev.h>
|
|
#include <net/netdev_netlink.h>
|
|
#include <asm/page.h>
|
|
|
|
/* Integer value ranges */
|
|
static const struct netlink_range_validation netdev_a_page_pool_id_range = {
|
|
.min = 1ULL,
|
|
.max = U32_MAX,
|
|
};
|
|
|
|
static const struct netlink_range_validation netdev_a_page_pool_ifindex_range = {
|
|
.min = 1ULL,
|
|
.max = S32_MAX,
|
|
};
|
|
|
|
static const struct netlink_range_validation netdev_a_napi_defer_hard_irqs_range = {
|
|
.max = S32_MAX,
|
|
};
|
|
|
|
static const struct netlink_range_validation netdev_a_dmabuf_rx_page_size_range = {
|
|
.min = PAGE_SIZE,
|
|
.max = U32_MAX,
|
|
};
|
|
|
|
/* Common nested types */
|
|
const struct nla_policy netdev_lease_nl_policy[NETDEV_A_LEASE_NETNS_ID + 1] = {
|
|
[NETDEV_A_LEASE_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
|
|
[NETDEV_A_LEASE_QUEUE] = NLA_POLICY_NESTED(netdev_queue_id_nl_policy),
|
|
[NETDEV_A_LEASE_NETNS_ID] = NLA_POLICY_MIN(NLA_S32, 0),
|
|
};
|
|
|
|
const struct nla_policy netdev_page_pool_info_nl_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1] = {
|
|
[NETDEV_A_PAGE_POOL_ID] = NLA_POLICY_FULL_RANGE(NLA_UINT, &netdev_a_page_pool_id_range),
|
|
[NETDEV_A_PAGE_POOL_IFINDEX] = NLA_POLICY_FULL_RANGE(NLA_U32, &netdev_a_page_pool_ifindex_range),
|
|
};
|
|
|
|
const struct nla_policy netdev_queue_id_nl_policy[NETDEV_A_QUEUE_TYPE + 1] = {
|
|
[NETDEV_A_QUEUE_ID] = { .type = NLA_U32, },
|
|
[NETDEV_A_QUEUE_TYPE] = NLA_POLICY_MAX(NLA_U32, 1),
|
|
};
|
|
|
|
/* NETDEV_CMD_DEV_GET - do */
|
|
static const struct nla_policy netdev_dev_get_nl_policy[NETDEV_A_DEV_IFINDEX + 1] = {
|
|
[NETDEV_A_DEV_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
|
|
};
|
|
|
|
/* NETDEV_CMD_PAGE_POOL_GET - do */
|
|
#ifdef CONFIG_PAGE_POOL
|
|
static const struct nla_policy netdev_page_pool_get_do_nl_policy[NETDEV_A_PAGE_POOL_ID + 1] = {
|
|
[NETDEV_A_PAGE_POOL_ID] = NLA_POLICY_FULL_RANGE(NLA_UINT, &netdev_a_page_pool_id_range),
|
|
};
|
|
#endif /* CONFIG_PAGE_POOL */
|
|
|
|
/* NETDEV_CMD_PAGE_POOL_GET - dump */
|
|
#ifdef CONFIG_PAGE_POOL
|
|
static const struct nla_policy netdev_page_pool_get_dump_nl_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1] = {
|
|
[NETDEV_A_PAGE_POOL_IFINDEX] = NLA_POLICY_FULL_RANGE(NLA_U32, &netdev_a_page_pool_ifindex_range),
|
|
};
|
|
#endif /* CONFIG_PAGE_POOL */
|
|
|
|
/* NETDEV_CMD_PAGE_POOL_STATS_GET - do */
|
|
#ifdef CONFIG_PAGE_POOL_STATS
|
|
static const struct nla_policy netdev_page_pool_stats_get_do_nl_policy[NETDEV_A_PAGE_POOL_STATS_INFO + 1] = {
|
|
[NETDEV_A_PAGE_POOL_STATS_INFO] = NLA_POLICY_NESTED(netdev_page_pool_info_nl_policy),
|
|
};
|
|
#endif /* CONFIG_PAGE_POOL_STATS */
|
|
|
|
/* NETDEV_CMD_PAGE_POOL_STATS_GET - dump */
|
|
#ifdef CONFIG_PAGE_POOL_STATS
|
|
static const struct nla_policy netdev_page_pool_stats_get_dump_nl_policy[NETDEV_A_PAGE_POOL_STATS_INFO + 1] = {
|
|
[NETDEV_A_PAGE_POOL_STATS_INFO] = NLA_POLICY_NESTED(netdev_page_pool_info_nl_policy),
|
|
};
|
|
#endif /* CONFIG_PAGE_POOL_STATS */
|
|
|
|
/* NETDEV_CMD_QUEUE_GET - do */
|
|
static const struct nla_policy netdev_queue_get_do_nl_policy[NETDEV_A_QUEUE_TYPE + 1] = {
|
|
[NETDEV_A_QUEUE_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
|
|
[NETDEV_A_QUEUE_TYPE] = NLA_POLICY_MAX(NLA_U32, 1),
|
|
[NETDEV_A_QUEUE_ID] = { .type = NLA_U32, },
|
|
};
|
|
|
|
/* NETDEV_CMD_QUEUE_GET - dump */
|
|
static const struct nla_policy netdev_queue_get_dump_nl_policy[NETDEV_A_QUEUE_IFINDEX + 1] = {
|
|
[NETDEV_A_QUEUE_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
|
|
};
|
|
|
|
/* NETDEV_CMD_NAPI_GET - do */
|
|
static const struct nla_policy netdev_napi_get_do_nl_policy[NETDEV_A_NAPI_ID + 1] = {
|
|
[NETDEV_A_NAPI_ID] = { .type = NLA_U32, },
|
|
};
|
|
|
|
/* NETDEV_CMD_NAPI_GET - dump */
|
|
static const struct nla_policy netdev_napi_get_dump_nl_policy[NETDEV_A_NAPI_IFINDEX + 1] = {
|
|
[NETDEV_A_NAPI_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
|
|
};
|
|
|
|
/* NETDEV_CMD_QSTATS_GET - dump */
|
|
static const struct nla_policy netdev_qstats_get_nl_policy[NETDEV_A_QSTATS_SCOPE + 1] = {
|
|
[NETDEV_A_QSTATS_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
|
|
[NETDEV_A_QSTATS_SCOPE] = NLA_POLICY_MASK(NLA_UINT, 0x1),
|
|
};
|
|
|
|
/* NETDEV_CMD_BIND_RX - do */
|
|
static const struct nla_policy netdev_bind_rx_nl_policy[NETDEV_A_DMABUF_RX_PAGE_SIZE + 1] = {
|
|
[NETDEV_A_DMABUF_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
|
|
[NETDEV_A_DMABUF_FD] = { .type = NLA_U32, },
|
|
[NETDEV_A_DMABUF_QUEUES] = NLA_POLICY_NESTED(netdev_queue_id_nl_policy),
|
|
[NETDEV_A_DMABUF_RX_PAGE_SIZE] = NLA_POLICY_FULL_RANGE(NLA_U32, &netdev_a_dmabuf_rx_page_size_range),
|
|
};
|
|
|
|
/* NETDEV_CMD_NAPI_SET - do */
|
|
static const struct nla_policy netdev_napi_set_nl_policy[NETDEV_A_NAPI_THREADED + 1] = {
|
|
[NETDEV_A_NAPI_ID] = { .type = NLA_U32, },
|
|
[NETDEV_A_NAPI_DEFER_HARD_IRQS] = NLA_POLICY_FULL_RANGE(NLA_U32, &netdev_a_napi_defer_hard_irqs_range),
|
|
[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT] = { .type = NLA_UINT, },
|
|
[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT] = { .type = NLA_UINT, },
|
|
[NETDEV_A_NAPI_THREADED] = NLA_POLICY_MAX(NLA_U32, 2),
|
|
};
|
|
|
|
/* NETDEV_CMD_BIND_TX - do */
|
|
static const struct nla_policy netdev_bind_tx_nl_policy[NETDEV_A_DMABUF_FD + 1] = {
|
|
[NETDEV_A_DMABUF_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
|
|
[NETDEV_A_DMABUF_FD] = { .type = NLA_U32, },
|
|
};
|
|
|
|
/* NETDEV_CMD_QUEUE_CREATE - do */
|
|
static const struct nla_policy netdev_queue_create_nl_policy[NETDEV_A_QUEUE_LEASE + 1] = {
|
|
[NETDEV_A_QUEUE_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
|
|
[NETDEV_A_QUEUE_TYPE] = NLA_POLICY_MAX(NLA_U32, 1),
|
|
[NETDEV_A_QUEUE_LEASE] = NLA_POLICY_NESTED(netdev_lease_nl_policy),
|
|
};
|
|
|
|
/* Ops table for netdev */
|
|
static const struct genl_split_ops netdev_nl_ops[] = {
|
|
{
|
|
.cmd = NETDEV_CMD_DEV_GET,
|
|
.doit = netdev_nl_dev_get_doit,
|
|
.policy = netdev_dev_get_nl_policy,
|
|
.maxattr = NETDEV_A_DEV_IFINDEX,
|
|
.flags = GENL_CMD_CAP_DO,
|
|
},
|
|
{
|
|
.cmd = NETDEV_CMD_DEV_GET,
|
|
.dumpit = netdev_nl_dev_get_dumpit,
|
|
.flags = GENL_CMD_CAP_DUMP,
|
|
},
|
|
#ifdef CONFIG_PAGE_POOL
|
|
{
|
|
.cmd = NETDEV_CMD_PAGE_POOL_GET,
|
|
.doit = netdev_nl_page_pool_get_doit,
|
|
.policy = netdev_page_pool_get_do_nl_policy,
|
|
.maxattr = NETDEV_A_PAGE_POOL_ID,
|
|
.flags = GENL_CMD_CAP_DO,
|
|
},
|
|
{
|
|
.cmd = NETDEV_CMD_PAGE_POOL_GET,
|
|
.dumpit = netdev_nl_page_pool_get_dumpit,
|
|
.policy = netdev_page_pool_get_dump_nl_policy,
|
|
.maxattr = NETDEV_A_PAGE_POOL_IFINDEX,
|
|
.flags = GENL_CMD_CAP_DUMP,
|
|
},
|
|
#endif /* CONFIG_PAGE_POOL */
|
|
#ifdef CONFIG_PAGE_POOL_STATS
|
|
{
|
|
.cmd = NETDEV_CMD_PAGE_POOL_STATS_GET,
|
|
.doit = netdev_nl_page_pool_stats_get_doit,
|
|
.policy = netdev_page_pool_stats_get_do_nl_policy,
|
|
.maxattr = NETDEV_A_PAGE_POOL_STATS_INFO,
|
|
.flags = GENL_CMD_CAP_DO,
|
|
},
|
|
{
|
|
.cmd = NETDEV_CMD_PAGE_POOL_STATS_GET,
|
|
.dumpit = netdev_nl_page_pool_stats_get_dumpit,
|
|
.policy = netdev_page_pool_stats_get_dump_nl_policy,
|
|
.maxattr = NETDEV_A_PAGE_POOL_STATS_INFO,
|
|
.flags = GENL_CMD_CAP_DUMP,
|
|
},
|
|
#endif /* CONFIG_PAGE_POOL_STATS */
|
|
{
|
|
.cmd = NETDEV_CMD_QUEUE_GET,
|
|
.doit = netdev_nl_queue_get_doit,
|
|
.policy = netdev_queue_get_do_nl_policy,
|
|
.maxattr = NETDEV_A_QUEUE_TYPE,
|
|
.flags = GENL_CMD_CAP_DO,
|
|
},
|
|
{
|
|
.cmd = NETDEV_CMD_QUEUE_GET,
|
|
.dumpit = netdev_nl_queue_get_dumpit,
|
|
.policy = netdev_queue_get_dump_nl_policy,
|
|
.maxattr = NETDEV_A_QUEUE_IFINDEX,
|
|
.flags = GENL_CMD_CAP_DUMP,
|
|
},
|
|
{
|
|
.cmd = NETDEV_CMD_NAPI_GET,
|
|
.doit = netdev_nl_napi_get_doit,
|
|
.policy = netdev_napi_get_do_nl_policy,
|
|
.maxattr = NETDEV_A_NAPI_ID,
|
|
.flags = GENL_CMD_CAP_DO,
|
|
},
|
|
{
|
|
.cmd = NETDEV_CMD_NAPI_GET,
|
|
.dumpit = netdev_nl_napi_get_dumpit,
|
|
.policy = netdev_napi_get_dump_nl_policy,
|
|
.maxattr = NETDEV_A_NAPI_IFINDEX,
|
|
.flags = GENL_CMD_CAP_DUMP,
|
|
},
|
|
{
|
|
.cmd = NETDEV_CMD_QSTATS_GET,
|
|
.dumpit = netdev_nl_qstats_get_dumpit,
|
|
.policy = netdev_qstats_get_nl_policy,
|
|
.maxattr = NETDEV_A_QSTATS_SCOPE,
|
|
.flags = GENL_CMD_CAP_DUMP,
|
|
},
|
|
{
|
|
.cmd = NETDEV_CMD_BIND_RX,
|
|
.doit = netdev_nl_bind_rx_doit,
|
|
.policy = netdev_bind_rx_nl_policy,
|
|
.maxattr = NETDEV_A_DMABUF_RX_PAGE_SIZE,
|
|
.flags = GENL_UNS_ADMIN_PERM | GENL_CMD_CAP_DO,
|
|
},
|
|
{
|
|
.cmd = NETDEV_CMD_NAPI_SET,
|
|
.doit = netdev_nl_napi_set_doit,
|
|
.policy = netdev_napi_set_nl_policy,
|
|
.maxattr = NETDEV_A_NAPI_THREADED,
|
|
.flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
|
|
},
|
|
{
|
|
.cmd = NETDEV_CMD_BIND_TX,
|
|
.doit = netdev_nl_bind_tx_doit,
|
|
.policy = netdev_bind_tx_nl_policy,
|
|
.maxattr = NETDEV_A_DMABUF_FD,
|
|
.flags = GENL_CMD_CAP_DO,
|
|
},
|
|
{
|
|
.cmd = NETDEV_CMD_QUEUE_CREATE,
|
|
.doit = netdev_nl_queue_create_doit,
|
|
.policy = netdev_queue_create_nl_policy,
|
|
.maxattr = NETDEV_A_QUEUE_LEASE,
|
|
.flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
|
|
},
|
|
};
|
|
|
|
static const struct genl_multicast_group netdev_nl_mcgrps[] = {
|
|
[NETDEV_NLGRP_MGMT] = { "mgmt", },
|
|
[NETDEV_NLGRP_PAGE_POOL] = { "page-pool", },
|
|
};
|
|
|
|
static void __netdev_nl_sock_priv_init(void *priv)
|
|
{
|
|
netdev_nl_sock_priv_init(priv);
|
|
}
|
|
|
|
static void __netdev_nl_sock_priv_destroy(void *priv)
|
|
{
|
|
netdev_nl_sock_priv_destroy(priv);
|
|
}
|
|
|
|
struct genl_family netdev_nl_family __ro_after_init = {
|
|
.name = NETDEV_FAMILY_NAME,
|
|
.version = NETDEV_FAMILY_VERSION,
|
|
.netnsok = true,
|
|
.parallel_ops = true,
|
|
.module = THIS_MODULE,
|
|
.split_ops = netdev_nl_ops,
|
|
.n_split_ops = ARRAY_SIZE(netdev_nl_ops),
|
|
.mcgrps = netdev_nl_mcgrps,
|
|
.n_mcgrps = ARRAY_SIZE(netdev_nl_mcgrps),
|
|
.sock_priv_size = sizeof(struct netdev_nl_sock),
|
|
.sock_priv_init = __netdev_nl_sock_priv_init,
|
|
.sock_priv_destroy = __netdev_nl_sock_priv_destroy,
|
|
};
|