github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'hardware-pacing-offload'

Browse Source 
Willem de Bruijn says:

====================
hardware pacing offload

Enable safe hardware pacing offload on a first device (idpf), plus support.

- Add rtnetlink support for admin control over the device feature
- Add device support to the idpf driver

Besides the main feature, also

- Small optimization to FQ offload: avoid unnecessary ktime_.._get().
- Expand so_txtime drv-net test with hw offload testcases.

Details and detailed changelog in the individual patches.
====================

Link: https://patch.msgid.link/20260910171131.2532487-1-willemdebruijn.kernel@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski 2026-09-14 17:42:35 -07:00
commit ad6ff1a67b
13 changed files with 444 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Documentation/netlink/specs/rt-link.yaml
View File

@ -1091,6 +1091,10 @@ attribute-sets:
-
name: tailroom
type: u16
-
name: pacing-offload
type: u32
doc: Enable EDT pacing offload (0 - disabled, 1 - enabled).
-
name: prop-list-link-attrs
subset-of: link-attrs
@ -2559,6 +2563,7 @@ operations:
- devlink-port
- gso-ipv4-max-size
- gro-ipv4-max-size
- pacing-offload
dump:
request:
value: 18

1
Documentation/networking/net_cachelines/net_device.rst
View File

@ -11,6 +11,7 @@ Type Name fastpath_tx_acce
unsigned_long:32 priv_flags read_mostly __dev_queue_xmit(tx)
unsigned_long:1 lltx read_mostly HARD_TX_LOCK,HARD_TX_TRYLOCK,HARD_TX_UNLOCK(tx)
unsigned_long:2 netmem_tx:2; read_mostly
unsigned_long:1 pacing_offload read_mostly sch_fq
char name[16]
struct netdev_name_node* name_node
struct dev_ifalias* ifalias

12
drivers/net/ethernet/intel/idpf/idpf.h
View File

@ -604,6 +604,16 @@ struct idpf_vport_config {
DECLARE_BITMAP(flags, IDPF_VPORT_CONFIG_FLAGS_NBITS);
};
/**
* struct idpf_edt_caps_ilog2 - Host parsed EDT capabilities.
* @time_horizon_ns: Total time window in nanoseconds.
* @tstamp_granularity_pow2: Log2 of timestamp granularity in nanoseconds.
*/
struct idpf_edt_caps_ilog2 {
u32 time_horizon_ns;
u8 tstamp_granularity_pow2;
};
#define idpf_for_each_vport(adapter, iter) \
for (struct idpf_vport **__##iter = &(adapter)->vports[0], \
*iter = (adapter)->max_vports ? *__##iter : NULL; \
@ -657,6 +667,7 @@ struct idpf_vport_config {
* @stats_task: Periodic statistics retrieval task
* @stats_wq: Workqueue for statistics task
* @caps: Negotiated capabilities with device
* @edt_caps: Negotiated EDT capabilities with device
* @dev_ops: See idpf_dev_ops
* @cdev_info: IDC core device info pointer
* @num_vfs: Number of allocated VFs through sysfs. PF does not directly talk
@ -720,6 +731,7 @@ struct idpf_adapter {
struct delayed_work stats_task;
struct workqueue_struct *stats_wq;
struct virtchnl2_get_capabilities caps;
struct idpf_edt_caps_ilog2 edt_caps;
struct idpf_dev_ops dev_ops;
struct iidc_rdma_core_dev_info *cdev_info;

19
drivers/net/ethernet/intel/idpf/idpf_lib.c
View File

@ -817,6 +817,21 @@ static void idpf_attach_and_open(struct idpf_adapter *adapter)
}
}
static void idpf_vport_set_pacing_offload(struct idpf_vport *vport,
struct net_device *netdev)
{
struct idpf_adapter *adapter = vport->adapter;
u32 max_horizon = 0;
if (adapter->edt_caps.time_horizon_ns &&
idpf_is_queue_model_split(vport->dflt_qv_rsrc.txq_model) &&
!idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS,
VIRTCHNL2_CAP_SPLITQ_QSCHED))
max_horizon = adapter->edt_caps.time_horizon_ns;
WRITE_ONCE(netdev->max_pacing_offload_horizon, max_horizon);
}
/**
* idpf_cfg_netdev - Allocate, configure and register a netdev
* @vport: main vport structure
@ -850,6 +865,8 @@ static int idpf_cfg_netdev(struct idpf_vport *vport)
np->max_tx_hdr_size = idpf_get_max_tx_hdr_size(adapter);
vport->netdev = netdev;
idpf_vport_set_pacing_offload(vport, netdev);
return idpf_init_mac_addr(vport, netdev);
}
@ -890,6 +907,8 @@ static int idpf_cfg_netdev(struct idpf_vport *vport)
netdev->min_mtu = ETH_MIN_MTU;
netdev->max_mtu = vport->max_mtu;
idpf_vport_set_pacing_offload(vport, netdev);
dflt_features = NETIF_F_SG |
NETIF_F_HIGHDMA;

69
drivers/net/ethernet/intel/idpf/idpf_txrx.c
View File

@ -1742,6 +1742,7 @@ static int idpf_txq_group_alloc(struct idpf_vport *vport,
q->desc_count = rsrc->txq_desc_count;
q->tx_max_bufs = idpf_get_max_tx_bufs(adapter);
q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter);
q->ts_gran_pow2 = adapter->edt_caps.tstamp_granularity_pow2;
q->netdev = vport->netdev;
q->txq_grp = tx_qgrp;
q->rel_q_id = j;
@ -2408,7 +2409,12 @@ void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc,
struct idpf_tx_splitq_params *params,
u16 td_cmd, u16 size)
{
*(__le32 *)&desc->flow.qw1.cmd_dtype = cpu_to_le32((u8)(params->dtype | td_cmd));
desc->flow.qw1.cmd_dtype = (u8)(params->dtype | td_cmd);
desc->flow.qw1.ts[0] = params->offload.desc_ts[0];
desc->flow.qw1.ts[1] = params->offload.desc_ts[1];
desc->flow.qw1.ts[2] = params->offload.desc_ts[2];
desc->flow.qw1.rxr_bufsize = cpu_to_le16((u16)size);
desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag);
}
@ -3011,6 +3017,63 @@ static bool idpf_tx_splitq_need_re(struct idpf_tx_queue *tx_q)
return gap >= IDPF_TX_SPLITQ_RE_MIN_GAP;
}
static void idpf_tx_splitq_set_txtime(const struct sk_buff *skb,
const struct idpf_tx_queue *tx_q,
struct idpf_tx_splitq_params *tx_params)
{
const int offload_slack_ns = 400;
u64 ts, now, horizon;
if (!tx_q->netdev->pacing_offload)
return;
horizon = READ_ONCE(tx_q->netdev->max_pacing_offload_horizon);
if (!horizon)
return;
switch (skb->tstamp_type) {
case SKB_CLOCK_REALTIME:
ts = ktime_to_ns(ktime_add(skb->tstamp,
ktime_mono_to_any(0, TK_OFFS_TAI) -
ktime_mono_to_any(0, TK_OFFS_REAL)));
break;
case SKB_CLOCK_MONOTONIC:
ts = ktime_to_ns(ktime_mono_to_any(skb->tstamp, TK_OFFS_TAI));
break;
case SKB_CLOCK_TAI:
ts = ktime_to_ns(skb->tstamp);
break;
default:
WARN_ON_ONCE(1);
return;
}
now = ktime_get_clocktai_ns();
if (ts < now + offload_slack_ns)
return;
/* beyond offload horizon? set overflow bit only */
if (ts > now + horizon) {
tx_params->offload.desc_ts[2] =
IDPF_TXD_FLOW_SCH_HORIZON_OVERFLOW_M;
return;
}
ts >>= tx_q->ts_gran_pow2;
/* 0 is valid 23b timestamp, but also means field unset.
* Increase by one to avoid this case
*/
if ((ts & 0x7fffff) == 0) {
tx_params->offload.desc_ts[0] = 1;
return;
}
tx_params->offload.desc_ts[0] = ts & 0xff;
tx_params->offload.desc_ts[1] = (ts >> 8) & 0xff;
tx_params->offload.desc_ts[2] = ((ts >> 16) & 0x7f);
}
/**
* idpf_tx_splitq_frame - Sends buffer on Tx ring using flex descriptors
* @skb: send buffer
@ -3097,6 +3160,10 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb,
tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_FLOW_SCHE;
tx_params.eop_cmd = IDPF_TXD_FLEX_FLOW_CMD_EOP;
if (skb->tstamp)
idpf_tx_splitq_set_txtime(skb, tx_q, &tx_params);
/* Set the RE bit periodically to "clean" the descriptor ring */
if (idpf_tx_splitq_need_re(tx_q)) {
tx_params.eop_cmd |= IDPF_TXD_FLEX_FLOW_CMD_RE;

8
drivers/net/ethernet/intel/idpf/idpf_txrx.h
View File

@ -161,6 +161,7 @@ union idpf_tx_flex_desc {
* @tso_segs: Number of segments to be sent
* @tso_hdr_len: Length of headers to be duplicated
* @td_cmd: Command field to be inserted into descriptor
* @desc_ts: Flow scheduling offload timestamp
*/
struct idpf_tx_offload_params {
u32 tx_flags;
@ -174,6 +175,7 @@ struct idpf_tx_offload_params {
u16 tso_hdr_len;
u16 td_cmd;
u8 desc_ts[3];
};
/**
@ -608,6 +610,7 @@ libeth_cacheline_set_assert(struct idpf_rx_queue,
* hot path TX pointers stored in vport. Used in both singleq/splitq.
* @desc_count: Number of descriptors
* @tx_min_pkt_len: Min supported packet length
* @ts_gran_pow2: Txtime timestamp granularity in nanoseconds (log2).
* @thresh: XDP queue cleaning threshold
* @netdev: &net_device corresponding to this queue
* @next_to_use: Next descriptor to use
@ -666,7 +669,10 @@ struct idpf_tx_queue {
u16 desc_count;
union {
u16 tx_min_pkt_len;
struct {
u16 tx_min_pkt_len;
u8 ts_gran_pow2;
};
u32 thresh;
};

60
drivers/net/ethernet/intel/idpf/idpf_virtchnl.c
View File

@ -3,6 +3,7 @@
#include <linux/export.h>
#include <linux/net/intel/libie/pci.h>
#include <linux/log2.h>
#include <net/libeth/rx.h>
#include "idpf.h"
@ -553,7 +554,8 @@ static int idpf_send_get_caps_msg(struct idpf_adapter *adapter)
VIRTCHNL2_CAP_SPLITQ_QSCHED |
VIRTCHNL2_CAP_PROMISC |
VIRTCHNL2_CAP_LOOPBACK |
VIRTCHNL2_CAP_PTP);
VIRTCHNL2_CAP_PTP |
VIRTCHNL2_CAP_EDT);
err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps);
if (err)
@ -573,6 +575,54 @@ static int idpf_send_get_caps_msg(struct idpf_adapter *adapter)
return err;
}
/**
* idpf_send_get_edt_caps_msg - Send virtchnl get EDT caps msg
* @adapter: Driver specific private struct
*
* Return: 0 on success or error code on failure.
*/
static int idpf_send_get_edt_caps_msg(struct idpf_adapter *adapter)
{
struct libie_ctlq_xn_send_params xn_params = {
.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
.chnl_opcode = VIRTCHNL2_OP_GET_EDT_CAPS,
};
struct virtchnl2_edt_caps caps = {};
u64 gran_ns, horizon_ns;
int err;
err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps);
if (err)
return err;
if (xn_params.recv_mem.iov_len < sizeof(caps)) {
err = -EIO;
goto free_rx_buf;
}
memcpy(&caps, xn_params.recv_mem.iov_base, sizeof(caps));
horizon_ns = le64_to_cpu(caps.time_horizon_ns);
gran_ns = le64_to_cpu(caps.tstamp_granularity_ns);
if (horizon_ns > U32_MAX) {
dev_warn(&adapter->pdev->dev, "EDT horizon exceeds U32\n");
err = -EINVAL;
goto free_rx_buf;
}
if (!gran_ns || !is_power_of_2(gran_ns)) {
dev_warn(&adapter->pdev->dev, "Invalid EDT granularity\n");
err = -EINVAL;
goto free_rx_buf;
}
adapter->edt_caps.time_horizon_ns = horizon_ns;
adapter->edt_caps.tstamp_granularity_pow2 = ilog2(gran_ns);
free_rx_buf:
libie_ctlq_release_rx_buf(&xn_params.recv_mem);
return err;
}
/**
* idpf_mmio_region_non_static - Check if region is not static
* @mmio_info: PCI resources info
@ -3085,6 +3135,14 @@ int idpf_vc_core_init(struct idpf_adapter *adapter)
}
}
memset(&adapter->edt_caps, 0, sizeof(adapter->edt_caps));
if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_EDT)) {
err = idpf_send_get_edt_caps_msg(adapter);
if (err)
dev_err(&adapter->pdev->dev,
"EDT init failed, err=%d\n", err);
}
pci_sriov_set_totalvfs(adapter->pdev, idpf_get_max_vfs(adapter));
num_max_vports = idpf_get_max_vports(adapter);
adapter->vports = kzalloc_objs(*adapter->vports, num_max_vports);

2
include/linux/netdevice.h
View File

@ -1840,6 +1840,7 @@ enum netdev_reg_state {
* drivers. Mainly used by logical interfaces, such as
* bonding and tunnels
* @netmem_tx: device netmem TX mode
* @pacing_offload: enable EDT pacing offload.
*
* @name: This is the first field of the "visible" part of this structure
* (i.e. as seen by users in the "Space.c" file). It is the name
@ -2170,6 +2171,7 @@ struct net_device {
unsigned long priv_flags:32;
unsigned long lltx:1;
unsigned long netmem_tx:2;
unsigned long pacing_offload:1;
);
const struct net_device_ops *netdev_ops;
const struct header_ops *header_ops;

1
include/uapi/linux/if_link.h
View File

@ -381,6 +381,7 @@ enum {
IFLA_NETNS_IMMUTABLE,
IFLA_HEADROOM,
IFLA_TAILROOM,
IFLA_PACING_OFFLOAD,
__IFLA_MAX
};

22
net/core/rtnetlink.c
View File

@ -1396,6 +1396,7 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev,
+ rtnl_devlink_port_size(dev)
+ rtnl_dpll_pin_size()
+ nla_total_size(8) /* IFLA_MAX_PACING_OFFLOAD_HORIZON */
+ nla_total_size(4) /* IFLA_PACING_OFFLOAD */
+ nla_total_size(2) /* IFLA_HEADROOM */
+ nla_total_size(2) /* IFLA_TAILROOM */
+ rtnl_dev_parent_size(dev)
@ -2176,6 +2177,8 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb,
READ_ONCE(dev->tso_max_segs)) ||
nla_put_uint(skb, IFLA_MAX_PACING_OFFLOAD_HORIZON,
READ_ONCE(dev->max_pacing_offload_horizon)) ||
nla_put_u32(skb, IFLA_PACING_OFFLOAD,
dev->pacing_offload) ||
#ifdef CONFIG_RPS
nla_put_u32(skb, IFLA_NUM_RX_QUEUES,
READ_ONCE(dev->num_rx_queues)) ||
@ -2349,9 +2352,11 @@ static const struct nla_policy ifla_policy[IFLA_MAX+1] = {
[IFLA_ALLMULTI] = { .type = NLA_REJECT },
[IFLA_GSO_IPV4_MAX_SIZE] = NLA_POLICY_MIN(NLA_U32, MAX_TCP_HEADER + 1),
[IFLA_GRO_IPV4_MAX_SIZE] = { .type = NLA_U32 },
[IFLA_MAX_PACING_OFFLOAD_HORIZON] = { .type = NLA_REJECT },
[IFLA_NETNS_IMMUTABLE] = { .type = NLA_REJECT },
[IFLA_HEADROOM] = { .type = NLA_REJECT },
[IFLA_TAILROOM] = { .type = NLA_REJECT },
[IFLA_PACING_OFFLOAD] = NLA_POLICY_MAX(NLA_U32, 1),
};
static const struct nla_policy ifla_info_policy[IFLA_INFO_MAX+1] = {
@ -2825,6 +2830,14 @@ static int validate_linkmsg(struct net_device *dev, struct nlattr *tb[],
return -EINVAL;
}
if (tb[IFLA_PACING_OFFLOAD]) {
if (nla_get_u32(tb[IFLA_PACING_OFFLOAD]) &&
!dev->max_pacing_offload_horizon) {
NL_SET_ERR_MSG(extack, "pacing offload not supported by device");
return -EOPNOTSUPP;
}
}
if (tb[IFLA_AF_SPEC]) {
struct nlattr *af;
int rem, err;
@ -3342,6 +3355,15 @@ static int do_setlink(const struct sk_buff *skb, struct net_device *dev,
}
}
if (tb[IFLA_PACING_OFFLOAD]) {
bool val = nla_get_u32(tb[IFLA_PACING_OFFLOAD]);
if (dev->pacing_offload != val) {
dev->pacing_offload = val;
status |= DO_SETLINK_MODIFIED;
}
}
if (tb[IFLA_OPERSTATE])
set_operstate(dev, nla_get_u8(tb[IFLA_OPERSTATE]));

71
net/sched/sch_fq.c
View File

@ -159,6 +159,9 @@ struct fq_sched_data {
u64 stat_allocation_errors;
};
/* EDT timestamps to clear beyond now. */
static const int fq_offload_slack_ns = 400;
/* return the i-th 2-bit value ("crumb") */
static u8 fq_prio2band(const u8 *prio2band, unsigned int prio)
{
@ -301,6 +304,26 @@ static void fq_gc(struct fq_sched_data *q,
q->stat_gc_flows += fcnt;
}
static u64 fq_offload_horizon(const struct Qdisc *sch,
const struct fq_sched_data *q)
{
const struct net_device *dev;
u64 offload_horizon;
offload_horizon = READ_ONCE(q->offload_horizon);
if (!offload_horizon)
return 0;
dev = qdisc_dev(sch);
if (!dev->pacing_offload)
return 0;
if (offload_horizon > READ_ONCE(dev->max_pacing_offload_horizon))
return 0;
return offload_horizon;
}
/* Fast path can be used if :
* 1) Packet tstamp is in the past, or within the pacing offload horizon.
* 2) FQ qlen == 0 OR
@ -312,12 +335,12 @@ static void fq_gc(struct fq_sched_data *q,
* FQ can not use generic TCQ_F_CAN_BYPASS infrastructure.
*/
static bool fq_fastpath_check(const struct Qdisc *sch, struct sk_buff *skb,
u64 now)
u64 now, u64 offload_horizon)
{
const struct fq_sched_data *q = qdisc_priv(sch);
const struct sock *sk;
if (fq_skb_cb(skb)->time_to_send > now + q->offload_horizon)
if (fq_skb_cb(skb)->time_to_send > now + offload_horizon)
return false;
if (sch->q.qlen != 0) {
@ -338,7 +361,7 @@ static bool fq_fastpath_check(const struct Qdisc *sch, struct sk_buff *skb,
/* Ordering invariants fall apart if some delayed flows
* are ready but we haven't serviced them, yet.
*/
if (q->time_next_delayed_flow <= now + q->offload_horizon)
if (q->time_next_delayed_flow <= now + offload_horizon)
return false;
}
@ -357,6 +380,7 @@ static struct fq_flow *fq_classify(struct Qdisc *sch, struct sk_buff *skb,
u64 now)
{
struct fq_sched_data *q = qdisc_priv(sch);
u64 offload_horizon = fq_offload_horizon(sch, q);
struct rb_node **p, *parent;
struct sock *sk = skb->sk;
struct rb_root *root;
@ -393,12 +417,17 @@ static struct fq_flow *fq_classify(struct Qdisc *sch, struct sk_buff *skb,
sk = (struct sock *)((hash << 1) | 1UL);
}
if (fq_fastpath_check(sch, skb, now)) {
if (fq_fastpath_check(sch, skb, now, offload_horizon)) {
q->internal.stat_fastpath_packets++;
if (skb->sk == sk && q->rate_enable &&
READ_ONCE(sk->sk_pacing_status) != SK_PACING_FQ)
smp_store_release(&sk->sk_pacing_status,
SK_PACING_FQ);
if (offload_horizon &&
fq_skb_cb(skb)->time_to_send <= now + fq_offload_slack_ns)
skb_set_delivery_time(skb, 0, SKB_CLOCK_REALTIME);
return &q->internal;
}
@ -661,12 +690,13 @@ static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
return NET_XMIT_SUCCESS;
}
static void fq_check_throttled(struct fq_sched_data *q, u64 now)
static void fq_check_throttled(struct fq_sched_data *q, u64 now,
u64 offload_horizon)
{
unsigned long sample;
struct rb_node *p;
if (q->time_next_delayed_flow > now + q->offload_horizon)
if (q->time_next_delayed_flow > now + offload_horizon)
return;
/* Update unthrottle latency EWMA.
@ -677,7 +707,7 @@ static void fq_check_throttled(struct fq_sched_data *q, u64 now)
q->unthrottle_latency_ns -= q->unthrottle_latency_ns >> 3;
q->unthrottle_latency_ns += sample >> 3;
}
now += q->offload_horizon;
now += offload_horizon;
q->time_next_delayed_flow = ~0ULL;
while ((p = rb_first(&q->delayed)) != NULL) {
@ -705,8 +735,10 @@ static struct fq_flow_head *fq_pband_head_select(struct fq_perband_flows *pband)
static struct sk_buff *fq_dequeue(struct Qdisc *sch)
{
struct fq_sched_data *q = qdisc_priv(sch);
u64 offload_horizon = fq_offload_horizon(sch, q);
struct fq_perband_flows *pband;
struct fq_flow_head *head;
u64 time_next_packet = 0;
struct sk_buff *skb;
struct fq_flow *f;
unsigned long rate;
@ -721,11 +753,11 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
if (skb) {
q->internal.qlen--;
fq_dequeue_skb(sch, &q->internal, skb);
goto out;
return skb;
}
now = ktime_get_ns();
fq_check_throttled(q, now);
fq_check_throttled(q, now, offload_horizon);
retry = 0;
pband = &q->band_flows[q->band_nr];
begin:
@ -758,10 +790,10 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
skb = fq_peek(f);
if (skb) {
u64 time_next_packet = max_t(u64, fq_skb_cb(skb)->time_to_send,
f->time_next_packet);
time_next_packet = max_t(u64, fq_skb_cb(skb)->time_to_send,
f->time_next_packet);
if (now + q->offload_horizon < time_next_packet) {
if (now + offload_horizon < time_next_packet) {
head->first = f->next;
f->time_next_packet = time_next_packet;
fq_flow_set_throttled(q, f);
@ -836,7 +868,12 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
}
f->time_next_packet = now + len;
}
out:
if (offload_horizon &&
time_next_packet && time_next_packet <= now + fq_offload_slack_ns)
skb_set_delivery_time(skb, 0, SKB_CLOCK_REALTIME);
return skb;
}
@ -1179,11 +1216,15 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt,
u64 offload_horizon = (u64)NSEC_PER_USEC *
nla_get_u32(tb[TCA_FQ_OFFLOAD_HORIZON]);
if (offload_horizon <= qdisc_dev(sch)->max_pacing_offload_horizon) {
WRITE_ONCE(q->offload_horizon, offload_horizon);
} else {
if (offload_horizon && !qdisc_dev(sch)->pacing_offload) {
NL_SET_ERR_MSG_MOD(extack, "device pacing offload is disabled");
err = -EINVAL;
} else if (offload_horizon >
qdisc_dev(sch)->max_pacing_offload_horizon) {
NL_SET_ERR_MSG_MOD(extack, "invalid offload_horizon");
err = -EINVAL;
} else {
WRITE_ONCE(q->offload_horizon, offload_horizon);
}
}
if (!err) {

161
tools/testing/selftests/drivers/net/so_txtime.c
View File

@ -42,8 +42,10 @@ static bool cfg_machine_slow;
static uint64_t cfg_start_time_ns;
static int cfg_mark;
static bool cfg_rx;
static bool cfg_verify_hw_offload;
static uint64_t glob_tstart;
static uint64_t glob_tstart_real;
static uint64_t tdeliver_max;
static int errors;
@ -153,23 +155,75 @@ static void do_recv_verify_empty(int fdr)
char rbuf[1];
int ret;
ret = recv(fdr, rbuf, sizeof(rbuf), 0);
ret = recv(fdr, rbuf, sizeof(rbuf), MSG_DONTWAIT);
if (ret != -1 || errno != EAGAIN)
error(1, 0, "recv: not empty as expected (%d, %d)", ret, errno);
}
static int do_recv_errqueue_timeout(int fdt)
static int do_recv_errqueue_txtime(struct sock_extended_err *err,
const char payload_char)
{
const char *reason = NULL;
int64_t tstamp = 0;
switch (err->ee_errno) {
case ECANCELED:
if (err->ee_code != SO_EE_CODE_TXTIME_MISSED)
error(1, 0, "errqueue: unknown ECANCELED %u\n",
err->ee_code);
reason = "missed txtime";
break;
case EINVAL:
if (err->ee_code != SO_EE_CODE_TXTIME_INVALID_PARAM)
error(1, 0, "errqueue: unknown EINVAL %u\n",
err->ee_code);
reason = "invalid txtime";
break;
default:
error(1, 0, "errqueue: errno %u code %u\n",
err->ee_errno, err->ee_code);
}
tstamp = ((int64_t)err->ee_data) << 32 | err->ee_info;
tstamp -= (int64_t)glob_tstart;
tstamp /= 1000 * 1000;
fprintf(stderr, "send: pkt %c at %" PRId64 "ms dropped: %s\n",
payload_char, tstamp, reason);
return 1;
}
static int do_recv_errqueue_timestamping(struct scm_timestamping *tss)
{
int64_t ts;
ts = tss->ts[0].tv_sec * 1000ULL * 1000 * 1000;
ts += tss->ts[0].tv_nsec;
ts -= glob_tstart_real;
ts /= 1000;
if (ts > cfg_variance_us) {
fprintf(stderr, "sw delay %" PRId64 "us exceeds bounds\n", ts);
if (!cfg_machine_slow)
errors++;
}
return 1;
}
static int do_recv_errqueue(int fdt, int *num_ts)
{
char control[CMSG_SPACE(sizeof(struct sock_extended_err)) +
CMSG_SPACE(sizeof(struct scm_timestamping)) +
CMSG_SPACE(sizeof(struct sockaddr_in6))] = {0};
char data[sizeof(struct ethhdr) + sizeof(struct ipv6hdr) +
sizeof(struct udphdr) + 1];
struct scm_timestamping *tss;
struct sock_extended_err *err;
int ret, num_tstamp = 0;
struct msghdr msg = {0};
struct iovec iov = {0};
struct cmsghdr *cm;
int64_t tstamp = 0;
iov.iov_base = data;
iov.iov_len = sizeof(data);
@ -181,8 +235,6 @@ static int do_recv_errqueue_timeout(int fdt)
msg.msg_controllen = sizeof(control);
while (1) {
const char *reason = NULL;
ret = recvmsg(fdt, &msg, MSG_ERRQUEUE);
if (ret == -1 && errno == EAGAIN)
break;
@ -192,42 +244,31 @@ static int do_recv_errqueue_timeout(int fdt)
error(1, 0, "errqueue: flags 0x%x\n", msg.msg_flags);
cm = CMSG_FIRSTHDR(&msg);
tss = NULL;
if (cm->cmsg_level == SOL_SOCKET &&
cm->cmsg_type == SCM_TIMESTAMPING) {
tss = (void *)CMSG_DATA(cm);
cm = CMSG_NXTHDR(&msg, cm);
if (!cm)
error(1, 0, "timestamp missing ip err\n");
}
if (cm->cmsg_level != cfg_errq_level ||
cm->cmsg_type != cfg_errq_type)
error(1, 0, "errqueue: type 0x%x.0x%x\n",
cm->cmsg_level, cm->cmsg_type);
err = (struct sock_extended_err *)CMSG_DATA(cm);
if (err->ee_origin != SO_EE_ORIGIN_TXTIME)
if (err->ee_origin == SO_EE_ORIGIN_TXTIME)
num_tstamp += do_recv_errqueue_txtime(err, data[ret - 1]);
else if (err->ee_origin == SO_EE_ORIGIN_TIMESTAMPING && tss)
*num_ts += do_recv_errqueue_timestamping(tss);
else
error(1, 0, "errqueue: origin 0x%x\n", err->ee_origin);
switch (err->ee_errno) {
case ECANCELED:
if (err->ee_code != SO_EE_CODE_TXTIME_MISSED)
error(1, 0, "errqueue: unknown ECANCELED %u\n",
err->ee_code);
reason = "missed txtime";
break;
case EINVAL:
if (err->ee_code != SO_EE_CODE_TXTIME_INVALID_PARAM)
error(1, 0, "errqueue: unknown EINVAL %u\n",
err->ee_code);
reason = "invalid txtime";
break;
default:
error(1, 0, "errqueue: errno %u code %u\n",
err->ee_errno, err->ee_code);
}
tstamp = ((int64_t) err->ee_data) << 32 | err->ee_info;
tstamp -= (int64_t) glob_tstart;
tstamp /= 1000 * 1000;
fprintf(stderr, "send: pkt %c at %" PRId64 "ms dropped: %s\n",
data[ret - 1], tstamp, reason);
msg.msg_flags = 0;
msg.msg_controllen = sizeof(control);
num_tstamp++;
}
return num_tstamp;
@ -237,7 +278,7 @@ static void recv_errqueue_msgs(int fdt)
{
struct pollfd pfd = { .fd = fdt, .events = POLLERR };
const int timeout_ms = 10;
int ret, num_tstamp = 0;
int ret, num_tstamp = 0, num_ts = 0;
do {
ret = poll(&pfd, 1, timeout_ms);
@ -245,12 +286,20 @@ static void recv_errqueue_msgs(int fdt)
error(1, errno, "poll");
if (ret && (pfd.revents & POLLERR))
num_tstamp += do_recv_errqueue_timeout(fdt);
num_tstamp += do_recv_errqueue(fdt, &num_ts);
if (num_tstamp == cfg_num_pkt)
if (num_tstamp == cfg_num_pkt || num_ts == cfg_num_pkt)
break;
} while (gettime_ns(cfg_clockid) < tdeliver_max);
} while (gettime_ns(cfg_clockid) <
tdeliver_max + (cfg_variance_us * 1000));
if (cfg_verify_hw_offload && num_ts != cfg_num_pkt) {
fprintf(stderr, "missing timestamps: expected %d, got %d\n",
cfg_num_pkt, num_ts);
if (!cfg_machine_slow)
errors++;
}
}
static void start_time_wait(void)
@ -295,6 +344,17 @@ static void setsockopt_txtime(int fd)
error(1, 0, "getsockopt txtime: mismatch");
}
static void setsockopt_txtimestamping(int fd)
{
int val = SOF_TIMESTAMPING_TX_SOFTWARE |
SOF_TIMESTAMPING_SOFTWARE |
SOF_TIMESTAMPING_OPT_TSONLY;
if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING,
&val, sizeof(val)))
error(1, errno, "setsockopt timestamping");
}
static int setup_tx(struct sockaddr *addr, socklen_t alen)
{
int fd;
@ -308,6 +368,9 @@ static int setup_tx(struct sockaddr *addr, socklen_t alen)
setsockopt_txtime(fd);
if (cfg_verify_hw_offload)
setsockopt_txtimestamping(fd);
if (cfg_mark &&
setsockopt(fd, SOL_SOCKET, SO_MARK, &cfg_mark, sizeof(cfg_mark)))
error(1, errno, "setsockopt mark");
@ -317,7 +380,7 @@ static int setup_tx(struct sockaddr *addr, socklen_t alen)
static int setup_rx(struct sockaddr *addr, socklen_t alen)
{
struct timeval tv = { .tv_usec = 100 * 1000 };
struct timeval tv = { .tv_usec = 600 * 1000 };
int fd;
fd = socket(addr->sa_family, SOCK_DGRAM, 0);
@ -348,6 +411,8 @@ static void do_test_tx(struct sockaddr *addr, socklen_t alen)
start_time_wait();
glob_tstart = gettime_ns(cfg_clockid);
glob_tstart_real = gettime_ns(CLOCK_REALTIME);
tdeliver_max = glob_tstart;
for (i = 0; i < cfg_num_pkt; i++)
do_send_one(fdt, &cfg_buf[i]);
@ -440,10 +505,11 @@ static void usage(const char *progname)
" -6 only IPv6\n"
" -c <clock> monotonic or tai (default)\n"
" -D <addr> destination IP address (server)\n"
" -S <addr> source IP address (client)\n"
" -r run rx mode\n"
" -t <nsec> start time (UTC nanoseconds)\n"
" -H verify hardware offload (tx)\n"
" -m <mark> socket mark\n"
" -r run rx mode\n"
" -S <addr> source IP address (client)\n"
" -t <nsec> start time (UTC nanoseconds)\n"
"\n",
progname);
exit(1);
@ -455,7 +521,7 @@ static void parse_opts(int argc, char **argv)
int domain = PF_UNSPEC;
int c;
while ((c = getopt(argc, argv, "46c:S:D:rt:m:")) != -1) {
while ((c = getopt(argc, argv, "46c:D:Hm:rS:t:")) != -1) {
switch (c) {
case '4':
if (domain != PF_UNSPEC)
@ -482,21 +548,24 @@ static void parse_opts(int argc, char **argv)
else
error(1, 0, "unknown clock id %s", optarg);
break;
case 'S':
saddr = optarg;
break;
case 'D':
daddr = optarg;
break;
case 'H':
cfg_verify_hw_offload = true;
break;
case 'm':
cfg_mark = strtol(optarg, NULL, 0);
break;
case 'r':
cfg_rx = true;
break;
case 'S':
saddr = optarg;
break;
case 't':
cfg_start_time_ns = strtoll(optarg, NULL, 0);
break;
case 'm':
cfg_mark = strtol(optarg, NULL, 0);
break;
default:
usage(argv[0]);
}

80
tools/testing/selftests/drivers/net/so_txtime.py
View File

@ -12,10 +12,12 @@ import time
from lib.py import ksft_exit, ksft_run, ksft_variants
from lib.py import KsftNamedVariant, KsftSkipEx
from lib.py import NetDrvEpEnv, bkg, cmd, defer, tc
from lib.py import CmdExitFailure
from lib.py import CmdExitFailure, RtnlFamily, NlError
_HW_OFFLOAD_HORIZON_MS = 50
def test_so_txtime(cfg, clockid, ipver, args_tx, args_rx, expect_success):
def test_so_txtime(cfg, clockid, ipver, args_tx, args_rx, expect_success,
timing_sensitive=False):
"""Main function. Run so_txtime as sender and receiver."""
slow_machine = os.environ.get('KSFT_MACHINE_SLOW')
@ -33,12 +35,42 @@ def test_so_txtime(cfg, clockid, ipver, args_tx, args_rx, expect_success):
expect_fail = not expect_success
if slow_machine:
expect_success = False
if timing_sensitive:
expect_fail = None
with bkg(cmd_rx, host=cfg.remote, fail=expect_success,
expect_fail=expect_fail, exit_wait=True):
cmd(cmd_tx)
def _dev_setup_pacing_offload(cfg):
"""Configure pacing-offload."""
rtnl = RtnlFamily()
try:
link = rtnl.getlink({'ifi-index': cfg.ifindex})
except NlError as e:
raise KsftSkipEx('getlink not supported by device') from e
if 'pacing-offload' not in link or \
'max-pacing-offload-horizon' not in link:
raise KsftSkipEx('pacing offload not supported by device')
horizon = _HW_OFFLOAD_HORIZON_MS * 1000_000
if link['max-pacing-offload-horizon'] < horizon:
raise KsftSkipEx('pacing offload max horizon too small')
cur_offload = link['pacing-offload']
rtnl.setlink({
'ifi-index': cfg.ifindex,
'pacing-offload': 1,
})
defer(rtnl.setlink, {
'ifi-index': cfg.ifindex,
'pacing-offload': cur_offload
})
def _qdisc_setup(ifname, qdisc, optargs=""):
"""Replace root qdisc. Restore the original after the test.
@ -61,6 +93,7 @@ def _test_variants_fq():
["one_pkt", "a,10", "a,10"],
["in_order", "a,10,b,20", "a,10,b,20"],
["reverse_order", "a,20,b,10", "b,10,a,20"],
["beyond_hw_horizon", "a,70", "a,70"],
]:
name = f"v{ipver}_{testcase[0]}"
yield KsftNamedVariant(name, ipver, testcase[1], testcase[2])
@ -74,6 +107,41 @@ def test_so_txtime_fq_mono(cfg, ipver, args_tx, args_rx):
test_so_txtime(cfg, "mono", ipver, args_tx, args_rx, True)
@ksft_variants(_test_variants_fq())
def test_so_txtime_fq_mono_hw(cfg, ipver, args_tx, args_rx):
"""Run all variants of monotonic fq tests, with offload horizon."""
cfg.require_ipver(ipver)
cfg.require_nsim(nsim_test=False)
_dev_setup_pacing_offload(cfg)
try:
_qdisc_setup(cfg.ifname, "fq", f"offload_horizon {_HW_OFFLOAD_HORIZON_MS}ms")
except Exception as e:
raise KsftSkipEx("netdev does not support offload. skipping") from e
# Expect all tests to use only hw pacing, except beyond_hw_horizon.
# Do not pass -H to that test so that with sw pacing fallback it passes.
hw_only = "-H" if args_tx != "a,70" else ""
test_so_txtime(cfg, "mono", ipver, f"{hw_only} {args_tx}", args_rx, True)
@ksft_variants(_test_variants_fq())
def test_so_txtime_pfifofast_mono_hw(cfg, ipver, args_tx, args_rx):
"""Run all variants of monotonic tests, without fq pacing sw backup."""
cfg.require_ipver(ipver)
cfg.require_nsim(nsim_test=False)
_dev_setup_pacing_offload(cfg)
_qdisc_setup(cfg.ifname, "pfifo_fast")
# Expect all tests to pass, except beyond_hw_horizon without sw fallback.
# It will send immediately, failing the receiver arrival bounds check.
expect_pass = args_tx != "a,70"
timing_sensitive = not expect_pass
test_so_txtime(cfg, "mono", ipver, f"-H {args_tx}", args_rx, expect_pass,
timing_sensitive=timing_sensitive)
@ksft_variants(_test_variants_fq())
def test_so_txtime_fq_tai(cfg, ipver, args_tx, args_rx):
"""Run all variants of fq tests, but pass CLOCK_TAI to test conversion."""
@ -123,7 +191,13 @@ def main() -> None:
"""Boilerplate ksft main."""
with NetDrvEpEnv(__file__) as cfg:
ksft_run(
[test_so_txtime_fq_mono, test_so_txtime_fq_tai, test_so_txtime_etf],
[
test_so_txtime_fq_mono,
test_so_txtime_fq_mono_hw,
test_so_txtime_pfifofast_mono_hw,
test_so_txtime_fq_tai,
test_so_txtime_etf,
],
args=(cfg,),
)
ksft_exit()