mirror of
https://github.com/torvalds/linux.git
synced 2026-08-01 12:11:59 +02:00
Merge branch 'preserve-msg_zerocopy-with-forwarding'
Willem de Bruijn says:
====================
preserve MSG_ZEROCOPY with forwarding
Avoid false positive copying of zerocopy skb frags when entering the
ingress path if the skb is not queued locally but forwarded.
Patch 1 for more details and feature.
Patch 2 converts the existing selftest to a pass/fail test and adds
coverage for this new feature.
====================
Link: https://patch.msgid.link/20250630194312.1571410-1-willemdebruijn.kernel@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski
commit
285c895fba
|
|
@ -5937,8 +5937,6 @@ static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
|
|||
}
|
||||
|
||||
if (pt_prev) {
|
||||
if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
|
||||
goto drop;
|
||||
*ppt_prev = pt_prev;
|
||||
} else {
|
||||
drop:
|
||||
|
|
|
|||
|
|
@ -226,6 +226,12 @@ void ip_protocol_deliver_rcu(struct net *net, struct sk_buff *skb, int protocol)
|
|||
|
||||
static int ip_local_deliver_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
|
||||
{
|
||||
if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) {
|
||||
__IP_INC_STATS(net, IPSTATS_MIB_INDISCARDS);
|
||||
kfree_skb_reason(skb, SKB_DROP_REASON_NOMEM);
|
||||
return 0;
|
||||
}
|
||||
|
||||
skb_clear_delivery_time(skb);
|
||||
__skb_pull(skb, skb_network_header_len(skb));
|
||||
|
||||
|
|
|
|||
|
|
@ -478,6 +478,13 @@ void ip6_protocol_deliver_rcu(struct net *net, struct sk_buff *skb, int nexthdr,
|
|||
|
||||
static int ip6_input_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
|
||||
{
|
||||
if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) {
|
||||
__IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
|
||||
IPSTATS_MIB_INDISCARDS);
|
||||
kfree_skb_reason(skb, SKB_DROP_REASON_NOMEM);
|
||||
return 0;
|
||||
}
|
||||
|
||||
skb_clear_delivery_time(skb);
|
||||
ip6_protocol_deliver_rcu(net, skb, 0, false);
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@
|
|||
static int cfg_cork;
|
||||
static bool cfg_cork_mixed;
|
||||
static int cfg_cpu = -1; /* default: pin to last cpu */
|
||||
static int cfg_expect_zerocopy = -1;
|
||||
static int cfg_family = PF_UNSPEC;
|
||||
static int cfg_ifindex = 1;
|
||||
static int cfg_payload_len;
|
||||
|
|
@ -92,9 +93,9 @@ static socklen_t cfg_alen;
|
|||
static struct sockaddr_storage cfg_dst_addr;
|
||||
static struct sockaddr_storage cfg_src_addr;
|
||||
|
||||
static int exitcode;
|
||||
static char payload[IP_MAXPACKET];
|
||||
static long packets, bytes, completions, expected_completions;
|
||||
static int zerocopied = -1;
|
||||
static uint32_t next_completion;
|
||||
static uint32_t sends_since_notify;
|
||||
|
||||
|
|
@ -444,11 +445,13 @@ static bool do_recv_completion(int fd, int domain)
|
|||
next_completion = hi + 1;
|
||||
|
||||
zerocopy = !(serr->ee_code & SO_EE_CODE_ZEROCOPY_COPIED);
|
||||
if (zerocopied == -1)
|
||||
zerocopied = zerocopy;
|
||||
else if (zerocopied != zerocopy) {
|
||||
fprintf(stderr, "serr: inconsistent\n");
|
||||
zerocopied = zerocopy;
|
||||
if (cfg_expect_zerocopy != -1 &&
|
||||
cfg_expect_zerocopy != zerocopy) {
|
||||
fprintf(stderr, "serr: ee_code: %u != expected %u\n",
|
||||
zerocopy, cfg_expect_zerocopy);
|
||||
exitcode = 1;
|
||||
/* suppress repeated messages */
|
||||
cfg_expect_zerocopy = zerocopy;
|
||||
}
|
||||
|
||||
if (cfg_verbose >= 2)
|
||||
|
|
@ -571,7 +574,7 @@ static void do_tx(int domain, int type, int protocol)
|
|||
|
||||
fprintf(stderr, "tx=%lu (%lu MB) txc=%lu zc=%c\n",
|
||||
packets, bytes >> 20, completions,
|
||||
zerocopied == 1 ? 'y' : 'n');
|
||||
cfg_zerocopy && cfg_expect_zerocopy == 1 ? 'y' : 'n');
|
||||
}
|
||||
|
||||
static int do_setup_rx(int domain, int type, int protocol)
|
||||
|
|
@ -715,7 +718,7 @@ static void parse_opts(int argc, char **argv)
|
|||
|
||||
cfg_payload_len = max_payload_len;
|
||||
|
||||
while ((c = getopt(argc, argv, "46c:C:D:i:l:mp:rs:S:t:vz")) != -1) {
|
||||
while ((c = getopt(argc, argv, "46c:C:D:i:l:mp:rs:S:t:vzZ:")) != -1) {
|
||||
switch (c) {
|
||||
case '4':
|
||||
if (cfg_family != PF_UNSPEC)
|
||||
|
|
@ -770,6 +773,9 @@ static void parse_opts(int argc, char **argv)
|
|||
case 'z':
|
||||
cfg_zerocopy = true;
|
||||
break;
|
||||
case 'Z':
|
||||
cfg_expect_zerocopy = !!atoi(optarg);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -817,5 +823,5 @@ int main(int argc, char **argv)
|
|||
else
|
||||
error(1, 0, "unknown cfg_test %s", cfg_test);
|
||||
|
||||
return 0;
|
||||
return exitcode;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
set -e
|
||||
|
||||
readonly DEV="veth0"
|
||||
readonly DUMMY_DEV="dummy0"
|
||||
readonly DEV_MTU=65535
|
||||
readonly BIN="./msg_zerocopy"
|
||||
|
||||
|
|
@ -14,21 +15,25 @@ readonly NSPREFIX="ns-${RAND}"
|
|||
readonly NS1="${NSPREFIX}1"
|
||||
readonly NS2="${NSPREFIX}2"
|
||||
|
||||
readonly SADDR4='192.168.1.1'
|
||||
readonly DADDR4='192.168.1.2'
|
||||
readonly SADDR6='fd::1'
|
||||
readonly DADDR6='fd::2'
|
||||
readonly LPREFIX4='192.168.1'
|
||||
readonly RPREFIX4='192.168.2'
|
||||
readonly LPREFIX6='fd'
|
||||
readonly RPREFIX6='fc'
|
||||
|
||||
|
||||
readonly path_sysctl_mem="net.core.optmem_max"
|
||||
|
||||
# No arguments: automated test
|
||||
if [[ "$#" -eq "0" ]]; then
|
||||
$0 4 tcp -t 1
|
||||
$0 6 tcp -t 1
|
||||
$0 4 udp -t 1
|
||||
$0 6 udp -t 1
|
||||
echo "OK. All tests passed"
|
||||
exit 0
|
||||
ret=0
|
||||
|
||||
$0 4 tcp -t 1 || ret=1
|
||||
$0 6 tcp -t 1 || ret=1
|
||||
$0 4 udp -t 1 || ret=1
|
||||
$0 6 udp -t 1 || ret=1
|
||||
|
||||
[[ "$ret" == "0" ]] && echo "OK. All tests passed"
|
||||
exit $ret
|
||||
fi
|
||||
|
||||
# Argument parsing
|
||||
|
|
@ -45,11 +50,18 @@ readonly EXTRA_ARGS="$@"
|
|||
|
||||
# Argument parsing: configure addresses
|
||||
if [[ "${IP}" == "4" ]]; then
|
||||
readonly SADDR="${SADDR4}"
|
||||
readonly DADDR="${DADDR4}"
|
||||
readonly SADDR="${LPREFIX4}.1"
|
||||
readonly DADDR="${LPREFIX4}.2"
|
||||
readonly DUMMY_ADDR="${RPREFIX4}.1"
|
||||
readonly DADDR_TXONLY="${RPREFIX4}.2"
|
||||
readonly MASK="24"
|
||||
elif [[ "${IP}" == "6" ]]; then
|
||||
readonly SADDR="${SADDR6}"
|
||||
readonly DADDR="${DADDR6}"
|
||||
readonly SADDR="${LPREFIX6}::1"
|
||||
readonly DADDR="${LPREFIX6}::2"
|
||||
readonly DUMMY_ADDR="${RPREFIX6}::1"
|
||||
readonly DADDR_TXONLY="${RPREFIX6}::2"
|
||||
readonly MASK="64"
|
||||
readonly NODAD="nodad"
|
||||
else
|
||||
echo "Invalid IP version ${IP}"
|
||||
exit 1
|
||||
|
|
@ -89,33 +101,61 @@ ip netns exec "${NS2}" sysctl -w -q "${path_sysctl_mem}=1000000"
|
|||
ip link add "${DEV}" mtu "${DEV_MTU}" netns "${NS1}" type veth \
|
||||
peer name "${DEV}" mtu "${DEV_MTU}" netns "${NS2}"
|
||||
|
||||
ip link add "${DUMMY_DEV}" mtu "${DEV_MTU}" netns "${NS2}" type dummy
|
||||
|
||||
# Bring the devices up
|
||||
ip -netns "${NS1}" link set "${DEV}" up
|
||||
ip -netns "${NS2}" link set "${DEV}" up
|
||||
ip -netns "${NS2}" link set "${DUMMY_DEV}" up
|
||||
|
||||
# Set fixed MAC addresses on the devices
|
||||
ip -netns "${NS1}" link set dev "${DEV}" address 02:02:02:02:02:02
|
||||
ip -netns "${NS2}" link set dev "${DEV}" address 06:06:06:06:06:06
|
||||
|
||||
# Add fixed IP addresses to the devices
|
||||
ip -netns "${NS1}" addr add 192.168.1.1/24 dev "${DEV}"
|
||||
ip -netns "${NS2}" addr add 192.168.1.2/24 dev "${DEV}"
|
||||
ip -netns "${NS1}" addr add fd::1/64 dev "${DEV}" nodad
|
||||
ip -netns "${NS2}" addr add fd::2/64 dev "${DEV}" nodad
|
||||
ip -netns "${NS1}" addr add "${SADDR}/${MASK}" dev "${DEV}" ${NODAD}
|
||||
ip -netns "${NS2}" addr add "${DADDR}/${MASK}" dev "${DEV}" ${NODAD}
|
||||
ip -netns "${NS2}" addr add "${DUMMY_ADDR}/${MASK}" dev "${DUMMY_DEV}" ${NODAD}
|
||||
|
||||
ip -netns "${NS1}" route add default via "${DADDR}" dev "${DEV}"
|
||||
ip -netns "${NS2}" route add default via "${DADDR_TXONLY}" dev "${DUMMY_DEV}"
|
||||
|
||||
ip netns exec "${NS2}" sysctl -wq net.ipv4.ip_forward=1
|
||||
ip netns exec "${NS2}" sysctl -wq net.ipv6.conf.all.forwarding=1
|
||||
|
||||
# Optionally disable sg or csum offload to test edge cases
|
||||
# ip netns exec "${NS1}" ethtool -K "${DEV}" sg off
|
||||
|
||||
ret=0
|
||||
|
||||
do_test() {
|
||||
local readonly ARGS="$1"
|
||||
|
||||
echo "ipv${IP} ${TXMODE} ${ARGS}"
|
||||
ip netns exec "${NS2}" "${BIN}" "-${IP}" -i "${DEV}" -t 2 -C 2 -S "${SADDR}" -D "${DADDR}" ${ARGS} -r "${RXMODE}" &
|
||||
# tx-rx test
|
||||
# packets queued to a local socket are copied,
|
||||
# sender notification has SO_EE_CODE_ZEROCOPY_COPIED.
|
||||
|
||||
echo -e "\nipv${IP} ${TXMODE} ${ARGS} tx-rx\n"
|
||||
ip netns exec "${NS2}" "${BIN}" "-${IP}" -i "${DEV}" -t 2 -C 2 \
|
||||
-S "${SADDR}" -D "${DADDR}" ${ARGS} -r "${RXMODE}" &
|
||||
sleep 0.2
|
||||
ip netns exec "${NS1}" "${BIN}" "-${IP}" -i "${DEV}" -t 1 -C 3 -S "${SADDR}" -D "${DADDR}" ${ARGS} "${TXMODE}"
|
||||
ip netns exec "${NS1}" "${BIN}" "-${IP}" -i "${DEV}" -t 1 -C 3 \
|
||||
-S "${SADDR}" -D "${DADDR}" ${ARGS} "${TXMODE}" -Z 0 || ret=1
|
||||
wait
|
||||
|
||||
# next test is unconnected tx to dummy0, cannot exercise with tcp
|
||||
[[ "${TXMODE}" == "tcp" ]] && return
|
||||
|
||||
# tx-only test: send out dummy0
|
||||
# packets leaving the host are not copied,
|
||||
# sender notification does not have SO_EE_CODE_ZEROCOPY_COPIED.
|
||||
|
||||
echo -e "\nipv${IP} ${TXMODE} ${ARGS} tx-only\n"
|
||||
ip netns exec "${NS1}" "${BIN}" "-${IP}" -i "${DEV}" -t 1 -C 3 \
|
||||
-S "${SADDR}" -D "${DADDR_TXONLY}" ${ARGS} "${TXMODE}" -Z 1 || ret=1
|
||||
}
|
||||
|
||||
do_test "${EXTRA_ARGS}"
|
||||
do_test "-z ${EXTRA_ARGS}"
|
||||
echo ok
|
||||
|
||||
[[ "$ret" == "0" ]] && echo "OK"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user