mirror of
https://github.com/torvalds/linux.git
synced 2026-07-28 10:09:10 +02:00
selftests: rds: Add ROCE support to test.py
This patch adds support for testing rds rdma over ROCE in test.py A new -T flag is added, which takes a transport option, tcp or rdma. A new setup_rdma() function is added that will configure rdma interfaces and sockets for use in the test case. Signed-off-by: Allison Henderson <achender@kernel.org> Link: https://patch.msgid.link/20260518012443.2629206-11-achender@kernel.org Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Allison Henderson
Jakub Kicinski
parent
a887620348
commit
9d4fc64132
|
|
@ -11,10 +11,12 @@ import errno
|
|||
import hashlib
|
||||
import os
|
||||
import select
|
||||
import re
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
# Allow utils module to be imported from different directory
|
||||
this_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
|
|
@ -41,6 +43,27 @@ tcp_addrs = [
|
|||
('10.0.0.2', 20000),
|
||||
]
|
||||
|
||||
# RDMA network configs
|
||||
RXE_DEV0 = 'rxe0'
|
||||
RXE_DEV1 = 'rxe1'
|
||||
|
||||
VETH_RDMA0 = 'veth_rdma0'
|
||||
VETH_RDMA1 = 'veth_rdma1'
|
||||
|
||||
rdma_addrs = [
|
||||
('10.0.0.3', 30000),
|
||||
('10.0.0.4', 30000),
|
||||
]
|
||||
|
||||
# send_packets flag space
|
||||
OP_FLAG_TCP = 0x1
|
||||
OP_FLAG_RDMA = 0x2
|
||||
|
||||
signal_handler_label = ""
|
||||
|
||||
tap_idx = 0
|
||||
nr_pass = 0
|
||||
nr_fail = 0
|
||||
|
||||
# Helper function for creating a socket inside a network namespace.
|
||||
# We need this because otherwise RDS will detect that the two TCP
|
||||
|
|
@ -169,18 +192,35 @@ def verify_hashes(snd_hashes, rcv_hashes):
|
|||
ksft_pr(f"{key[0]}/{key[1]}: ok")
|
||||
return 0
|
||||
|
||||
def snd_rcv_packets(addrs, netns_list):
|
||||
def snd_rcv_packets(env):
|
||||
"""
|
||||
Send packets on the given network interfaces
|
||||
|
||||
:param addrs: list of (ip, port) tuples matching the sockets
|
||||
:param netns_list: list of network namespaces
|
||||
:param env: transport-environment dict for setup_tcp() / setup_rdma().
|
||||
"addrs": list of (ip, port) tuples matching the sockets
|
||||
"netns": list of netns names for TCP or None for RDMA
|
||||
"flags": OP_FLAG_TCP or OP_FLAG_RDMA, selects sockets
|
||||
"""
|
||||
|
||||
sockets = [
|
||||
netns_socket(netns_list[0], socket.AF_RDS, socket.SOCK_SEQPACKET),
|
||||
netns_socket(netns_list[1], socket.AF_RDS, socket.SOCK_SEQPACKET),
|
||||
]
|
||||
addrs = env["addrs"]
|
||||
netns_list = env["netns"]
|
||||
flags = env.get("flags", 0)
|
||||
|
||||
if (flags & OP_FLAG_TCP) and (flags & OP_FLAG_RDMA):
|
||||
raise RuntimeError(f"Invalid transport flag sets multiple transports: {flags}")
|
||||
|
||||
if flags & OP_FLAG_TCP:
|
||||
sockets = [
|
||||
netns_socket(netns_list[0], socket.AF_RDS, socket.SOCK_SEQPACKET),
|
||||
netns_socket(netns_list[1], socket.AF_RDS, socket.SOCK_SEQPACKET),
|
||||
]
|
||||
elif flags & OP_FLAG_RDMA:
|
||||
sockets = [
|
||||
socket.socket(socket.AF_RDS, socket.SOCK_SEQPACKET),
|
||||
socket.socket(socket.AF_RDS, socket.SOCK_SEQPACKET),
|
||||
]
|
||||
else:
|
||||
raise RuntimeError(f"Invalid transport flag sets no transports: {flags}")
|
||||
|
||||
for s, addr in zip(sockets, addrs):
|
||||
s.bind(addr)
|
||||
|
|
@ -210,9 +250,10 @@ def snd_rcv_packets(addrs, netns_list):
|
|||
nr_recv = recv_burst(ep, sockets, addrs, recv_hashes, nr_recv)
|
||||
|
||||
# exercise net/rds/tcp.c:rds_tcp_sysctl_reset()
|
||||
for net in netns_list:
|
||||
ip(f"netns exec {net} /usr/sbin/sysctl net.rds.tcp.rds_tcp_rcvbuf=10000")
|
||||
ip(f"netns exec {net} /usr/sbin/sysctl net.rds.tcp.rds_tcp_sndbuf=10000")
|
||||
if netns_list:
|
||||
for net in netns_list:
|
||||
ip(f"netns exec {net} /usr/sbin/sysctl net.rds.tcp.rds_tcp_rcvbuf=10000")
|
||||
ip(f"netns exec {net} /usr/sbin/sysctl net.rds.tcp.rds_tcp_sndbuf=10000")
|
||||
|
||||
ksft_pr("done", nr_send, nr_recv)
|
||||
|
||||
|
|
@ -220,7 +261,13 @@ def snd_rcv_packets(addrs, netns_list):
|
|||
|
||||
# We're done sending and receiving stuff, now let's check if what
|
||||
# we received is what we sent.
|
||||
return verify_hashes(send_hashes, recv_hashes)
|
||||
rc = verify_hashes(send_hashes, recv_hashes)
|
||||
|
||||
ep.close()
|
||||
for s in sockets:
|
||||
s.close()
|
||||
|
||||
return rc
|
||||
|
||||
def stop_pcaps():
|
||||
"""Stop tcpdump processes.
|
||||
|
|
@ -247,8 +294,8 @@ def signal_handler(_sig, _frame):
|
|||
"""
|
||||
Test timed out signal handler
|
||||
"""
|
||||
ksft_pr("Test timed out")
|
||||
print("not ok 1 rds selftest")
|
||||
ksft_pr(f"Test timed out: {signal_handler_label}")
|
||||
print(f"not ok {tap_idx} rds selftest {signal_handler_label}")
|
||||
sys.exit(1)
|
||||
|
||||
def setup_tcp():
|
||||
|
|
@ -314,12 +361,107 @@ def teardown_tcp():
|
|||
cmd(f"ip netns del {NET0}", fail=False)
|
||||
cmd(f"ip netns del {NET1}", fail=False)
|
||||
|
||||
def get_iface_mac(iface):
|
||||
"""Return the MAC address of a local network interface."""
|
||||
out = subprocess.check_output(['ip', 'link', 'show', iface], text=True)
|
||||
mac = re.search(r'link/ether\s+([0-9a-f:]+)', out)
|
||||
if not mac:
|
||||
raise RuntimeError(f"Cannot determine MAC address of {iface}")
|
||||
return mac.group(1)
|
||||
|
||||
def setup_rdma():
|
||||
"""
|
||||
Configure rdma network
|
||||
"""
|
||||
|
||||
# remove links left over by previously interrupted run.
|
||||
teardown_rdma()
|
||||
|
||||
# use call here since modprobe may fail if the rdma_rxe
|
||||
# module is built-in
|
||||
subprocess.call(['modprobe', 'rdma_rxe'],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
ip(f"link add {VETH_RDMA0} type veth peer name {VETH_RDMA1}")
|
||||
|
||||
ip(f"link set {VETH_RDMA0} up")
|
||||
ip(f"link set {VETH_RDMA1} up")
|
||||
|
||||
# Since both addresses are in the same namespace, the source address
|
||||
# is always local, so enable accept_local
|
||||
cmd(f"/usr/sbin/sysctl -q net.ipv4.conf.{VETH_RDMA0}.accept_local=1")
|
||||
cmd(f"/usr/sbin/sysctl -q net.ipv4.conf.{VETH_RDMA1}.accept_local=1")
|
||||
|
||||
# Reverse path filters must be disabled so that the local routes don't
|
||||
# cause RPF failures.
|
||||
cmd(f"/usr/sbin/sysctl -q net.ipv4.conf.{VETH_RDMA0}.rp_filter=0")
|
||||
cmd(f"/usr/sbin/sysctl -q net.ipv4.conf.{VETH_RDMA1}.rp_filter=0")
|
||||
|
||||
# add addresses
|
||||
ip(f"addr add {rdma_addrs[0][0]}/32 dev {VETH_RDMA0}")
|
||||
ip(f"addr add {rdma_addrs[1][0]}/32 dev {VETH_RDMA1}")
|
||||
|
||||
# add routes
|
||||
ip(f"route add {rdma_addrs[1][0]}/32 dev {VETH_RDMA0}")
|
||||
ip(f"route add {rdma_addrs[0][0]}/32 dev {VETH_RDMA1}")
|
||||
|
||||
# ARP will not resolve neighbor IPs on /32 routes without a subnet.
|
||||
# Avoid this by adding neighbors directly so RDMA CM can populate path
|
||||
# records with correct mac addrs without waiting for the ARP.
|
||||
mac0 = get_iface_mac(VETH_RDMA0)
|
||||
mac1 = get_iface_mac(VETH_RDMA1)
|
||||
ip(f"neigh add {rdma_addrs[1][0]} lladdr {mac1} dev {VETH_RDMA0} nud permanent")
|
||||
ip(f"neigh add {rdma_addrs[0][0]} lladdr {mac0} dev {VETH_RDMA1} nud permanent")
|
||||
|
||||
cmd(f'rdma link add {RXE_DEV0} type rxe netdev {VETH_RDMA0}')
|
||||
cmd(f'rdma link add {RXE_DEV1} type rxe netdev {VETH_RDMA1}')
|
||||
|
||||
time.sleep(1) # allow RXE devices to initialise
|
||||
|
||||
# Start a packet capture on each network
|
||||
if logdir is not None:
|
||||
for iface in [VETH_RDMA0, VETH_RDMA1]:
|
||||
pcap = logdir+'/rds-roce-'+iface+'.pcap'
|
||||
|
||||
tcpdump_cmd = ['/usr/sbin/tcpdump']
|
||||
sudo_user = os.environ.get('SUDO_USER')
|
||||
if sudo_user:
|
||||
tcpdump_cmd.extend(['-Z', sudo_user])
|
||||
tcpdump_cmd.extend(['-i', iface, '-w', pcap])
|
||||
|
||||
# pylint: disable-next=consider-using-with
|
||||
p = subprocess.Popen(tcpdump_cmd,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
tcpdump_procs.append(p)
|
||||
|
||||
# simulate packet loss, duplication and corruption
|
||||
for iface in [VETH_RDMA0, VETH_RDMA1]:
|
||||
cmd(f"/usr/sbin/tc qdisc add dev {iface} root netem \
|
||||
corrupt {PACKET_CORRUPTION} loss {PACKET_LOSS} duplicate \
|
||||
{PACKET_DUPLICATE}")
|
||||
|
||||
def teardown_rdma():
|
||||
"""
|
||||
Tear down the rdma network configured by setup_rdma().
|
||||
"""
|
||||
|
||||
# remove links left over by previously interrupted run.
|
||||
cmd(f'rdma link del {RXE_DEV0}', fail=False)
|
||||
cmd(f'rdma link del {RXE_DEV1}', fail=False)
|
||||
cmd(f'ip link del {VETH_RDMA0}', fail=False)
|
||||
|
||||
|
||||
#Parse out command line arguments. We take an optional
|
||||
# timeout parameter and an optional log output folder
|
||||
parser = argparse.ArgumentParser(description="init script args",
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument("-d", "--logdir", action="store",
|
||||
help="directory to store logs", default=None)
|
||||
parser.add_argument("-T", "--transport", default="tcp",
|
||||
help="Comma-separated list of transports to test: "
|
||||
"tcp, rdma, or tcp,rdma. Each matching test "
|
||||
"is run once per transport. "
|
||||
"'rdma' requires CONFIG_RDS_RDMA and rdma_rxe.")
|
||||
parser.add_argument('-t', '--timeout', help="timeout to terminate hung test",
|
||||
type=int, default=0)
|
||||
parser.add_argument('-l', '--loss', help="Simulate tcp packet loss",
|
||||
|
|
@ -334,31 +476,63 @@ PACKET_LOSS=str(args.loss)+'%'
|
|||
PACKET_CORRUPTION=str(args.corruption)+'%'
|
||||
PACKET_DUPLICATE=str(args.duplicate)+'%'
|
||||
|
||||
# Register cleanup before setup so a partial-setup crash still tears down
|
||||
# whatever state did get created.
|
||||
atexit.register(teardown_tcp)
|
||||
# check transport is either tcp or rdma
|
||||
transports = [t.strip() for t in args.transport.split(',')]
|
||||
for t in transports:
|
||||
if t not in ('tcp', 'rdma'):
|
||||
raise SystemExit(f"test.py: unknown transport: {t!r}")
|
||||
|
||||
# Register stop_pcaps before any network setups so that any partially setup
|
||||
# tcpdumps are still cleaned up on error
|
||||
atexit.register(stop_pcaps)
|
||||
|
||||
setup_tcp()
|
||||
# Set up all requested transports upfront so network plumbing is
|
||||
# ready before any test runs.
|
||||
transport_envs = {}
|
||||
FLAGS = 0
|
||||
if 'tcp' in transports:
|
||||
# Register cleanups before setups to handle partial setups that error'd out
|
||||
atexit.register(teardown_tcp)
|
||||
setup_tcp()
|
||||
transport_envs['tcp'] = {
|
||||
'addrs': tcp_addrs,
|
||||
'netns': [NET0, NET1],
|
||||
'flags': FLAGS | OP_FLAG_TCP,
|
||||
}
|
||||
|
||||
if 'rdma' in transports:
|
||||
atexit.register(teardown_rdma)
|
||||
setup_rdma()
|
||||
transport_envs['rdma'] = {
|
||||
'addrs': rdma_addrs,
|
||||
'netns': None,
|
||||
'flags': FLAGS | OP_FLAG_RDMA,
|
||||
}
|
||||
|
||||
print("TAP version 13")
|
||||
print("1..1")
|
||||
print(f"1..{len(transport_envs)}")
|
||||
|
||||
# add a timeout
|
||||
if args.timeout > 0:
|
||||
signal.alarm(args.timeout)
|
||||
signal.signal(signal.SIGALRM, signal_handler)
|
||||
for transport, tenv in transport_envs.items():
|
||||
tap_idx += 1
|
||||
|
||||
ret = snd_rcv_packets(tcp_addrs, [NET0, NET1])
|
||||
# add a timeout
|
||||
if args.timeout > 0:
|
||||
signal_handler_label = transport
|
||||
signal.alarm(args.timeout)
|
||||
signal.signal(signal.SIGALRM, signal_handler)
|
||||
|
||||
# cancel timeout
|
||||
signal.alarm(0)
|
||||
ret = snd_rcv_packets(tenv)
|
||||
|
||||
if ret == 0:
|
||||
ksft_pr("Success")
|
||||
print("ok 1 rds selftest")
|
||||
else:
|
||||
print("not ok 1 rds selftest")
|
||||
# cancel timeout
|
||||
signal.alarm(0)
|
||||
|
||||
ksft_pr(f"Totals: pass:{1-ret} fail:{ret} skip:0")
|
||||
sys.exit(ret)
|
||||
if ret == 0:
|
||||
ksft_pr("Success")
|
||||
print(f"ok {tap_idx} rds selftest {transport}")
|
||||
nr_pass += 1
|
||||
else:
|
||||
print(f"not ok {tap_idx} rds selftest {transport}")
|
||||
nr_fail += 1
|
||||
|
||||
ksft_pr(f"Totals: pass:{nr_pass} fail:{nr_fail} skip:0")
|
||||
sys.exit(1 if nr_fail else 0)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user