|
|
|
|
@ -80,6 +80,7 @@
|
|
|
|
|
#include <linux/if_ether.h>
|
|
|
|
|
#include <linux/mman.h>
|
|
|
|
|
#include <linux/netdev.h>
|
|
|
|
|
#include <linux/bitmap.h>
|
|
|
|
|
#include <arpa/inet.h>
|
|
|
|
|
#include <net/if.h>
|
|
|
|
|
#include <locale.h>
|
|
|
|
|
@ -102,10 +103,7 @@
|
|
|
|
|
#include <bpf/bpf.h>
|
|
|
|
|
#include <linux/filter.h>
|
|
|
|
|
#include "../kselftest.h"
|
|
|
|
|
#include "xsk_xdp_metadata.h"
|
|
|
|
|
|
|
|
|
|
static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62";
|
|
|
|
|
static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61";
|
|
|
|
|
#include "xsk_xdp_common.h"
|
|
|
|
|
|
|
|
|
|
static bool opt_verbose;
|
|
|
|
|
static bool opt_print_tests;
|
|
|
|
|
@ -159,10 +157,10 @@ static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size)
|
|
|
|
|
ptr[i] = htonl(pkt_nb << 16 | (i + start));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr)
|
|
|
|
|
static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr)
|
|
|
|
|
{
|
|
|
|
|
memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);
|
|
|
|
|
memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);
|
|
|
|
|
memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN);
|
|
|
|
|
memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN);
|
|
|
|
|
eth_hdr->h_proto = htons(ETH_P_LOOPBACK);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -260,7 +258,7 @@ static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_i
|
|
|
|
|
cfg.bind_flags = ifobject->bind_flags;
|
|
|
|
|
if (shared)
|
|
|
|
|
cfg.bind_flags |= XDP_SHARED_UMEM;
|
|
|
|
|
if (ifobject->pkt_stream && ifobject->mtu > MAX_ETH_PKT_SIZE)
|
|
|
|
|
if (ifobject->mtu > MAX_ETH_PKT_SIZE)
|
|
|
|
|
cfg.bind_flags |= XDP_USE_SG;
|
|
|
|
|
|
|
|
|
|
txr = ifobject->tx_on ? &xsk->tx : NULL;
|
|
|
|
|
@ -429,11 +427,9 @@ static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
|
|
|
|
|
if (i == 0) {
|
|
|
|
|
ifobj->rx_on = false;
|
|
|
|
|
ifobj->tx_on = true;
|
|
|
|
|
ifobj->pkt_stream = test->tx_pkt_stream_default;
|
|
|
|
|
} else {
|
|
|
|
|
ifobj->rx_on = true;
|
|
|
|
|
ifobj->tx_on = false;
|
|
|
|
|
ifobj->pkt_stream = test->rx_pkt_stream_default;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
memset(ifobj->umem, 0, sizeof(*ifobj->umem));
|
|
|
|
|
@ -443,6 +439,15 @@ static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
|
|
|
|
|
for (j = 0; j < MAX_SOCKETS; j++) {
|
|
|
|
|
memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
|
|
|
|
|
ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
|
|
|
|
|
if (i == 0)
|
|
|
|
|
ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default;
|
|
|
|
|
else
|
|
|
|
|
ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default;
|
|
|
|
|
|
|
|
|
|
memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN);
|
|
|
|
|
memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN);
|
|
|
|
|
ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0);
|
|
|
|
|
ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -526,8 +531,10 @@ static int test_spec_set_mtu(struct test_spec *test, int mtu)
|
|
|
|
|
|
|
|
|
|
static void pkt_stream_reset(struct pkt_stream *pkt_stream)
|
|
|
|
|
{
|
|
|
|
|
if (pkt_stream)
|
|
|
|
|
if (pkt_stream) {
|
|
|
|
|
pkt_stream->current_pkt_nb = 0;
|
|
|
|
|
pkt_stream->nb_rx_pkts = 0;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream)
|
|
|
|
|
@ -557,17 +564,17 @@ static void pkt_stream_delete(struct pkt_stream *pkt_stream)
|
|
|
|
|
|
|
|
|
|
static void pkt_stream_restore_default(struct test_spec *test)
|
|
|
|
|
{
|
|
|
|
|
struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream;
|
|
|
|
|
struct pkt_stream *rx_pkt_stream = test->ifobj_rx->pkt_stream;
|
|
|
|
|
struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream;
|
|
|
|
|
struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream;
|
|
|
|
|
|
|
|
|
|
if (tx_pkt_stream != test->tx_pkt_stream_default) {
|
|
|
|
|
pkt_stream_delete(test->ifobj_tx->pkt_stream);
|
|
|
|
|
test->ifobj_tx->pkt_stream = test->tx_pkt_stream_default;
|
|
|
|
|
pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream);
|
|
|
|
|
test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (rx_pkt_stream != test->rx_pkt_stream_default) {
|
|
|
|
|
pkt_stream_delete(test->ifobj_rx->pkt_stream);
|
|
|
|
|
test->ifobj_rx->pkt_stream = test->rx_pkt_stream_default;
|
|
|
|
|
pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream);
|
|
|
|
|
test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -627,14 +634,16 @@ static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pk
|
|
|
|
|
return nb_frags;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, int offset, u32 len)
|
|
|
|
|
static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len)
|
|
|
|
|
{
|
|
|
|
|
pkt->offset = offset;
|
|
|
|
|
pkt->len = len;
|
|
|
|
|
if (len > MAX_ETH_JUMBO_SIZE)
|
|
|
|
|
if (len > MAX_ETH_JUMBO_SIZE) {
|
|
|
|
|
pkt->valid = false;
|
|
|
|
|
else
|
|
|
|
|
} else {
|
|
|
|
|
pkt->valid = true;
|
|
|
|
|
pkt_stream->nb_valid_entries++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len)
|
|
|
|
|
@ -642,7 +651,7 @@ static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len)
|
|
|
|
|
return ceil_u32(len, umem->frame_size) * umem->frame_size;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb_pkts, u32 pkt_len)
|
|
|
|
|
static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off)
|
|
|
|
|
{
|
|
|
|
|
struct pkt_stream *pkt_stream;
|
|
|
|
|
u32 i;
|
|
|
|
|
@ -656,41 +665,45 @@ static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb
|
|
|
|
|
for (i = 0; i < nb_pkts; i++) {
|
|
|
|
|
struct pkt *pkt = &pkt_stream->pkts[i];
|
|
|
|
|
|
|
|
|
|
pkt_set(umem, pkt, 0, pkt_len);
|
|
|
|
|
pkt->pkt_nb = i;
|
|
|
|
|
pkt_set(pkt_stream, pkt, 0, pkt_len);
|
|
|
|
|
pkt->pkt_nb = nb_start + i * nb_off;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return pkt_stream;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static struct pkt_stream *pkt_stream_clone(struct xsk_umem_info *umem,
|
|
|
|
|
struct pkt_stream *pkt_stream)
|
|
|
|
|
static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len)
|
|
|
|
|
{
|
|
|
|
|
return pkt_stream_generate(umem, pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
|
|
|
|
|
return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream)
|
|
|
|
|
{
|
|
|
|
|
return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
|
|
|
|
|
{
|
|
|
|
|
struct pkt_stream *pkt_stream;
|
|
|
|
|
|
|
|
|
|
pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len);
|
|
|
|
|
test->ifobj_tx->pkt_stream = pkt_stream;
|
|
|
|
|
pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, nb_pkts, pkt_len);
|
|
|
|
|
test->ifobj_rx->pkt_stream = pkt_stream;
|
|
|
|
|
pkt_stream = pkt_stream_generate(nb_pkts, pkt_len);
|
|
|
|
|
test->ifobj_tx->xsk->pkt_stream = pkt_stream;
|
|
|
|
|
pkt_stream = pkt_stream_generate(nb_pkts, pkt_len);
|
|
|
|
|
test->ifobj_rx->xsk->pkt_stream = pkt_stream;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
|
|
|
|
|
int offset)
|
|
|
|
|
{
|
|
|
|
|
struct xsk_umem_info *umem = ifobj->umem;
|
|
|
|
|
struct pkt_stream *pkt_stream;
|
|
|
|
|
u32 i;
|
|
|
|
|
|
|
|
|
|
pkt_stream = pkt_stream_clone(umem, ifobj->pkt_stream);
|
|
|
|
|
for (i = 1; i < ifobj->pkt_stream->nb_pkts; i += 2)
|
|
|
|
|
pkt_set(umem, &pkt_stream->pkts[i], offset, pkt_len);
|
|
|
|
|
pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream);
|
|
|
|
|
for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2)
|
|
|
|
|
pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len);
|
|
|
|
|
|
|
|
|
|
ifobj->pkt_stream = pkt_stream;
|
|
|
|
|
ifobj->xsk->pkt_stream = pkt_stream;
|
|
|
|
|
pkt_stream->nb_valid_entries /= 2;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
|
|
|
|
|
@ -701,15 +714,34 @@ static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int off
|
|
|
|
|
|
|
|
|
|
static void pkt_stream_receive_half(struct test_spec *test)
|
|
|
|
|
{
|
|
|
|
|
struct xsk_umem_info *umem = test->ifobj_rx->umem;
|
|
|
|
|
struct pkt_stream *pkt_stream = test->ifobj_tx->pkt_stream;
|
|
|
|
|
struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream;
|
|
|
|
|
u32 i;
|
|
|
|
|
|
|
|
|
|
test->ifobj_rx->pkt_stream = pkt_stream_generate(umem, pkt_stream->nb_pkts,
|
|
|
|
|
pkt_stream->pkts[0].len);
|
|
|
|
|
pkt_stream = test->ifobj_rx->pkt_stream;
|
|
|
|
|
test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts,
|
|
|
|
|
pkt_stream->pkts[0].len);
|
|
|
|
|
pkt_stream = test->ifobj_rx->xsk->pkt_stream;
|
|
|
|
|
for (i = 1; i < pkt_stream->nb_pkts; i += 2)
|
|
|
|
|
pkt_stream->pkts[i].valid = false;
|
|
|
|
|
|
|
|
|
|
pkt_stream->nb_valid_entries /= 2;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void pkt_stream_even_odd_sequence(struct test_spec *test)
|
|
|
|
|
{
|
|
|
|
|
struct pkt_stream *pkt_stream;
|
|
|
|
|
u32 i;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < test->nb_sockets; i++) {
|
|
|
|
|
pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream;
|
|
|
|
|
pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,
|
|
|
|
|
pkt_stream->pkts[0].len, i, 2);
|
|
|
|
|
test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream;
|
|
|
|
|
|
|
|
|
|
pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream;
|
|
|
|
|
pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,
|
|
|
|
|
pkt_stream->pkts[0].len, i, 2);
|
|
|
|
|
test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem)
|
|
|
|
|
@ -724,16 +756,16 @@ static void pkt_stream_cancel(struct pkt_stream *pkt_stream)
|
|
|
|
|
pkt_stream->current_pkt_nb--;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void pkt_generate(struct ifobject *ifobject, u64 addr, u32 len, u32 pkt_nb,
|
|
|
|
|
u32 bytes_written)
|
|
|
|
|
static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len,
|
|
|
|
|
u32 pkt_nb, u32 bytes_written)
|
|
|
|
|
{
|
|
|
|
|
void *data = xsk_umem__get_data(ifobject->umem->buffer, addr);
|
|
|
|
|
void *data = xsk_umem__get_data(umem->buffer, addr);
|
|
|
|
|
|
|
|
|
|
if (len < MIN_PKT_SIZE)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
if (!bytes_written) {
|
|
|
|
|
gen_eth_hdr(ifobject, data);
|
|
|
|
|
gen_eth_hdr(xsk, data);
|
|
|
|
|
|
|
|
|
|
len -= PKT_HDR_SIZE;
|
|
|
|
|
data += PKT_HDR_SIZE;
|
|
|
|
|
@ -783,6 +815,10 @@ static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, s
|
|
|
|
|
|
|
|
|
|
if (pkt->valid && pkt->len > pkt_stream->max_pkt_len)
|
|
|
|
|
pkt_stream->max_pkt_len = pkt->len;
|
|
|
|
|
|
|
|
|
|
if (pkt->valid)
|
|
|
|
|
pkt_stream->nb_valid_entries++;
|
|
|
|
|
|
|
|
|
|
pkt_nb++;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -796,10 +832,10 @@ static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts,
|
|
|
|
|
struct pkt_stream *pkt_stream;
|
|
|
|
|
|
|
|
|
|
pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true);
|
|
|
|
|
test->ifobj_tx->pkt_stream = pkt_stream;
|
|
|
|
|
test->ifobj_tx->xsk->pkt_stream = pkt_stream;
|
|
|
|
|
|
|
|
|
|
pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false);
|
|
|
|
|
test->ifobj_rx->pkt_stream = pkt_stream;
|
|
|
|
|
test->ifobj_rx->xsk->pkt_stream = pkt_stream;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void pkt_print_data(u32 *data, u32 cnt)
|
|
|
|
|
@ -1004,145 +1040,190 @@ static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
|
|
|
|
|
return TEST_PASS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int receive_pkts(struct test_spec *test, struct pollfd *fds)
|
|
|
|
|
static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk)
|
|
|
|
|
{
|
|
|
|
|
struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
|
|
|
|
|
struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream;
|
|
|
|
|
struct xsk_socket_info *xsk = test->ifobj_rx->xsk;
|
|
|
|
|
u32 frags_processed = 0, nb_frags = 0, pkt_len = 0;
|
|
|
|
|
u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0;
|
|
|
|
|
struct pkt_stream *pkt_stream = xsk->pkt_stream;
|
|
|
|
|
struct ifobject *ifobj = test->ifobj_rx;
|
|
|
|
|
struct xsk_umem_info *umem = xsk->umem;
|
|
|
|
|
struct pollfd fds = { };
|
|
|
|
|
struct pkt *pkt;
|
|
|
|
|
u64 first_addr;
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
ret = gettimeofday(&tv_now, NULL);
|
|
|
|
|
fds.fd = xsk_socket__fd(xsk->xsk);
|
|
|
|
|
fds.events = POLLIN;
|
|
|
|
|
|
|
|
|
|
ret = kick_rx(xsk);
|
|
|
|
|
if (ret)
|
|
|
|
|
exit_with_error(errno);
|
|
|
|
|
timeradd(&tv_now, &tv_timeout, &tv_end);
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
|
|
|
|
|
pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
|
|
|
|
|
while (pkt) {
|
|
|
|
|
u32 frags_processed = 0, nb_frags = 0, pkt_len = 0;
|
|
|
|
|
u64 first_addr;
|
|
|
|
|
|
|
|
|
|
ret = gettimeofday(&tv_now, NULL);
|
|
|
|
|
if (ret)
|
|
|
|
|
exit_with_error(errno);
|
|
|
|
|
if (timercmp(&tv_now, &tv_end, >)) {
|
|
|
|
|
ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
|
|
|
|
|
if (ifobj->use_poll) {
|
|
|
|
|
ret = poll(&fds, 1, POLL_TMOUT);
|
|
|
|
|
if (ret < 0)
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
|
|
|
|
|
if (!ret) {
|
|
|
|
|
if (!is_umem_valid(test->ifobj_tx))
|
|
|
|
|
return TEST_PASS;
|
|
|
|
|
|
|
|
|
|
ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
|
|
|
|
|
return TEST_CONTINUE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ret = kick_rx(xsk);
|
|
|
|
|
if (ret)
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
if (!(fds.revents & POLLIN))
|
|
|
|
|
return TEST_CONTINUE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (ifobj->use_poll) {
|
|
|
|
|
ret = poll(fds, 1, POLL_TMOUT);
|
|
|
|
|
if (ret < 0)
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
|
|
|
|
|
if (!rcvd)
|
|
|
|
|
return TEST_CONTINUE;
|
|
|
|
|
|
|
|
|
|
if (!ret) {
|
|
|
|
|
if (!is_umem_valid(test->ifobj_tx))
|
|
|
|
|
return TEST_PASS;
|
|
|
|
|
|
|
|
|
|
ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
if (ifobj->use_fill_ring) {
|
|
|
|
|
ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
|
|
|
|
|
while (ret != rcvd) {
|
|
|
|
|
if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
|
|
|
|
|
ret = poll(&fds, 1, POLL_TMOUT);
|
|
|
|
|
if (ret < 0)
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!(fds->revents & POLLIN))
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
|
|
|
|
|
if (!rcvd)
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
if (ifobj->use_fill_ring) {
|
|
|
|
|
ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
|
|
|
|
|
while (ret != rcvd) {
|
|
|
|
|
if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
|
|
|
|
|
ret = poll(fds, 1, POLL_TMOUT);
|
|
|
|
|
if (ret < 0)
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
}
|
|
|
|
|
ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
while (frags_processed < rcvd) {
|
|
|
|
|
const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
|
|
|
|
|
u64 addr = desc->addr, orig;
|
|
|
|
|
while (frags_processed < rcvd) {
|
|
|
|
|
const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
|
|
|
|
|
u64 addr = desc->addr, orig;
|
|
|
|
|
|
|
|
|
|
orig = xsk_umem__extract_addr(addr);
|
|
|
|
|
addr = xsk_umem__add_offset_to_addr(addr);
|
|
|
|
|
orig = xsk_umem__extract_addr(addr);
|
|
|
|
|
addr = xsk_umem__add_offset_to_addr(addr);
|
|
|
|
|
|
|
|
|
|
if (!nb_frags) {
|
|
|
|
|
pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
|
|
|
|
|
if (!pkt) {
|
|
|
|
|
ksft_print_msg("[%s] received too many packets addr: %lx len %u\n",
|
|
|
|
|
__func__, addr, desc->len);
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n",
|
|
|
|
|
addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid);
|
|
|
|
|
|
|
|
|
|
if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) ||
|
|
|
|
|
!is_offset_correct(umem, pkt, addr) ||
|
|
|
|
|
(ifobj->use_metadata && !is_metadata_correct(pkt, umem->buffer, addr)))
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
|
|
|
|
|
if (!nb_frags++)
|
|
|
|
|
first_addr = addr;
|
|
|
|
|
frags_processed++;
|
|
|
|
|
pkt_len += desc->len;
|
|
|
|
|
if (ifobj->use_fill_ring)
|
|
|
|
|
*xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
|
|
|
|
|
|
|
|
|
|
if (pkt_continues(desc->options))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
/* The complete packet has been received */
|
|
|
|
|
if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) ||
|
|
|
|
|
!is_offset_correct(umem, pkt, addr))
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
|
|
|
|
|
pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
|
|
|
|
|
nb_frags = 0;
|
|
|
|
|
pkt_len = 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (nb_frags) {
|
|
|
|
|
/* In the middle of a packet. Start over from beginning of packet. */
|
|
|
|
|
idx_rx -= nb_frags;
|
|
|
|
|
xsk_ring_cons__cancel(&xsk->rx, nb_frags);
|
|
|
|
|
if (ifobj->use_fill_ring) {
|
|
|
|
|
idx_fq -= nb_frags;
|
|
|
|
|
xsk_ring_prod__cancel(&umem->fq, nb_frags);
|
|
|
|
|
}
|
|
|
|
|
frags_processed -= nb_frags;
|
|
|
|
|
}
|
|
|
|
|
print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n",
|
|
|
|
|
addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid);
|
|
|
|
|
|
|
|
|
|
if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) ||
|
|
|
|
|
!is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata &&
|
|
|
|
|
!is_metadata_correct(pkt, umem->buffer, addr)))
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
|
|
|
|
|
if (!nb_frags++)
|
|
|
|
|
first_addr = addr;
|
|
|
|
|
frags_processed++;
|
|
|
|
|
pkt_len += desc->len;
|
|
|
|
|
if (ifobj->use_fill_ring)
|
|
|
|
|
xsk_ring_prod__submit(&umem->fq, frags_processed);
|
|
|
|
|
if (ifobj->release_rx)
|
|
|
|
|
xsk_ring_cons__release(&xsk->rx, frags_processed);
|
|
|
|
|
*xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
|
|
|
|
|
|
|
|
|
|
pthread_mutex_lock(&pacing_mutex);
|
|
|
|
|
pkts_in_flight -= pkts_sent;
|
|
|
|
|
pthread_mutex_unlock(&pacing_mutex);
|
|
|
|
|
pkts_sent = 0;
|
|
|
|
|
if (pkt_continues(desc->options))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
/* The complete packet has been received */
|
|
|
|
|
if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) ||
|
|
|
|
|
!is_offset_correct(umem, pkt, addr))
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
|
|
|
|
|
pkt_stream->nb_rx_pkts++;
|
|
|
|
|
nb_frags = 0;
|
|
|
|
|
pkt_len = 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (nb_frags) {
|
|
|
|
|
/* In the middle of a packet. Start over from beginning of packet. */
|
|
|
|
|
idx_rx -= nb_frags;
|
|
|
|
|
xsk_ring_cons__cancel(&xsk->rx, nb_frags);
|
|
|
|
|
if (ifobj->use_fill_ring) {
|
|
|
|
|
idx_fq -= nb_frags;
|
|
|
|
|
xsk_ring_prod__cancel(&umem->fq, nb_frags);
|
|
|
|
|
}
|
|
|
|
|
frags_processed -= nb_frags;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (ifobj->use_fill_ring)
|
|
|
|
|
xsk_ring_prod__submit(&umem->fq, frags_processed);
|
|
|
|
|
if (ifobj->release_rx)
|
|
|
|
|
xsk_ring_cons__release(&xsk->rx, frags_processed);
|
|
|
|
|
|
|
|
|
|
pthread_mutex_lock(&pacing_mutex);
|
|
|
|
|
pkts_in_flight -= pkts_sent;
|
|
|
|
|
pthread_mutex_unlock(&pacing_mutex);
|
|
|
|
|
pkts_sent = 0;
|
|
|
|
|
|
|
|
|
|
return TEST_CONTINUE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num,
|
|
|
|
|
unsigned long *bitmap)
|
|
|
|
|
{
|
|
|
|
|
struct pkt_stream *pkt_stream = xsk->pkt_stream;
|
|
|
|
|
|
|
|
|
|
if (!pkt_stream) {
|
|
|
|
|
__set_bit(sock_num, bitmap);
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) {
|
|
|
|
|
__set_bit(sock_num, bitmap);
|
|
|
|
|
if (bitmap_full(bitmap, test->nb_sockets))
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int receive_pkts(struct test_spec *test)
|
|
|
|
|
{
|
|
|
|
|
struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
|
|
|
|
|
DECLARE_BITMAP(bitmap, test->nb_sockets);
|
|
|
|
|
struct xsk_socket_info *xsk;
|
|
|
|
|
u32 sock_num = 0;
|
|
|
|
|
int res, ret;
|
|
|
|
|
|
|
|
|
|
ret = gettimeofday(&tv_now, NULL);
|
|
|
|
|
if (ret)
|
|
|
|
|
exit_with_error(errno);
|
|
|
|
|
|
|
|
|
|
timeradd(&tv_now, &tv_timeout, &tv_end);
|
|
|
|
|
|
|
|
|
|
while (1) {
|
|
|
|
|
xsk = &test->ifobj_rx->xsk_arr[sock_num];
|
|
|
|
|
|
|
|
|
|
if ((all_packets_received(test, xsk, sock_num, bitmap)))
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
res = __receive_pkts(test, xsk);
|
|
|
|
|
if (!(res == TEST_PASS || res == TEST_CONTINUE))
|
|
|
|
|
return res;
|
|
|
|
|
|
|
|
|
|
ret = gettimeofday(&tv_now, NULL);
|
|
|
|
|
if (ret)
|
|
|
|
|
exit_with_error(errno);
|
|
|
|
|
|
|
|
|
|
if (timercmp(&tv_now, &tv_end, >)) {
|
|
|
|
|
ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
}
|
|
|
|
|
sock_num = (sock_num + 1) % test->nb_sockets;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return TEST_PASS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int __send_pkts(struct ifobject *ifobject, struct pollfd *fds, bool timeout)
|
|
|
|
|
static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout)
|
|
|
|
|
{
|
|
|
|
|
u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len;
|
|
|
|
|
struct pkt_stream *pkt_stream = ifobject->pkt_stream;
|
|
|
|
|
struct xsk_socket_info *xsk = ifobject->xsk;
|
|
|
|
|
struct pkt_stream *pkt_stream = xsk->pkt_stream;
|
|
|
|
|
struct xsk_umem_info *umem = ifobject->umem;
|
|
|
|
|
bool use_poll = ifobject->use_poll;
|
|
|
|
|
struct pollfd fds = { };
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len);
|
|
|
|
|
@ -1154,9 +1235,12 @@ static int __send_pkts(struct ifobject *ifobject, struct pollfd *fds, bool timeo
|
|
|
|
|
return TEST_CONTINUE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fds.fd = xsk_socket__fd(xsk->xsk);
|
|
|
|
|
fds.events = POLLOUT;
|
|
|
|
|
|
|
|
|
|
while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) {
|
|
|
|
|
if (use_poll) {
|
|
|
|
|
ret = poll(fds, 1, POLL_TMOUT);
|
|
|
|
|
ret = poll(&fds, 1, POLL_TMOUT);
|
|
|
|
|
if (timeout) {
|
|
|
|
|
if (ret < 0) {
|
|
|
|
|
ksft_print_msg("ERROR: [%s] Poll error %d\n",
|
|
|
|
|
@ -1207,7 +1291,7 @@ static int __send_pkts(struct ifobject *ifobject, struct pollfd *fds, bool timeo
|
|
|
|
|
tx_desc->options = 0;
|
|
|
|
|
}
|
|
|
|
|
if (pkt->valid)
|
|
|
|
|
pkt_generate(ifobject, tx_desc->addr, tx_desc->len, pkt->pkt_nb,
|
|
|
|
|
pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb,
|
|
|
|
|
bytes_written);
|
|
|
|
|
bytes_written += tx_desc->len;
|
|
|
|
|
|
|
|
|
|
@ -1235,7 +1319,7 @@ static int __send_pkts(struct ifobject *ifobject, struct pollfd *fds, bool timeo
|
|
|
|
|
xsk->outstanding_tx += valid_frags;
|
|
|
|
|
|
|
|
|
|
if (use_poll) {
|
|
|
|
|
ret = poll(fds, 1, POLL_TMOUT);
|
|
|
|
|
ret = poll(&fds, 1, POLL_TMOUT);
|
|
|
|
|
if (ret <= 0) {
|
|
|
|
|
if (ret == 0 && timeout)
|
|
|
|
|
return TEST_PASS;
|
|
|
|
|
@ -1281,27 +1365,43 @@ static int wait_for_tx_completion(struct xsk_socket_info *xsk)
|
|
|
|
|
return TEST_PASS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool all_packets_sent(struct test_spec *test, unsigned long *bitmap)
|
|
|
|
|
{
|
|
|
|
|
return bitmap_full(bitmap, test->nb_sockets);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
|
|
|
|
|
{
|
|
|
|
|
struct pkt_stream *pkt_stream = ifobject->pkt_stream;
|
|
|
|
|
bool timeout = !is_umem_valid(test->ifobj_rx);
|
|
|
|
|
struct pollfd fds = { };
|
|
|
|
|
u32 ret;
|
|
|
|
|
DECLARE_BITMAP(bitmap, test->nb_sockets);
|
|
|
|
|
u32 i, ret;
|
|
|
|
|
|
|
|
|
|
fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
|
|
|
|
|
fds.events = POLLOUT;
|
|
|
|
|
while (!(all_packets_sent(test, bitmap))) {
|
|
|
|
|
for (i = 0; i < test->nb_sockets; i++) {
|
|
|
|
|
struct pkt_stream *pkt_stream;
|
|
|
|
|
|
|
|
|
|
while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) {
|
|
|
|
|
ret = __send_pkts(ifobject, &fds, timeout);
|
|
|
|
|
if (ret == TEST_CONTINUE && !test->fail)
|
|
|
|
|
continue;
|
|
|
|
|
if ((ret || test->fail) && !timeout)
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
if (ret == TEST_PASS && timeout)
|
|
|
|
|
return ret;
|
|
|
|
|
pkt_stream = ifobject->xsk_arr[i].pkt_stream;
|
|
|
|
|
if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) {
|
|
|
|
|
__set_bit(i, bitmap);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout);
|
|
|
|
|
if (ret == TEST_CONTINUE && !test->fail)
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
if ((ret || test->fail) && !timeout)
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
|
|
|
|
|
if (ret == TEST_PASS && timeout)
|
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
|
|
ret = wait_for_tx_completion(&ifobject->xsk_arr[i]);
|
|
|
|
|
if (ret)
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return wait_for_tx_completion(ifobject->xsk);
|
|
|
|
|
return TEST_PASS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)
|
|
|
|
|
@ -1347,8 +1447,8 @@ static int validate_rx_dropped(struct ifobject *ifobject)
|
|
|
|
|
* packet being invalid). Since the last packet may or may not have
|
|
|
|
|
* been dropped already, both outcomes must be allowed.
|
|
|
|
|
*/
|
|
|
|
|
if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2 ||
|
|
|
|
|
stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2 - 1)
|
|
|
|
|
if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 ||
|
|
|
|
|
stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1)
|
|
|
|
|
return TEST_PASS;
|
|
|
|
|
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
@ -1412,9 +1512,10 @@ static int validate_tx_invalid_descs(struct ifobject *ifobject)
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) {
|
|
|
|
|
if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) {
|
|
|
|
|
ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%u] expected [%u]\n",
|
|
|
|
|
__func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);
|
|
|
|
|
__func__, stats.tx_invalid_descs,
|
|
|
|
|
ifobject->xsk->pkt_stream->nb_pkts);
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -1506,6 +1607,7 @@ static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
|
|
|
|
|
LIBBPF_OPTS(bpf_xdp_query_opts, opts);
|
|
|
|
|
void *bufs;
|
|
|
|
|
int ret;
|
|
|
|
|
u32 i;
|
|
|
|
|
|
|
|
|
|
if (ifobject->umem->unaligned_mode)
|
|
|
|
|
mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB;
|
|
|
|
|
@ -1528,11 +1630,14 @@ static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
|
|
|
|
|
if (!ifobject->rx_on)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream, ifobject->use_fill_ring);
|
|
|
|
|
xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring);
|
|
|
|
|
|
|
|
|
|
ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk);
|
|
|
|
|
if (ret)
|
|
|
|
|
exit_with_error(errno);
|
|
|
|
|
for (i = 0; i < test->nb_sockets; i++) {
|
|
|
|
|
ifobject->xsk = &ifobject->xsk_arr[i];
|
|
|
|
|
ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i);
|
|
|
|
|
if (ret)
|
|
|
|
|
exit_with_error(errno);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void *worker_testapp_validate_tx(void *arg)
|
|
|
|
|
@ -1562,14 +1667,13 @@ static void *worker_testapp_validate_rx(void *arg)
|
|
|
|
|
{
|
|
|
|
|
struct test_spec *test = (struct test_spec *)arg;
|
|
|
|
|
struct ifobject *ifobject = test->ifobj_rx;
|
|
|
|
|
struct pollfd fds = { };
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
if (test->current_step == 1) {
|
|
|
|
|
thread_common_ops(test, ifobject);
|
|
|
|
|
} else {
|
|
|
|
|
xsk_clear_xskmap(ifobject->xskmap);
|
|
|
|
|
err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk);
|
|
|
|
|
err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0);
|
|
|
|
|
if (err) {
|
|
|
|
|
ksft_print_msg("Error: Failed to update xskmap, error %s\n",
|
|
|
|
|
strerror(-err));
|
|
|
|
|
@ -1577,12 +1681,9 @@ static void *worker_testapp_validate_rx(void *arg)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
|
|
|
|
|
fds.events = POLLIN;
|
|
|
|
|
|
|
|
|
|
pthread_barrier_wait(&barr);
|
|
|
|
|
|
|
|
|
|
err = receive_pkts(test, &fds);
|
|
|
|
|
err = receive_pkts(test);
|
|
|
|
|
|
|
|
|
|
if (!err && ifobject->validation_func)
|
|
|
|
|
err = ifobject->validation_func(ifobject);
|
|
|
|
|
@ -1691,11 +1792,11 @@ static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *i
|
|
|
|
|
if (ifobj2) {
|
|
|
|
|
if (pthread_barrier_init(&barr, NULL, 2))
|
|
|
|
|
exit_with_error(errno);
|
|
|
|
|
pkt_stream_reset(ifobj2->pkt_stream);
|
|
|
|
|
pkt_stream_reset(ifobj2->xsk->pkt_stream);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
test->current_step++;
|
|
|
|
|
pkt_stream_reset(ifobj1->pkt_stream);
|
|
|
|
|
pkt_stream_reset(ifobj1->xsk->pkt_stream);
|
|
|
|
|
pkts_in_flight = 0;
|
|
|
|
|
|
|
|
|
|
signal(SIGUSR1, handler);
|
|
|
|
|
@ -1719,9 +1820,15 @@ static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *i
|
|
|
|
|
pthread_join(t0, NULL);
|
|
|
|
|
|
|
|
|
|
if (test->total_steps == test->current_step || test->fail) {
|
|
|
|
|
u32 i;
|
|
|
|
|
|
|
|
|
|
if (ifobj2)
|
|
|
|
|
xsk_socket__delete(ifobj2->xsk->xsk);
|
|
|
|
|
xsk_socket__delete(ifobj1->xsk->xsk);
|
|
|
|
|
for (i = 0; i < test->nb_sockets; i++)
|
|
|
|
|
xsk_socket__delete(ifobj2->xsk_arr[i].xsk);
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < test->nb_sockets; i++)
|
|
|
|
|
xsk_socket__delete(ifobj1->xsk_arr[i].xsk);
|
|
|
|
|
|
|
|
|
|
testapp_clean_xsk_umem(ifobj1);
|
|
|
|
|
if (ifobj2 && !ifobj2->shared_umem)
|
|
|
|
|
testapp_clean_xsk_umem(ifobj2);
|
|
|
|
|
@ -1793,16 +1900,18 @@ static int testapp_bidirectional(struct test_spec *test)
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx)
|
|
|
|
|
static int swap_xsk_resources(struct test_spec *test)
|
|
|
|
|
{
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
xsk_socket__delete(ifobj_tx->xsk->xsk);
|
|
|
|
|
xsk_socket__delete(ifobj_rx->xsk->xsk);
|
|
|
|
|
ifobj_tx->xsk = &ifobj_tx->xsk_arr[1];
|
|
|
|
|
ifobj_rx->xsk = &ifobj_rx->xsk_arr[1];
|
|
|
|
|
test->ifobj_tx->xsk_arr[0].pkt_stream = NULL;
|
|
|
|
|
test->ifobj_rx->xsk_arr[0].pkt_stream = NULL;
|
|
|
|
|
test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default;
|
|
|
|
|
test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default;
|
|
|
|
|
test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1];
|
|
|
|
|
test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1];
|
|
|
|
|
|
|
|
|
|
ret = xsk_update_xskmap(ifobj_rx->xskmap, ifobj_rx->xsk->xsk);
|
|
|
|
|
ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0);
|
|
|
|
|
if (ret)
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
|
|
|
|
|
@ -1816,7 +1925,7 @@ static int testapp_xdp_prog_cleanup(struct test_spec *test)
|
|
|
|
|
if (testapp_validate_traffic(test))
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
|
|
|
|
|
if (swap_xsk_resources(test->ifobj_tx, test->ifobj_rx))
|
|
|
|
|
if (swap_xsk_resources(test))
|
|
|
|
|
return TEST_FAILURE;
|
|
|
|
|
return testapp_validate_traffic(test);
|
|
|
|
|
}
|
|
|
|
|
@ -1852,8 +1961,7 @@ static int testapp_stats_tx_invalid_descs(struct test_spec *test)
|
|
|
|
|
static int testapp_stats_rx_full(struct test_spec *test)
|
|
|
|
|
{
|
|
|
|
|
pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);
|
|
|
|
|
test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
|
|
|
|
|
DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
|
|
|
|
|
test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
|
|
|
|
|
|
|
|
|
|
test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;
|
|
|
|
|
test->ifobj_rx->release_rx = false;
|
|
|
|
|
@ -1864,8 +1972,7 @@ static int testapp_stats_rx_full(struct test_spec *test)
|
|
|
|
|
static int testapp_stats_fill_empty(struct test_spec *test)
|
|
|
|
|
{
|
|
|
|
|
pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);
|
|
|
|
|
test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
|
|
|
|
|
DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
|
|
|
|
|
test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
|
|
|
|
|
|
|
|
|
|
test->ifobj_rx->use_fill_ring = false;
|
|
|
|
|
test->ifobj_rx->validation_func = validate_fill_empty;
|
|
|
|
|
@ -2031,6 +2138,23 @@ static int testapp_xdp_metadata_copy(struct test_spec *test)
|
|
|
|
|
return testapp_validate_traffic(test);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int testapp_xdp_shared_umem(struct test_spec *test)
|
|
|
|
|
{
|
|
|
|
|
struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
|
|
|
|
|
struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
|
|
|
|
|
|
|
|
|
|
test->total_steps = 1;
|
|
|
|
|
test->nb_sockets = 2;
|
|
|
|
|
|
|
|
|
|
test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem,
|
|
|
|
|
skel_tx->progs.xsk_xdp_shared_umem,
|
|
|
|
|
skel_rx->maps.xsk, skel_tx->maps.xsk);
|
|
|
|
|
|
|
|
|
|
pkt_stream_even_odd_sequence(test);
|
|
|
|
|
|
|
|
|
|
return testapp_validate_traffic(test);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int testapp_poll_txq_tmout(struct test_spec *test)
|
|
|
|
|
{
|
|
|
|
|
test->ifobj_tx->use_poll = true;
|
|
|
|
|
@ -2117,15 +2241,11 @@ static bool hugepages_present(void)
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac,
|
|
|
|
|
thread_func_t func_ptr)
|
|
|
|
|
static void init_iface(struct ifobject *ifobj, thread_func_t func_ptr)
|
|
|
|
|
{
|
|
|
|
|
LIBBPF_OPTS(bpf_xdp_query_opts, query_opts);
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN);
|
|
|
|
|
memcpy(ifobj->src_mac, src_mac, ETH_ALEN);
|
|
|
|
|
|
|
|
|
|
ifobj->func_ptr = func_ptr;
|
|
|
|
|
|
|
|
|
|
err = xsk_load_xdp_programs(ifobj);
|
|
|
|
|
@ -2336,6 +2456,7 @@ static const struct test_spec tests[] = {
|
|
|
|
|
{.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty},
|
|
|
|
|
{.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup},
|
|
|
|
|
{.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop},
|
|
|
|
|
{.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem},
|
|
|
|
|
{.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata},
|
|
|
|
|
{.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb},
|
|
|
|
|
{.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb},
|
|
|
|
|
@ -2401,12 +2522,12 @@ int main(int argc, char **argv)
|
|
|
|
|
modes++;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
init_iface(ifobj_rx, MAC1, MAC2, worker_testapp_validate_rx);
|
|
|
|
|
init_iface(ifobj_tx, MAC2, MAC1, worker_testapp_validate_tx);
|
|
|
|
|
init_iface(ifobj_rx, worker_testapp_validate_rx);
|
|
|
|
|
init_iface(ifobj_tx, worker_testapp_validate_tx);
|
|
|
|
|
|
|
|
|
|
test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]);
|
|
|
|
|
tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, MIN_PKT_SIZE);
|
|
|
|
|
rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, MIN_PKT_SIZE);
|
|
|
|
|
tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
|
|
|
|
|
rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
|
|
|
|
|
if (!tx_pkt_stream_default || !rx_pkt_stream_default)
|
|
|
|
|
exit_with_error(ENOMEM);
|
|
|
|
|
test.tx_pkt_stream_default = tx_pkt_stream_default;
|
|
|
|
|
|