1be43a6b2SStanislav Fomichev // SPDX-License-Identifier: GPL-2.0
2be43a6b2SStanislav Fomichev /*
3be43a6b2SStanislav Fomichev * tcpdevmem netcat. Works similarly to netcat but does device memory TCP
4be43a6b2SStanislav Fomichev * instead of regular TCP. Uses udmabuf to mock a dmabuf provider.
5be43a6b2SStanislav Fomichev *
6be43a6b2SStanislav Fomichev * Usage:
7be43a6b2SStanislav Fomichev *
8be43a6b2SStanislav Fomichev * On server:
9be43a6b2SStanislav Fomichev * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201
10be43a6b2SStanislav Fomichev *
11be43a6b2SStanislav Fomichev * On client:
122f1a805fSMina Almasry * echo -n "hello\nworld" | \
132f1a805fSMina Almasry * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1
14be43a6b2SStanislav Fomichev *
152f1a805fSMina Almasry * Note this is compatible with regular netcat. i.e. the sender or receiver can
162f1a805fSMina Almasry * be replaced with regular netcat to test the RX or TX path in isolation.
172f1a805fSMina Almasry *
182f1a805fSMina Almasry * Test data validation (devmem TCP on RX only):
19be43a6b2SStanislav Fomichev *
20be43a6b2SStanislav Fomichev * On server:
21be43a6b2SStanislav Fomichev * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201 -v 7
22be43a6b2SStanislav Fomichev *
23be43a6b2SStanislav Fomichev * On client:
24be43a6b2SStanislav Fomichev * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06) | \
252f1a805fSMina Almasry * head -c 1G | \
26be43a6b2SStanislav Fomichev * nc <server IP> 5201 -p 5201
27be43a6b2SStanislav Fomichev *
282f1a805fSMina Almasry * Test data validation (devmem TCP on RX and TX, validation happens on RX):
29be43a6b2SStanislav Fomichev *
302f1a805fSMina Almasry * On server:
312f1a805fSMina Almasry * ncdevmem -s <server IP> [-c <client IP>] -l -p 5201 -v 8 -f eth1
322f1a805fSMina Almasry *
332f1a805fSMina Almasry * On client:
342f1a805fSMina Almasry * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06\\x07) | \
352f1a805fSMina Almasry * head -c 1M | \
362f1a805fSMina Almasry * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1
37be43a6b2SStanislav Fomichev */
38be43a6b2SStanislav Fomichev #define _GNU_SOURCE
39be43a6b2SStanislav Fomichev #define __EXPORTED_HEADERS__
40be43a6b2SStanislav Fomichev
41be43a6b2SStanislav Fomichev #include <linux/uio.h>
42be43a6b2SStanislav Fomichev #include <stdio.h>
43be43a6b2SStanislav Fomichev #include <stdlib.h>
44be43a6b2SStanislav Fomichev #include <unistd.h>
45be43a6b2SStanislav Fomichev #include <stdbool.h>
46be43a6b2SStanislav Fomichev #include <string.h>
47be43a6b2SStanislav Fomichev #include <errno.h>
48be43a6b2SStanislav Fomichev #define __iovec_defined
49be43a6b2SStanislav Fomichev #include <fcntl.h>
50be43a6b2SStanislav Fomichev #include <malloc.h>
51be43a6b2SStanislav Fomichev #include <error.h>
522f1a805fSMina Almasry #include <poll.h>
53be43a6b2SStanislav Fomichev
54be43a6b2SStanislav Fomichev #include <arpa/inet.h>
55be43a6b2SStanislav Fomichev #include <sys/socket.h>
56be43a6b2SStanislav Fomichev #include <sys/mman.h>
57be43a6b2SStanislav Fomichev #include <sys/ioctl.h>
58be43a6b2SStanislav Fomichev #include <sys/syscall.h>
592f1a805fSMina Almasry #include <sys/time.h>
60be43a6b2SStanislav Fomichev
61be43a6b2SStanislav Fomichev #include <linux/memfd.h>
62be43a6b2SStanislav Fomichev #include <linux/dma-buf.h>
632f1a805fSMina Almasry #include <linux/errqueue.h>
64be43a6b2SStanislav Fomichev #include <linux/udmabuf.h>
65be43a6b2SStanislav Fomichev #include <linux/types.h>
66be43a6b2SStanislav Fomichev #include <linux/netlink.h>
67be43a6b2SStanislav Fomichev #include <linux/genetlink.h>
68be43a6b2SStanislav Fomichev #include <linux/netdev.h>
69be43a6b2SStanislav Fomichev #include <linux/ethtool_netlink.h>
70be43a6b2SStanislav Fomichev #include <time.h>
71be43a6b2SStanislav Fomichev #include <net/if.h>
72be43a6b2SStanislav Fomichev
73be43a6b2SStanislav Fomichev #include "netdev-user.h"
74be43a6b2SStanislav Fomichev #include "ethtool-user.h"
75be43a6b2SStanislav Fomichev #include <ynl.h>
76be43a6b2SStanislav Fomichev
77be43a6b2SStanislav Fomichev #define PAGE_SHIFT 12
78be43a6b2SStanislav Fomichev #define TEST_PREFIX "ncdevmem"
79be43a6b2SStanislav Fomichev #define NUM_PAGES 16000
80be43a6b2SStanislav Fomichev
81be43a6b2SStanislav Fomichev #ifndef MSG_SOCK_DEVMEM
82be43a6b2SStanislav Fomichev #define MSG_SOCK_DEVMEM 0x2000000
83be43a6b2SStanislav Fomichev #endif
84be43a6b2SStanislav Fomichev
8561f24c68SStanislav Fomichev #define MAX_IOV 1024
8661f24c68SStanislav Fomichev
8761f24c68SStanislav Fomichev static size_t max_chunk;
88be43a6b2SStanislav Fomichev static char *server_ip;
89be43a6b2SStanislav Fomichev static char *client_ip;
90be43a6b2SStanislav Fomichev static char *port;
91be43a6b2SStanislav Fomichev static size_t do_validation;
92be43a6b2SStanislav Fomichev static int start_queue = -1;
93be43a6b2SStanislav Fomichev static int num_queues = -1;
94be43a6b2SStanislav Fomichev static char *ifname;
95be43a6b2SStanislav Fomichev static unsigned int ifindex;
96be43a6b2SStanislav Fomichev static unsigned int dmabuf_id;
972f1a805fSMina Almasry static uint32_t tx_dmabuf_id;
982f1a805fSMina Almasry static int waittime_ms = 500;
99be43a6b2SStanislav Fomichev
100be43a6b2SStanislav Fomichev struct memory_buffer {
101be43a6b2SStanislav Fomichev int fd;
102be43a6b2SStanislav Fomichev size_t size;
103be43a6b2SStanislav Fomichev
104be43a6b2SStanislav Fomichev int devfd;
105be43a6b2SStanislav Fomichev int memfd;
106be43a6b2SStanislav Fomichev char *buf_mem;
107be43a6b2SStanislav Fomichev };
108be43a6b2SStanislav Fomichev
109be43a6b2SStanislav Fomichev struct memory_provider {
110be43a6b2SStanislav Fomichev struct memory_buffer *(*alloc)(size_t size);
111be43a6b2SStanislav Fomichev void (*free)(struct memory_buffer *ctx);
1122f1a805fSMina Almasry void (*memcpy_to_device)(struct memory_buffer *dst, size_t off,
1132f1a805fSMina Almasry void *src, int n);
114be43a6b2SStanislav Fomichev void (*memcpy_from_device)(void *dst, struct memory_buffer *src,
115be43a6b2SStanislav Fomichev size_t off, int n);
116be43a6b2SStanislav Fomichev };
117be43a6b2SStanislav Fomichev
udmabuf_alloc(size_t size)118be43a6b2SStanislav Fomichev static struct memory_buffer *udmabuf_alloc(size_t size)
119be43a6b2SStanislav Fomichev {
120be43a6b2SStanislav Fomichev struct udmabuf_create create;
121be43a6b2SStanislav Fomichev struct memory_buffer *ctx;
122be43a6b2SStanislav Fomichev int ret;
123be43a6b2SStanislav Fomichev
124be43a6b2SStanislav Fomichev ctx = malloc(sizeof(*ctx));
125be43a6b2SStanislav Fomichev if (!ctx)
126be43a6b2SStanislav Fomichev error(1, ENOMEM, "malloc failed");
127be43a6b2SStanislav Fomichev
128be43a6b2SStanislav Fomichev ctx->size = size;
129be43a6b2SStanislav Fomichev
130be43a6b2SStanislav Fomichev ctx->devfd = open("/dev/udmabuf", O_RDWR);
131be43a6b2SStanislav Fomichev if (ctx->devfd < 0)
132be43a6b2SStanislav Fomichev error(1, errno,
133be43a6b2SStanislav Fomichev "%s: [skip,no-udmabuf: Unable to access DMA buffer device file]\n",
134be43a6b2SStanislav Fomichev TEST_PREFIX);
135be43a6b2SStanislav Fomichev
136be43a6b2SStanislav Fomichev ctx->memfd = memfd_create("udmabuf-test", MFD_ALLOW_SEALING);
137be43a6b2SStanislav Fomichev if (ctx->memfd < 0)
138be43a6b2SStanislav Fomichev error(1, errno, "%s: [skip,no-memfd]\n", TEST_PREFIX);
139be43a6b2SStanislav Fomichev
140be43a6b2SStanislav Fomichev ret = fcntl(ctx->memfd, F_ADD_SEALS, F_SEAL_SHRINK);
141be43a6b2SStanislav Fomichev if (ret < 0)
142be43a6b2SStanislav Fomichev error(1, errno, "%s: [skip,fcntl-add-seals]\n", TEST_PREFIX);
143be43a6b2SStanislav Fomichev
144be43a6b2SStanislav Fomichev ret = ftruncate(ctx->memfd, size);
145be43a6b2SStanislav Fomichev if (ret == -1)
146be43a6b2SStanislav Fomichev error(1, errno, "%s: [FAIL,memfd-truncate]\n", TEST_PREFIX);
147be43a6b2SStanislav Fomichev
148be43a6b2SStanislav Fomichev memset(&create, 0, sizeof(create));
149be43a6b2SStanislav Fomichev
150be43a6b2SStanislav Fomichev create.memfd = ctx->memfd;
151be43a6b2SStanislav Fomichev create.offset = 0;
152be43a6b2SStanislav Fomichev create.size = size;
153be43a6b2SStanislav Fomichev ctx->fd = ioctl(ctx->devfd, UDMABUF_CREATE, &create);
154be43a6b2SStanislav Fomichev if (ctx->fd < 0)
155be43a6b2SStanislav Fomichev error(1, errno, "%s: [FAIL, create udmabuf]\n", TEST_PREFIX);
156be43a6b2SStanislav Fomichev
157be43a6b2SStanislav Fomichev ctx->buf_mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED,
158be43a6b2SStanislav Fomichev ctx->fd, 0);
159be43a6b2SStanislav Fomichev if (ctx->buf_mem == MAP_FAILED)
160be43a6b2SStanislav Fomichev error(1, errno, "%s: [FAIL, map udmabuf]\n", TEST_PREFIX);
161be43a6b2SStanislav Fomichev
162be43a6b2SStanislav Fomichev return ctx;
163be43a6b2SStanislav Fomichev }
164be43a6b2SStanislav Fomichev
udmabuf_free(struct memory_buffer * ctx)165be43a6b2SStanislav Fomichev static void udmabuf_free(struct memory_buffer *ctx)
166be43a6b2SStanislav Fomichev {
167be43a6b2SStanislav Fomichev munmap(ctx->buf_mem, ctx->size);
168be43a6b2SStanislav Fomichev close(ctx->fd);
169be43a6b2SStanislav Fomichev close(ctx->memfd);
170be43a6b2SStanislav Fomichev close(ctx->devfd);
171be43a6b2SStanislav Fomichev free(ctx);
172be43a6b2SStanislav Fomichev }
173be43a6b2SStanislav Fomichev
udmabuf_memcpy_to_device(struct memory_buffer * dst,size_t off,void * src,int n)1742f1a805fSMina Almasry static void udmabuf_memcpy_to_device(struct memory_buffer *dst, size_t off,
1752f1a805fSMina Almasry void *src, int n)
1762f1a805fSMina Almasry {
1772f1a805fSMina Almasry struct dma_buf_sync sync = {};
1782f1a805fSMina Almasry
1792f1a805fSMina Almasry sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE;
1802f1a805fSMina Almasry ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync);
1812f1a805fSMina Almasry
1822f1a805fSMina Almasry memcpy(dst->buf_mem + off, src, n);
1832f1a805fSMina Almasry
1842f1a805fSMina Almasry sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE;
1852f1a805fSMina Almasry ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync);
1862f1a805fSMina Almasry }
1872f1a805fSMina Almasry
udmabuf_memcpy_from_device(void * dst,struct memory_buffer * src,size_t off,int n)188be43a6b2SStanislav Fomichev static void udmabuf_memcpy_from_device(void *dst, struct memory_buffer *src,
189be43a6b2SStanislav Fomichev size_t off, int n)
190be43a6b2SStanislav Fomichev {
191be43a6b2SStanislav Fomichev struct dma_buf_sync sync = {};
192be43a6b2SStanislav Fomichev
193be43a6b2SStanislav Fomichev sync.flags = DMA_BUF_SYNC_START;
194be43a6b2SStanislav Fomichev ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync);
195be43a6b2SStanislav Fomichev
196be43a6b2SStanislav Fomichev memcpy(dst, src->buf_mem + off, n);
197be43a6b2SStanislav Fomichev
198be43a6b2SStanislav Fomichev sync.flags = DMA_BUF_SYNC_END;
199be43a6b2SStanislav Fomichev ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync);
200be43a6b2SStanislav Fomichev }
201be43a6b2SStanislav Fomichev
202be43a6b2SStanislav Fomichev static struct memory_provider udmabuf_memory_provider = {
203be43a6b2SStanislav Fomichev .alloc = udmabuf_alloc,
204be43a6b2SStanislav Fomichev .free = udmabuf_free,
2052f1a805fSMina Almasry .memcpy_to_device = udmabuf_memcpy_to_device,
206be43a6b2SStanislav Fomichev .memcpy_from_device = udmabuf_memcpy_from_device,
207be43a6b2SStanislav Fomichev };
208be43a6b2SStanislav Fomichev
209be43a6b2SStanislav Fomichev static struct memory_provider *provider = &udmabuf_memory_provider;
210be43a6b2SStanislav Fomichev
print_nonzero_bytes(void * ptr,size_t size)211be43a6b2SStanislav Fomichev static void print_nonzero_bytes(void *ptr, size_t size)
212be43a6b2SStanislav Fomichev {
213be43a6b2SStanislav Fomichev unsigned char *p = ptr;
214be43a6b2SStanislav Fomichev unsigned int i;
215be43a6b2SStanislav Fomichev
216be43a6b2SStanislav Fomichev for (i = 0; i < size; i++)
217be43a6b2SStanislav Fomichev putchar(p[i]);
218be43a6b2SStanislav Fomichev }
219be43a6b2SStanislav Fomichev
validate_buffer(void * line,size_t size)220be43a6b2SStanislav Fomichev void validate_buffer(void *line, size_t size)
221be43a6b2SStanislav Fomichev {
222be43a6b2SStanislav Fomichev static unsigned char seed = 1;
223be43a6b2SStanislav Fomichev unsigned char *ptr = line;
2242f1a805fSMina Almasry unsigned char expected;
2252f1a805fSMina Almasry static int errors;
226be43a6b2SStanislav Fomichev size_t i;
227be43a6b2SStanislav Fomichev
228be43a6b2SStanislav Fomichev for (i = 0; i < size; i++) {
2292f1a805fSMina Almasry expected = seed ? seed : '\n';
2302f1a805fSMina Almasry if (ptr[i] != expected) {
231be43a6b2SStanislav Fomichev fprintf(stderr,
232be43a6b2SStanislav Fomichev "Failed validation: expected=%u, actual=%u, index=%lu\n",
2332f1a805fSMina Almasry expected, ptr[i], i);
234be43a6b2SStanislav Fomichev errors++;
235be43a6b2SStanislav Fomichev if (errors > 20)
236be43a6b2SStanislav Fomichev error(1, 0, "validation failed.");
237be43a6b2SStanislav Fomichev }
238be43a6b2SStanislav Fomichev seed++;
239be43a6b2SStanislav Fomichev if (seed == do_validation)
240be43a6b2SStanislav Fomichev seed = 0;
241be43a6b2SStanislav Fomichev }
242be43a6b2SStanislav Fomichev
243be43a6b2SStanislav Fomichev fprintf(stdout, "Validated buffer\n");
244be43a6b2SStanislav Fomichev }
245be43a6b2SStanislav Fomichev
rxq_num(int ifindex)246be43a6b2SStanislav Fomichev static int rxq_num(int ifindex)
247be43a6b2SStanislav Fomichev {
248be43a6b2SStanislav Fomichev struct ethtool_channels_get_req *req;
249be43a6b2SStanislav Fomichev struct ethtool_channels_get_rsp *rsp;
250be43a6b2SStanislav Fomichev struct ynl_error yerr;
251be43a6b2SStanislav Fomichev struct ynl_sock *ys;
252be43a6b2SStanislav Fomichev int num = -1;
253be43a6b2SStanislav Fomichev
254be43a6b2SStanislav Fomichev ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
255be43a6b2SStanislav Fomichev if (!ys) {
256be43a6b2SStanislav Fomichev fprintf(stderr, "YNL: %s\n", yerr.msg);
257be43a6b2SStanislav Fomichev return -1;
258be43a6b2SStanislav Fomichev }
259be43a6b2SStanislav Fomichev
260be43a6b2SStanislav Fomichev req = ethtool_channels_get_req_alloc();
261be43a6b2SStanislav Fomichev ethtool_channels_get_req_set_header_dev_index(req, ifindex);
262be43a6b2SStanislav Fomichev rsp = ethtool_channels_get(ys, req);
263be43a6b2SStanislav Fomichev if (rsp)
264be43a6b2SStanislav Fomichev num = rsp->rx_count + rsp->combined_count;
265be43a6b2SStanislav Fomichev ethtool_channels_get_req_free(req);
266be43a6b2SStanislav Fomichev ethtool_channels_get_rsp_free(rsp);
267be43a6b2SStanislav Fomichev
268be43a6b2SStanislav Fomichev ynl_sock_destroy(ys);
269be43a6b2SStanislav Fomichev
270be43a6b2SStanislav Fomichev return num;
271be43a6b2SStanislav Fomichev }
272be43a6b2SStanislav Fomichev
273be43a6b2SStanislav Fomichev #define run_command(cmd, ...) \
274be43a6b2SStanislav Fomichev ({ \
275be43a6b2SStanislav Fomichev char command[256]; \
276be43a6b2SStanislav Fomichev memset(command, 0, sizeof(command)); \
277be43a6b2SStanislav Fomichev snprintf(command, sizeof(command), cmd, ##__VA_ARGS__); \
278be43a6b2SStanislav Fomichev fprintf(stderr, "Running: %s\n", command); \
279be43a6b2SStanislav Fomichev system(command); \
280be43a6b2SStanislav Fomichev })
281be43a6b2SStanislav Fomichev
reset_flow_steering(void)282be43a6b2SStanislav Fomichev static int reset_flow_steering(void)
283be43a6b2SStanislav Fomichev {
284be43a6b2SStanislav Fomichev /* Depending on the NIC, toggling ntuple off and on might not
285be43a6b2SStanislav Fomichev * be allowed. Additionally, attempting to delete existing filters
286be43a6b2SStanislav Fomichev * will fail if no filters are present. Therefore, do not enforce
287be43a6b2SStanislav Fomichev * the exit status.
288be43a6b2SStanislav Fomichev */
289be43a6b2SStanislav Fomichev
290be43a6b2SStanislav Fomichev run_command("sudo ethtool -K %s ntuple off >&2", ifname);
291be43a6b2SStanislav Fomichev run_command("sudo ethtool -K %s ntuple on >&2", ifname);
292be43a6b2SStanislav Fomichev run_command(
293be43a6b2SStanislav Fomichev "sudo ethtool -n %s | grep 'Filter:' | awk '{print $2}' | xargs -n1 ethtool -N %s delete >&2",
294be43a6b2SStanislav Fomichev ifname, ifname);
295be43a6b2SStanislav Fomichev return 0;
296be43a6b2SStanislav Fomichev }
297be43a6b2SStanislav Fomichev
tcp_data_split_str(int val)298be43a6b2SStanislav Fomichev static const char *tcp_data_split_str(int val)
299be43a6b2SStanislav Fomichev {
300be43a6b2SStanislav Fomichev switch (val) {
301be43a6b2SStanislav Fomichev case 0:
302be43a6b2SStanislav Fomichev return "off";
303be43a6b2SStanislav Fomichev case 1:
304be43a6b2SStanislav Fomichev return "auto";
305be43a6b2SStanislav Fomichev case 2:
306be43a6b2SStanislav Fomichev return "on";
307be43a6b2SStanislav Fomichev default:
308be43a6b2SStanislav Fomichev return "?";
309be43a6b2SStanislav Fomichev }
310be43a6b2SStanislav Fomichev }
311be43a6b2SStanislav Fomichev
configure_headersplit(bool on)312be43a6b2SStanislav Fomichev static int configure_headersplit(bool on)
313be43a6b2SStanislav Fomichev {
314be43a6b2SStanislav Fomichev struct ethtool_rings_get_req *get_req;
315be43a6b2SStanislav Fomichev struct ethtool_rings_get_rsp *get_rsp;
316be43a6b2SStanislav Fomichev struct ethtool_rings_set_req *req;
317be43a6b2SStanislav Fomichev struct ynl_error yerr;
318be43a6b2SStanislav Fomichev struct ynl_sock *ys;
319be43a6b2SStanislav Fomichev int ret;
320be43a6b2SStanislav Fomichev
321be43a6b2SStanislav Fomichev ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
322be43a6b2SStanislav Fomichev if (!ys) {
323be43a6b2SStanislav Fomichev fprintf(stderr, "YNL: %s\n", yerr.msg);
324be43a6b2SStanislav Fomichev return -1;
325be43a6b2SStanislav Fomichev }
326be43a6b2SStanislav Fomichev
327be43a6b2SStanislav Fomichev req = ethtool_rings_set_req_alloc();
328be43a6b2SStanislav Fomichev ethtool_rings_set_req_set_header_dev_index(req, ifindex);
329be43a6b2SStanislav Fomichev /* 0 - off, 1 - auto, 2 - on */
330be43a6b2SStanislav Fomichev ethtool_rings_set_req_set_tcp_data_split(req, on ? 2 : 0);
331be43a6b2SStanislav Fomichev ret = ethtool_rings_set(ys, req);
332be43a6b2SStanislav Fomichev if (ret < 0)
333be43a6b2SStanislav Fomichev fprintf(stderr, "YNL failed: %s\n", ys->err.msg);
334be43a6b2SStanislav Fomichev ethtool_rings_set_req_free(req);
335be43a6b2SStanislav Fomichev
336be43a6b2SStanislav Fomichev if (ret == 0) {
337be43a6b2SStanislav Fomichev get_req = ethtool_rings_get_req_alloc();
338be43a6b2SStanislav Fomichev ethtool_rings_get_req_set_header_dev_index(get_req, ifindex);
339be43a6b2SStanislav Fomichev get_rsp = ethtool_rings_get(ys, get_req);
340be43a6b2SStanislav Fomichev ethtool_rings_get_req_free(get_req);
341be43a6b2SStanislav Fomichev if (get_rsp)
342be43a6b2SStanislav Fomichev fprintf(stderr, "TCP header split: %s\n",
343be43a6b2SStanislav Fomichev tcp_data_split_str(get_rsp->tcp_data_split));
344be43a6b2SStanislav Fomichev ethtool_rings_get_rsp_free(get_rsp);
345be43a6b2SStanislav Fomichev }
346be43a6b2SStanislav Fomichev
347be43a6b2SStanislav Fomichev ynl_sock_destroy(ys);
348be43a6b2SStanislav Fomichev
349be43a6b2SStanislav Fomichev return ret;
350be43a6b2SStanislav Fomichev }
351be43a6b2SStanislav Fomichev
configure_rss(void)352be43a6b2SStanislav Fomichev static int configure_rss(void)
353be43a6b2SStanislav Fomichev {
354be43a6b2SStanislav Fomichev return run_command("sudo ethtool -X %s equal %d >&2", ifname, start_queue);
355be43a6b2SStanislav Fomichev }
356be43a6b2SStanislav Fomichev
configure_channels(unsigned int rx,unsigned int tx)357be43a6b2SStanislav Fomichev static int configure_channels(unsigned int rx, unsigned int tx)
358be43a6b2SStanislav Fomichev {
359be43a6b2SStanislav Fomichev return run_command("sudo ethtool -L %s rx %u tx %u", ifname, rx, tx);
360be43a6b2SStanislav Fomichev }
361be43a6b2SStanislav Fomichev
configure_flow_steering(struct sockaddr_in6 * server_sin)362be43a6b2SStanislav Fomichev static int configure_flow_steering(struct sockaddr_in6 *server_sin)
363be43a6b2SStanislav Fomichev {
364be43a6b2SStanislav Fomichev const char *type = "tcp6";
365be43a6b2SStanislav Fomichev const char *server_addr;
366be43a6b2SStanislav Fomichev char buf[40];
367be43a6b2SStanislav Fomichev
368be43a6b2SStanislav Fomichev inet_ntop(AF_INET6, &server_sin->sin6_addr, buf, sizeof(buf));
369be43a6b2SStanislav Fomichev server_addr = buf;
370be43a6b2SStanislav Fomichev
371be43a6b2SStanislav Fomichev if (IN6_IS_ADDR_V4MAPPED(&server_sin->sin6_addr)) {
372be43a6b2SStanislav Fomichev type = "tcp4";
373be43a6b2SStanislav Fomichev server_addr = strrchr(server_addr, ':') + 1;
374be43a6b2SStanislav Fomichev }
375be43a6b2SStanislav Fomichev
376243d47a5SMina Almasry /* Try configure 5-tuple */
377243d47a5SMina Almasry if (run_command("sudo ethtool -N %s flow-type %s %s %s dst-ip %s %s %s dst-port %s queue %d >&2",
378be43a6b2SStanislav Fomichev ifname,
379be43a6b2SStanislav Fomichev type,
380be43a6b2SStanislav Fomichev client_ip ? "src-ip" : "",
381be43a6b2SStanislav Fomichev client_ip ?: "",
382be43a6b2SStanislav Fomichev server_addr,
383be43a6b2SStanislav Fomichev client_ip ? "src-port" : "",
384be43a6b2SStanislav Fomichev client_ip ? port : "",
385243d47a5SMina Almasry port, start_queue))
386243d47a5SMina Almasry /* If that fails, try configure 3-tuple */
387243d47a5SMina Almasry if (run_command("sudo ethtool -N %s flow-type %s dst-ip %s dst-port %s queue %d >&2",
388243d47a5SMina Almasry ifname,
389243d47a5SMina Almasry type,
390243d47a5SMina Almasry server_addr,
391243d47a5SMina Almasry port, start_queue))
392243d47a5SMina Almasry /* If that fails, return error */
393243d47a5SMina Almasry return -1;
394243d47a5SMina Almasry
395243d47a5SMina Almasry return 0;
396be43a6b2SStanislav Fomichev }
397be43a6b2SStanislav Fomichev
bind_rx_queue(unsigned int ifindex,unsigned int dmabuf_fd,struct netdev_queue_id * queues,unsigned int n_queue_index,struct ynl_sock ** ys)398be43a6b2SStanislav Fomichev static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd,
399be43a6b2SStanislav Fomichev struct netdev_queue_id *queues,
400be43a6b2SStanislav Fomichev unsigned int n_queue_index, struct ynl_sock **ys)
401be43a6b2SStanislav Fomichev {
402be43a6b2SStanislav Fomichev struct netdev_bind_rx_req *req = NULL;
403be43a6b2SStanislav Fomichev struct netdev_bind_rx_rsp *rsp = NULL;
404be43a6b2SStanislav Fomichev struct ynl_error yerr;
405be43a6b2SStanislav Fomichev
406be43a6b2SStanislav Fomichev *ys = ynl_sock_create(&ynl_netdev_family, &yerr);
407be43a6b2SStanislav Fomichev if (!*ys) {
408be43a6b2SStanislav Fomichev fprintf(stderr, "YNL: %s\n", yerr.msg);
409be43a6b2SStanislav Fomichev return -1;
410be43a6b2SStanislav Fomichev }
411be43a6b2SStanislav Fomichev
412be43a6b2SStanislav Fomichev req = netdev_bind_rx_req_alloc();
413be43a6b2SStanislav Fomichev netdev_bind_rx_req_set_ifindex(req, ifindex);
414be43a6b2SStanislav Fomichev netdev_bind_rx_req_set_fd(req, dmabuf_fd);
415be43a6b2SStanislav Fomichev __netdev_bind_rx_req_set_queues(req, queues, n_queue_index);
416be43a6b2SStanislav Fomichev
417be43a6b2SStanislav Fomichev rsp = netdev_bind_rx(*ys, req);
418be43a6b2SStanislav Fomichev if (!rsp) {
419be43a6b2SStanislav Fomichev perror("netdev_bind_rx");
420be43a6b2SStanislav Fomichev goto err_close;
421be43a6b2SStanislav Fomichev }
422be43a6b2SStanislav Fomichev
423be43a6b2SStanislav Fomichev if (!rsp->_present.id) {
424be43a6b2SStanislav Fomichev perror("id not present");
425be43a6b2SStanislav Fomichev goto err_close;
426be43a6b2SStanislav Fomichev }
427be43a6b2SStanislav Fomichev
428be43a6b2SStanislav Fomichev fprintf(stderr, "got dmabuf id=%d\n", rsp->id);
429be43a6b2SStanislav Fomichev dmabuf_id = rsp->id;
430be43a6b2SStanislav Fomichev
431be43a6b2SStanislav Fomichev netdev_bind_rx_req_free(req);
432be43a6b2SStanislav Fomichev netdev_bind_rx_rsp_free(rsp);
433be43a6b2SStanislav Fomichev
434be43a6b2SStanislav Fomichev return 0;
435be43a6b2SStanislav Fomichev
436be43a6b2SStanislav Fomichev err_close:
437be43a6b2SStanislav Fomichev fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg);
438be43a6b2SStanislav Fomichev netdev_bind_rx_req_free(req);
439be43a6b2SStanislav Fomichev ynl_sock_destroy(*ys);
440be43a6b2SStanislav Fomichev return -1;
441be43a6b2SStanislav Fomichev }
442be43a6b2SStanislav Fomichev
bind_tx_queue(unsigned int ifindex,unsigned int dmabuf_fd,struct ynl_sock ** ys)4432f1a805fSMina Almasry static int bind_tx_queue(unsigned int ifindex, unsigned int dmabuf_fd,
4442f1a805fSMina Almasry struct ynl_sock **ys)
4452f1a805fSMina Almasry {
4462f1a805fSMina Almasry struct netdev_bind_tx_req *req = NULL;
4472f1a805fSMina Almasry struct netdev_bind_tx_rsp *rsp = NULL;
4482f1a805fSMina Almasry struct ynl_error yerr;
4492f1a805fSMina Almasry
4502f1a805fSMina Almasry *ys = ynl_sock_create(&ynl_netdev_family, &yerr);
4512f1a805fSMina Almasry if (!*ys) {
4522f1a805fSMina Almasry fprintf(stderr, "YNL: %s\n", yerr.msg);
4532f1a805fSMina Almasry return -1;
4542f1a805fSMina Almasry }
4552f1a805fSMina Almasry
4562f1a805fSMina Almasry req = netdev_bind_tx_req_alloc();
4572f1a805fSMina Almasry netdev_bind_tx_req_set_ifindex(req, ifindex);
4582f1a805fSMina Almasry netdev_bind_tx_req_set_fd(req, dmabuf_fd);
4592f1a805fSMina Almasry
4602f1a805fSMina Almasry rsp = netdev_bind_tx(*ys, req);
4612f1a805fSMina Almasry if (!rsp) {
4622f1a805fSMina Almasry perror("netdev_bind_tx");
4632f1a805fSMina Almasry goto err_close;
4642f1a805fSMina Almasry }
4652f1a805fSMina Almasry
4662f1a805fSMina Almasry if (!rsp->_present.id) {
4672f1a805fSMina Almasry perror("id not present");
4682f1a805fSMina Almasry goto err_close;
4692f1a805fSMina Almasry }
4702f1a805fSMina Almasry
4712f1a805fSMina Almasry fprintf(stderr, "got tx dmabuf id=%d\n", rsp->id);
4722f1a805fSMina Almasry tx_dmabuf_id = rsp->id;
4732f1a805fSMina Almasry
4742f1a805fSMina Almasry netdev_bind_tx_req_free(req);
4752f1a805fSMina Almasry netdev_bind_tx_rsp_free(rsp);
4762f1a805fSMina Almasry
4772f1a805fSMina Almasry return 0;
4782f1a805fSMina Almasry
4792f1a805fSMina Almasry err_close:
4802f1a805fSMina Almasry fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg);
4812f1a805fSMina Almasry netdev_bind_tx_req_free(req);
4822f1a805fSMina Almasry ynl_sock_destroy(*ys);
4832f1a805fSMina Almasry return -1;
4842f1a805fSMina Almasry }
4852f1a805fSMina Almasry
enable_reuseaddr(int fd)486be43a6b2SStanislav Fomichev static void enable_reuseaddr(int fd)
487be43a6b2SStanislav Fomichev {
488be43a6b2SStanislav Fomichev int opt = 1;
489be43a6b2SStanislav Fomichev int ret;
490be43a6b2SStanislav Fomichev
491be43a6b2SStanislav Fomichev ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt));
492be43a6b2SStanislav Fomichev if (ret)
493be43a6b2SStanislav Fomichev error(1, errno, "%s: [FAIL, SO_REUSEPORT]\n", TEST_PREFIX);
494be43a6b2SStanislav Fomichev
495be43a6b2SStanislav Fomichev ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
496be43a6b2SStanislav Fomichev if (ret)
497be43a6b2SStanislav Fomichev error(1, errno, "%s: [FAIL, SO_REUSEADDR]\n", TEST_PREFIX);
498be43a6b2SStanislav Fomichev }
499be43a6b2SStanislav Fomichev
parse_address(const char * str,int port,struct sockaddr_in6 * sin6)500be43a6b2SStanislav Fomichev static int parse_address(const char *str, int port, struct sockaddr_in6 *sin6)
501be43a6b2SStanislav Fomichev {
502be43a6b2SStanislav Fomichev int ret;
503be43a6b2SStanislav Fomichev
504be43a6b2SStanislav Fomichev sin6->sin6_family = AF_INET6;
505be43a6b2SStanislav Fomichev sin6->sin6_port = htons(port);
506be43a6b2SStanislav Fomichev
507be43a6b2SStanislav Fomichev ret = inet_pton(sin6->sin6_family, str, &sin6->sin6_addr);
508be43a6b2SStanislav Fomichev if (ret != 1) {
509be43a6b2SStanislav Fomichev /* fallback to plain IPv4 */
510be43a6b2SStanislav Fomichev ret = inet_pton(AF_INET, str, &sin6->sin6_addr.s6_addr32[3]);
511be43a6b2SStanislav Fomichev if (ret != 1)
512be43a6b2SStanislav Fomichev return -1;
513be43a6b2SStanislav Fomichev
514be43a6b2SStanislav Fomichev /* add ::ffff prefix */
515be43a6b2SStanislav Fomichev sin6->sin6_addr.s6_addr32[0] = 0;
516be43a6b2SStanislav Fomichev sin6->sin6_addr.s6_addr32[1] = 0;
517be43a6b2SStanislav Fomichev sin6->sin6_addr.s6_addr16[4] = 0;
518be43a6b2SStanislav Fomichev sin6->sin6_addr.s6_addr16[5] = 0xffff;
519be43a6b2SStanislav Fomichev }
520be43a6b2SStanislav Fomichev
521be43a6b2SStanislav Fomichev return 0;
522be43a6b2SStanislav Fomichev }
523be43a6b2SStanislav Fomichev
create_queues(void)52497c4e094SCosmin Ratiu static struct netdev_queue_id *create_queues(void)
52597c4e094SCosmin Ratiu {
52697c4e094SCosmin Ratiu struct netdev_queue_id *queues;
52797c4e094SCosmin Ratiu size_t i = 0;
52897c4e094SCosmin Ratiu
529*f70d9819SJakub Kicinski queues = netdev_queue_id_alloc(num_queues);
53097c4e094SCosmin Ratiu for (i = 0; i < num_queues; i++) {
531*f70d9819SJakub Kicinski netdev_queue_id_set_type(&queues[i], NETDEV_QUEUE_TYPE_RX);
532*f70d9819SJakub Kicinski netdev_queue_id_set_id(&queues[i], start_queue + i);
53397c4e094SCosmin Ratiu }
53497c4e094SCosmin Ratiu
53597c4e094SCosmin Ratiu return queues;
53697c4e094SCosmin Ratiu }
53797c4e094SCosmin Ratiu
do_server(struct memory_buffer * mem)5382f1a805fSMina Almasry static int do_server(struct memory_buffer *mem)
539be43a6b2SStanislav Fomichev {
540be43a6b2SStanislav Fomichev char ctrl_data[sizeof(int) * 20000];
541be43a6b2SStanislav Fomichev size_t non_page_aligned_frags = 0;
542be43a6b2SStanislav Fomichev struct sockaddr_in6 client_addr;
543be43a6b2SStanislav Fomichev struct sockaddr_in6 server_sin;
544be43a6b2SStanislav Fomichev size_t page_aligned_frags = 0;
545be43a6b2SStanislav Fomichev size_t total_received = 0;
546be43a6b2SStanislav Fomichev socklen_t client_addr_len;
547be43a6b2SStanislav Fomichev bool is_devmem = false;
548be43a6b2SStanislav Fomichev char *tmp_mem = NULL;
549be43a6b2SStanislav Fomichev struct ynl_sock *ys;
550be43a6b2SStanislav Fomichev char iobuf[819200];
551be43a6b2SStanislav Fomichev char buffer[256];
552be43a6b2SStanislav Fomichev int socket_fd;
553be43a6b2SStanislav Fomichev int client_fd;
554be43a6b2SStanislav Fomichev int ret;
555be43a6b2SStanislav Fomichev
556be43a6b2SStanislav Fomichev ret = parse_address(server_ip, atoi(port), &server_sin);
557be43a6b2SStanislav Fomichev if (ret < 0)
558be43a6b2SStanislav Fomichev error(1, 0, "parse server address");
559be43a6b2SStanislav Fomichev
560be43a6b2SStanislav Fomichev if (reset_flow_steering())
561be43a6b2SStanislav Fomichev error(1, 0, "Failed to reset flow steering\n");
562be43a6b2SStanislav Fomichev
563be43a6b2SStanislav Fomichev if (configure_headersplit(1))
564be43a6b2SStanislav Fomichev error(1, 0, "Failed to enable TCP header split\n");
565be43a6b2SStanislav Fomichev
566be43a6b2SStanislav Fomichev /* Configure RSS to divert all traffic from our devmem queues */
567be43a6b2SStanislav Fomichev if (configure_rss())
568be43a6b2SStanislav Fomichev error(1, 0, "Failed to configure rss\n");
569be43a6b2SStanislav Fomichev
570be43a6b2SStanislav Fomichev /* Flow steer our devmem flows to start_queue */
571be43a6b2SStanislav Fomichev if (configure_flow_steering(&server_sin))
572be43a6b2SStanislav Fomichev error(1, 0, "Failed to configure flow steering\n");
573be43a6b2SStanislav Fomichev
574be43a6b2SStanislav Fomichev sleep(1);
575be43a6b2SStanislav Fomichev
57697c4e094SCosmin Ratiu if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys))
577be43a6b2SStanislav Fomichev error(1, 0, "Failed to bind\n");
578be43a6b2SStanislav Fomichev
579be43a6b2SStanislav Fomichev tmp_mem = malloc(mem->size);
580be43a6b2SStanislav Fomichev if (!tmp_mem)
581be43a6b2SStanislav Fomichev error(1, ENOMEM, "malloc failed");
582be43a6b2SStanislav Fomichev
583be43a6b2SStanislav Fomichev socket_fd = socket(AF_INET6, SOCK_STREAM, 0);
584be43a6b2SStanislav Fomichev if (socket_fd < 0)
585be43a6b2SStanislav Fomichev error(1, errno, "%s: [FAIL, create socket]\n", TEST_PREFIX);
586be43a6b2SStanislav Fomichev
587be43a6b2SStanislav Fomichev enable_reuseaddr(socket_fd);
588be43a6b2SStanislav Fomichev
589be43a6b2SStanislav Fomichev fprintf(stderr, "binding to address %s:%d\n", server_ip,
590be43a6b2SStanislav Fomichev ntohs(server_sin.sin6_port));
591be43a6b2SStanislav Fomichev
592be43a6b2SStanislav Fomichev ret = bind(socket_fd, &server_sin, sizeof(server_sin));
593be43a6b2SStanislav Fomichev if (ret)
594be43a6b2SStanislav Fomichev error(1, errno, "%s: [FAIL, bind]\n", TEST_PREFIX);
595be43a6b2SStanislav Fomichev
596be43a6b2SStanislav Fomichev ret = listen(socket_fd, 1);
597be43a6b2SStanislav Fomichev if (ret)
598be43a6b2SStanislav Fomichev error(1, errno, "%s: [FAIL, listen]\n", TEST_PREFIX);
599be43a6b2SStanislav Fomichev
600be43a6b2SStanislav Fomichev client_addr_len = sizeof(client_addr);
601be43a6b2SStanislav Fomichev
602be43a6b2SStanislav Fomichev inet_ntop(AF_INET6, &server_sin.sin6_addr, buffer,
603be43a6b2SStanislav Fomichev sizeof(buffer));
604be43a6b2SStanislav Fomichev fprintf(stderr, "Waiting or connection on %s:%d\n", buffer,
605be43a6b2SStanislav Fomichev ntohs(server_sin.sin6_port));
606be43a6b2SStanislav Fomichev client_fd = accept(socket_fd, &client_addr, &client_addr_len);
607be43a6b2SStanislav Fomichev
608be43a6b2SStanislav Fomichev inet_ntop(AF_INET6, &client_addr.sin6_addr, buffer,
609be43a6b2SStanislav Fomichev sizeof(buffer));
610be43a6b2SStanislav Fomichev fprintf(stderr, "Got connection from %s:%d\n", buffer,
611be43a6b2SStanislav Fomichev ntohs(client_addr.sin6_port));
612be43a6b2SStanislav Fomichev
613be43a6b2SStanislav Fomichev while (1) {
614be43a6b2SStanislav Fomichev struct iovec iov = { .iov_base = iobuf,
615be43a6b2SStanislav Fomichev .iov_len = sizeof(iobuf) };
616be43a6b2SStanislav Fomichev struct dmabuf_cmsg *dmabuf_cmsg = NULL;
617be43a6b2SStanislav Fomichev struct cmsghdr *cm = NULL;
618be43a6b2SStanislav Fomichev struct msghdr msg = { 0 };
619be43a6b2SStanislav Fomichev struct dmabuf_token token;
620be43a6b2SStanislav Fomichev ssize_t ret;
621be43a6b2SStanislav Fomichev
622be43a6b2SStanislav Fomichev is_devmem = false;
623be43a6b2SStanislav Fomichev
624be43a6b2SStanislav Fomichev msg.msg_iov = &iov;
625be43a6b2SStanislav Fomichev msg.msg_iovlen = 1;
626be43a6b2SStanislav Fomichev msg.msg_control = ctrl_data;
627be43a6b2SStanislav Fomichev msg.msg_controllen = sizeof(ctrl_data);
628be43a6b2SStanislav Fomichev ret = recvmsg(client_fd, &msg, MSG_SOCK_DEVMEM);
629be43a6b2SStanislav Fomichev fprintf(stderr, "recvmsg ret=%ld\n", ret);
630be43a6b2SStanislav Fomichev if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
631be43a6b2SStanislav Fomichev continue;
632be43a6b2SStanislav Fomichev if (ret < 0) {
633be43a6b2SStanislav Fomichev perror("recvmsg");
634be43a6b2SStanislav Fomichev continue;
635be43a6b2SStanislav Fomichev }
636be43a6b2SStanislav Fomichev if (ret == 0) {
637be43a6b2SStanislav Fomichev fprintf(stderr, "client exited\n");
638be43a6b2SStanislav Fomichev goto cleanup;
639be43a6b2SStanislav Fomichev }
640be43a6b2SStanislav Fomichev
641be43a6b2SStanislav Fomichev for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {
642be43a6b2SStanislav Fomichev if (cm->cmsg_level != SOL_SOCKET ||
643be43a6b2SStanislav Fomichev (cm->cmsg_type != SCM_DEVMEM_DMABUF &&
644be43a6b2SStanislav Fomichev cm->cmsg_type != SCM_DEVMEM_LINEAR)) {
645be43a6b2SStanislav Fomichev fprintf(stderr, "skipping non-devmem cmsg\n");
646be43a6b2SStanislav Fomichev continue;
647be43a6b2SStanislav Fomichev }
648be43a6b2SStanislav Fomichev
649be43a6b2SStanislav Fomichev dmabuf_cmsg = (struct dmabuf_cmsg *)CMSG_DATA(cm);
650be43a6b2SStanislav Fomichev is_devmem = true;
651be43a6b2SStanislav Fomichev
652be43a6b2SStanislav Fomichev if (cm->cmsg_type == SCM_DEVMEM_LINEAR) {
653be43a6b2SStanislav Fomichev /* TODO: process data copied from skb's linear
654be43a6b2SStanislav Fomichev * buffer.
655be43a6b2SStanislav Fomichev */
656be43a6b2SStanislav Fomichev fprintf(stderr,
657be43a6b2SStanislav Fomichev "SCM_DEVMEM_LINEAR. dmabuf_cmsg->frag_size=%u\n",
658be43a6b2SStanislav Fomichev dmabuf_cmsg->frag_size);
659be43a6b2SStanislav Fomichev
660be43a6b2SStanislav Fomichev continue;
661be43a6b2SStanislav Fomichev }
662be43a6b2SStanislav Fomichev
663be43a6b2SStanislav Fomichev token.token_start = dmabuf_cmsg->frag_token;
664be43a6b2SStanislav Fomichev token.token_count = 1;
665be43a6b2SStanislav Fomichev
666be43a6b2SStanislav Fomichev total_received += dmabuf_cmsg->frag_size;
667be43a6b2SStanislav Fomichev fprintf(stderr,
668be43a6b2SStanislav Fomichev "received frag_page=%llu, in_page_offset=%llu, frag_offset=%llu, frag_size=%u, token=%u, total_received=%lu, dmabuf_id=%u\n",
669be43a6b2SStanislav Fomichev dmabuf_cmsg->frag_offset >> PAGE_SHIFT,
670be43a6b2SStanislav Fomichev dmabuf_cmsg->frag_offset % getpagesize(),
671be43a6b2SStanislav Fomichev dmabuf_cmsg->frag_offset,
672be43a6b2SStanislav Fomichev dmabuf_cmsg->frag_size, dmabuf_cmsg->frag_token,
673be43a6b2SStanislav Fomichev total_received, dmabuf_cmsg->dmabuf_id);
674be43a6b2SStanislav Fomichev
675be43a6b2SStanislav Fomichev if (dmabuf_cmsg->dmabuf_id != dmabuf_id)
676be43a6b2SStanislav Fomichev error(1, 0,
677be43a6b2SStanislav Fomichev "received on wrong dmabuf_id: flow steering error\n");
678be43a6b2SStanislav Fomichev
679be43a6b2SStanislav Fomichev if (dmabuf_cmsg->frag_size % getpagesize())
680be43a6b2SStanislav Fomichev non_page_aligned_frags++;
681be43a6b2SStanislav Fomichev else
682be43a6b2SStanislav Fomichev page_aligned_frags++;
683be43a6b2SStanislav Fomichev
684be43a6b2SStanislav Fomichev provider->memcpy_from_device(tmp_mem, mem,
685be43a6b2SStanislav Fomichev dmabuf_cmsg->frag_offset,
686be43a6b2SStanislav Fomichev dmabuf_cmsg->frag_size);
687be43a6b2SStanislav Fomichev
688be43a6b2SStanislav Fomichev if (do_validation)
689be43a6b2SStanislav Fomichev validate_buffer(tmp_mem,
690be43a6b2SStanislav Fomichev dmabuf_cmsg->frag_size);
691be43a6b2SStanislav Fomichev else
692be43a6b2SStanislav Fomichev print_nonzero_bytes(tmp_mem,
693be43a6b2SStanislav Fomichev dmabuf_cmsg->frag_size);
694be43a6b2SStanislav Fomichev
695be43a6b2SStanislav Fomichev ret = setsockopt(client_fd, SOL_SOCKET,
696be43a6b2SStanislav Fomichev SO_DEVMEM_DONTNEED, &token,
697be43a6b2SStanislav Fomichev sizeof(token));
698be43a6b2SStanislav Fomichev if (ret != 1)
699be43a6b2SStanislav Fomichev error(1, 0,
700be43a6b2SStanislav Fomichev "SO_DEVMEM_DONTNEED not enough tokens");
701be43a6b2SStanislav Fomichev }
702be43a6b2SStanislav Fomichev if (!is_devmem)
703be43a6b2SStanislav Fomichev error(1, 0, "flow steering error\n");
704be43a6b2SStanislav Fomichev
705be43a6b2SStanislav Fomichev fprintf(stderr, "total_received=%lu\n", total_received);
706be43a6b2SStanislav Fomichev }
707be43a6b2SStanislav Fomichev
708be43a6b2SStanislav Fomichev fprintf(stderr, "%s: ok\n", TEST_PREFIX);
709be43a6b2SStanislav Fomichev
710be43a6b2SStanislav Fomichev fprintf(stderr, "page_aligned_frags=%lu, non_page_aligned_frags=%lu\n",
711be43a6b2SStanislav Fomichev page_aligned_frags, non_page_aligned_frags);
712be43a6b2SStanislav Fomichev
713be43a6b2SStanislav Fomichev cleanup:
714be43a6b2SStanislav Fomichev
715be43a6b2SStanislav Fomichev free(tmp_mem);
716be43a6b2SStanislav Fomichev close(client_fd);
717be43a6b2SStanislav Fomichev close(socket_fd);
718be43a6b2SStanislav Fomichev ynl_sock_destroy(ys);
719be43a6b2SStanislav Fomichev
720be43a6b2SStanislav Fomichev return 0;
721be43a6b2SStanislav Fomichev }
722be43a6b2SStanislav Fomichev
run_devmem_tests(void)723be43a6b2SStanislav Fomichev void run_devmem_tests(void)
724be43a6b2SStanislav Fomichev {
725be43a6b2SStanislav Fomichev struct memory_buffer *mem;
726be43a6b2SStanislav Fomichev struct ynl_sock *ys;
727be43a6b2SStanislav Fomichev
728be43a6b2SStanislav Fomichev mem = provider->alloc(getpagesize() * NUM_PAGES);
729be43a6b2SStanislav Fomichev
730be43a6b2SStanislav Fomichev /* Configure RSS to divert all traffic from our devmem queues */
731be43a6b2SStanislav Fomichev if (configure_rss())
732be43a6b2SStanislav Fomichev error(1, 0, "rss error\n");
733be43a6b2SStanislav Fomichev
734be43a6b2SStanislav Fomichev if (configure_headersplit(1))
735be43a6b2SStanislav Fomichev error(1, 0, "Failed to configure header split\n");
736be43a6b2SStanislav Fomichev
73797c4e094SCosmin Ratiu if (!bind_rx_queue(ifindex, mem->fd,
73897c4e094SCosmin Ratiu calloc(num_queues, sizeof(struct netdev_queue_id)),
73997c4e094SCosmin Ratiu num_queues, &ys))
740be43a6b2SStanislav Fomichev error(1, 0, "Binding empty queues array should have failed\n");
741be43a6b2SStanislav Fomichev
742be43a6b2SStanislav Fomichev if (configure_headersplit(0))
743be43a6b2SStanislav Fomichev error(1, 0, "Failed to configure header split\n");
744be43a6b2SStanislav Fomichev
74597c4e094SCosmin Ratiu if (!bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys))
746be43a6b2SStanislav Fomichev error(1, 0, "Configure dmabuf with header split off should have failed\n");
747be43a6b2SStanislav Fomichev
748be43a6b2SStanislav Fomichev if (configure_headersplit(1))
749be43a6b2SStanislav Fomichev error(1, 0, "Failed to configure header split\n");
750be43a6b2SStanislav Fomichev
75197c4e094SCosmin Ratiu if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys))
752be43a6b2SStanislav Fomichev error(1, 0, "Failed to bind\n");
753be43a6b2SStanislav Fomichev
754be43a6b2SStanislav Fomichev /* Deactivating a bound queue should not be legal */
755be43a6b2SStanislav Fomichev if (!configure_channels(num_queues, num_queues - 1))
756be43a6b2SStanislav Fomichev error(1, 0, "Deactivating a bound queue should be illegal.\n");
757be43a6b2SStanislav Fomichev
758be43a6b2SStanislav Fomichev /* Closing the netlink socket does an implicit unbind */
759be43a6b2SStanislav Fomichev ynl_sock_destroy(ys);
760be43a6b2SStanislav Fomichev
761be43a6b2SStanislav Fomichev provider->free(mem);
762be43a6b2SStanislav Fomichev }
763be43a6b2SStanislav Fomichev
gettimeofday_ms(void)7642f1a805fSMina Almasry static uint64_t gettimeofday_ms(void)
7652f1a805fSMina Almasry {
7662f1a805fSMina Almasry struct timeval tv;
7672f1a805fSMina Almasry
7682f1a805fSMina Almasry gettimeofday(&tv, NULL);
7692f1a805fSMina Almasry return (tv.tv_sec * 1000ULL) + (tv.tv_usec / 1000ULL);
7702f1a805fSMina Almasry }
7712f1a805fSMina Almasry
do_poll(int fd)7722f1a805fSMina Almasry static int do_poll(int fd)
7732f1a805fSMina Almasry {
7742f1a805fSMina Almasry struct pollfd pfd;
7752f1a805fSMina Almasry int ret;
7762f1a805fSMina Almasry
7772f1a805fSMina Almasry pfd.revents = 0;
7782f1a805fSMina Almasry pfd.fd = fd;
7792f1a805fSMina Almasry
7802f1a805fSMina Almasry ret = poll(&pfd, 1, waittime_ms);
7812f1a805fSMina Almasry if (ret == -1)
7822f1a805fSMina Almasry error(1, errno, "poll");
7832f1a805fSMina Almasry
7842f1a805fSMina Almasry return ret && (pfd.revents & POLLERR);
7852f1a805fSMina Almasry }
7862f1a805fSMina Almasry
wait_compl(int fd)7872f1a805fSMina Almasry static void wait_compl(int fd)
7882f1a805fSMina Almasry {
7892f1a805fSMina Almasry int64_t tstop = gettimeofday_ms() + waittime_ms;
7902f1a805fSMina Almasry char control[CMSG_SPACE(100)] = {};
7912f1a805fSMina Almasry struct sock_extended_err *serr;
7922f1a805fSMina Almasry struct msghdr msg = {};
7932f1a805fSMina Almasry struct cmsghdr *cm;
7942f1a805fSMina Almasry __u32 hi, lo;
7952f1a805fSMina Almasry int ret;
7962f1a805fSMina Almasry
7972f1a805fSMina Almasry msg.msg_control = control;
7982f1a805fSMina Almasry msg.msg_controllen = sizeof(control);
7992f1a805fSMina Almasry
8002f1a805fSMina Almasry while (gettimeofday_ms() < tstop) {
8012f1a805fSMina Almasry if (!do_poll(fd))
8022f1a805fSMina Almasry continue;
8032f1a805fSMina Almasry
8042f1a805fSMina Almasry ret = recvmsg(fd, &msg, MSG_ERRQUEUE);
8052f1a805fSMina Almasry if (ret < 0) {
8062f1a805fSMina Almasry if (errno == EAGAIN)
8072f1a805fSMina Almasry continue;
8082f1a805fSMina Almasry error(1, errno, "recvmsg(MSG_ERRQUEUE)");
8092f1a805fSMina Almasry return;
8102f1a805fSMina Almasry }
8112f1a805fSMina Almasry if (msg.msg_flags & MSG_CTRUNC)
8122f1a805fSMina Almasry error(1, 0, "MSG_CTRUNC\n");
8132f1a805fSMina Almasry
8142f1a805fSMina Almasry for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {
8152f1a805fSMina Almasry if (cm->cmsg_level != SOL_IP &&
8162f1a805fSMina Almasry cm->cmsg_level != SOL_IPV6)
8172f1a805fSMina Almasry continue;
8182f1a805fSMina Almasry if (cm->cmsg_level == SOL_IP &&
8192f1a805fSMina Almasry cm->cmsg_type != IP_RECVERR)
8202f1a805fSMina Almasry continue;
8212f1a805fSMina Almasry if (cm->cmsg_level == SOL_IPV6 &&
8222f1a805fSMina Almasry cm->cmsg_type != IPV6_RECVERR)
8232f1a805fSMina Almasry continue;
8242f1a805fSMina Almasry
8252f1a805fSMina Almasry serr = (void *)CMSG_DATA(cm);
8262f1a805fSMina Almasry if (serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY)
8272f1a805fSMina Almasry error(1, 0, "wrong origin %u", serr->ee_origin);
8282f1a805fSMina Almasry if (serr->ee_errno != 0)
8292f1a805fSMina Almasry error(1, 0, "wrong errno %d", serr->ee_errno);
8302f1a805fSMina Almasry
8312f1a805fSMina Almasry hi = serr->ee_data;
8322f1a805fSMina Almasry lo = serr->ee_info;
8332f1a805fSMina Almasry
8342f1a805fSMina Almasry fprintf(stderr, "tx complete [%d,%d]\n", lo, hi);
8352f1a805fSMina Almasry return;
8362f1a805fSMina Almasry }
8372f1a805fSMina Almasry }
8382f1a805fSMina Almasry
8392f1a805fSMina Almasry error(1, 0, "did not receive tx completion");
8402f1a805fSMina Almasry }
8412f1a805fSMina Almasry
do_client(struct memory_buffer * mem)8422f1a805fSMina Almasry static int do_client(struct memory_buffer *mem)
8432f1a805fSMina Almasry {
8442f1a805fSMina Almasry char ctrl_data[CMSG_SPACE(sizeof(__u32))];
8452f1a805fSMina Almasry struct sockaddr_in6 server_sin;
8462f1a805fSMina Almasry struct sockaddr_in6 client_sin;
8472f1a805fSMina Almasry struct ynl_sock *ys = NULL;
84861f24c68SStanislav Fomichev struct iovec iov[MAX_IOV];
8492f1a805fSMina Almasry struct msghdr msg = {};
8502f1a805fSMina Almasry ssize_t line_size = 0;
8512f1a805fSMina Almasry struct cmsghdr *cmsg;
8522f1a805fSMina Almasry char *line = NULL;
8532f1a805fSMina Almasry size_t len = 0;
8542f1a805fSMina Almasry int socket_fd;
8552f1a805fSMina Almasry __u32 ddmabuf;
8562f1a805fSMina Almasry int opt = 1;
8572f1a805fSMina Almasry int ret;
8582f1a805fSMina Almasry
8592f1a805fSMina Almasry ret = parse_address(server_ip, atoi(port), &server_sin);
8602f1a805fSMina Almasry if (ret < 0)
8612f1a805fSMina Almasry error(1, 0, "parse server address");
8622f1a805fSMina Almasry
8632f1a805fSMina Almasry socket_fd = socket(AF_INET6, SOCK_STREAM, 0);
8642f1a805fSMina Almasry if (socket_fd < 0)
8652f1a805fSMina Almasry error(1, socket_fd, "create socket");
8662f1a805fSMina Almasry
8672f1a805fSMina Almasry enable_reuseaddr(socket_fd);
8682f1a805fSMina Almasry
8692f1a805fSMina Almasry ret = setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, ifname,
8702f1a805fSMina Almasry strlen(ifname) + 1);
8712f1a805fSMina Almasry if (ret)
8722f1a805fSMina Almasry error(1, errno, "bindtodevice");
8732f1a805fSMina Almasry
8742f1a805fSMina Almasry if (bind_tx_queue(ifindex, mem->fd, &ys))
8752f1a805fSMina Almasry error(1, 0, "Failed to bind\n");
8762f1a805fSMina Almasry
8772f1a805fSMina Almasry if (client_ip) {
8782f1a805fSMina Almasry ret = parse_address(client_ip, atoi(port), &client_sin);
8792f1a805fSMina Almasry if (ret < 0)
8802f1a805fSMina Almasry error(1, 0, "parse client address");
8812f1a805fSMina Almasry
8822f1a805fSMina Almasry ret = bind(socket_fd, &client_sin, sizeof(client_sin));
8832f1a805fSMina Almasry if (ret)
8842f1a805fSMina Almasry error(1, errno, "bind");
8852f1a805fSMina Almasry }
8862f1a805fSMina Almasry
8872f1a805fSMina Almasry ret = setsockopt(socket_fd, SOL_SOCKET, SO_ZEROCOPY, &opt, sizeof(opt));
8882f1a805fSMina Almasry if (ret)
8892f1a805fSMina Almasry error(1, errno, "set sock opt");
8902f1a805fSMina Almasry
8912f1a805fSMina Almasry fprintf(stderr, "Connect to %s %d (via %s)\n", server_ip,
8922f1a805fSMina Almasry ntohs(server_sin.sin6_port), ifname);
8932f1a805fSMina Almasry
8942f1a805fSMina Almasry ret = connect(socket_fd, &server_sin, sizeof(server_sin));
8952f1a805fSMina Almasry if (ret)
8962f1a805fSMina Almasry error(1, errno, "connect");
8972f1a805fSMina Almasry
8982f1a805fSMina Almasry while (1) {
8992f1a805fSMina Almasry free(line);
9002f1a805fSMina Almasry line = NULL;
9012f1a805fSMina Almasry line_size = getline(&line, &len, stdin);
9022f1a805fSMina Almasry
9032f1a805fSMina Almasry if (line_size < 0)
9042f1a805fSMina Almasry break;
9052f1a805fSMina Almasry
90661f24c68SStanislav Fomichev if (max_chunk) {
90761f24c68SStanislav Fomichev msg.msg_iovlen =
90861f24c68SStanislav Fomichev (line_size + max_chunk - 1) / max_chunk;
90961f24c68SStanislav Fomichev if (msg.msg_iovlen > MAX_IOV)
91061f24c68SStanislav Fomichev error(1, 0,
91161f24c68SStanislav Fomichev "can't partition %zd bytes into maximum of %d chunks",
91261f24c68SStanislav Fomichev line_size, MAX_IOV);
9132f1a805fSMina Almasry
91461f24c68SStanislav Fomichev for (int i = 0; i < msg.msg_iovlen; i++) {
91561f24c68SStanislav Fomichev iov[i].iov_base = (void *)(i * max_chunk);
91661f24c68SStanislav Fomichev iov[i].iov_len = max_chunk;
91761f24c68SStanislav Fomichev }
9182f1a805fSMina Almasry
91961f24c68SStanislav Fomichev iov[msg.msg_iovlen - 1].iov_len =
92061f24c68SStanislav Fomichev line_size - (msg.msg_iovlen - 1) * max_chunk;
92161f24c68SStanislav Fomichev } else {
92261f24c68SStanislav Fomichev iov[0].iov_base = 0;
92361f24c68SStanislav Fomichev iov[0].iov_len = line_size;
92461f24c68SStanislav Fomichev msg.msg_iovlen = 1;
92561f24c68SStanislav Fomichev }
9262f1a805fSMina Almasry
9272f1a805fSMina Almasry msg.msg_iov = iov;
92861f24c68SStanislav Fomichev provider->memcpy_to_device(mem, 0, line, line_size);
9292f1a805fSMina Almasry
9302f1a805fSMina Almasry msg.msg_control = ctrl_data;
9312f1a805fSMina Almasry msg.msg_controllen = sizeof(ctrl_data);
9322f1a805fSMina Almasry
9332f1a805fSMina Almasry cmsg = CMSG_FIRSTHDR(&msg);
9342f1a805fSMina Almasry cmsg->cmsg_level = SOL_SOCKET;
9352f1a805fSMina Almasry cmsg->cmsg_type = SCM_DEVMEM_DMABUF;
9362f1a805fSMina Almasry cmsg->cmsg_len = CMSG_LEN(sizeof(__u32));
9372f1a805fSMina Almasry
9382f1a805fSMina Almasry ddmabuf = tx_dmabuf_id;
9392f1a805fSMina Almasry
9402f1a805fSMina Almasry *((__u32 *)CMSG_DATA(cmsg)) = ddmabuf;
9412f1a805fSMina Almasry
9422f1a805fSMina Almasry ret = sendmsg(socket_fd, &msg, MSG_ZEROCOPY);
9432f1a805fSMina Almasry if (ret < 0)
9442f1a805fSMina Almasry error(1, errno, "Failed sendmsg");
9452f1a805fSMina Almasry
9462f1a805fSMina Almasry fprintf(stderr, "sendmsg_ret=%d\n", ret);
9472f1a805fSMina Almasry
9482f1a805fSMina Almasry if (ret != line_size)
94961f24c68SStanislav Fomichev error(1, errno, "Did not send all bytes %d vs %zd", ret,
95061f24c68SStanislav Fomichev line_size);
9512f1a805fSMina Almasry
9522f1a805fSMina Almasry wait_compl(socket_fd);
9532f1a805fSMina Almasry }
9542f1a805fSMina Almasry
9552f1a805fSMina Almasry fprintf(stderr, "%s: tx ok\n", TEST_PREFIX);
9562f1a805fSMina Almasry
9572f1a805fSMina Almasry free(line);
9582f1a805fSMina Almasry close(socket_fd);
9592f1a805fSMina Almasry
9602f1a805fSMina Almasry if (ys)
9612f1a805fSMina Almasry ynl_sock_destroy(ys);
9622f1a805fSMina Almasry
9632f1a805fSMina Almasry return 0;
9642f1a805fSMina Almasry }
9652f1a805fSMina Almasry
main(int argc,char * argv[])966be43a6b2SStanislav Fomichev int main(int argc, char *argv[])
967be43a6b2SStanislav Fomichev {
968be43a6b2SStanislav Fomichev struct memory_buffer *mem;
969be43a6b2SStanislav Fomichev int is_server = 0, opt;
970be43a6b2SStanislav Fomichev int ret;
971be43a6b2SStanislav Fomichev
97261f24c68SStanislav Fomichev while ((opt = getopt(argc, argv, "ls:c:p:v:q:t:f:z:")) != -1) {
973be43a6b2SStanislav Fomichev switch (opt) {
974be43a6b2SStanislav Fomichev case 'l':
975be43a6b2SStanislav Fomichev is_server = 1;
976be43a6b2SStanislav Fomichev break;
977be43a6b2SStanislav Fomichev case 's':
978be43a6b2SStanislav Fomichev server_ip = optarg;
979be43a6b2SStanislav Fomichev break;
980be43a6b2SStanislav Fomichev case 'c':
981be43a6b2SStanislav Fomichev client_ip = optarg;
982be43a6b2SStanislav Fomichev break;
983be43a6b2SStanislav Fomichev case 'p':
984be43a6b2SStanislav Fomichev port = optarg;
985be43a6b2SStanislav Fomichev break;
986be43a6b2SStanislav Fomichev case 'v':
987be43a6b2SStanislav Fomichev do_validation = atoll(optarg);
988be43a6b2SStanislav Fomichev break;
989be43a6b2SStanislav Fomichev case 'q':
990be43a6b2SStanislav Fomichev num_queues = atoi(optarg);
991be43a6b2SStanislav Fomichev break;
992be43a6b2SStanislav Fomichev case 't':
993be43a6b2SStanislav Fomichev start_queue = atoi(optarg);
994be43a6b2SStanislav Fomichev break;
995be43a6b2SStanislav Fomichev case 'f':
996be43a6b2SStanislav Fomichev ifname = optarg;
997be43a6b2SStanislav Fomichev break;
99861f24c68SStanislav Fomichev case 'z':
99961f24c68SStanislav Fomichev max_chunk = atoi(optarg);
100061f24c68SStanislav Fomichev break;
1001be43a6b2SStanislav Fomichev case '?':
1002be43a6b2SStanislav Fomichev fprintf(stderr, "unknown option: %c\n", optopt);
1003be43a6b2SStanislav Fomichev break;
1004be43a6b2SStanislav Fomichev }
1005be43a6b2SStanislav Fomichev }
1006be43a6b2SStanislav Fomichev
1007be43a6b2SStanislav Fomichev if (!ifname)
1008be43a6b2SStanislav Fomichev error(1, 0, "Missing -f argument\n");
1009be43a6b2SStanislav Fomichev
1010be43a6b2SStanislav Fomichev ifindex = if_nametoindex(ifname);
1011be43a6b2SStanislav Fomichev
10122f1a805fSMina Almasry fprintf(stderr, "using ifindex=%u\n", ifindex);
10132f1a805fSMina Almasry
1014be43a6b2SStanislav Fomichev if (!server_ip && !client_ip) {
1015be43a6b2SStanislav Fomichev if (start_queue < 0 && num_queues < 0) {
1016be43a6b2SStanislav Fomichev num_queues = rxq_num(ifindex);
1017be43a6b2SStanislav Fomichev if (num_queues < 0)
1018be43a6b2SStanislav Fomichev error(1, 0, "couldn't detect number of queues\n");
1019be43a6b2SStanislav Fomichev if (num_queues < 2)
1020be43a6b2SStanislav Fomichev error(1, 0,
1021be43a6b2SStanislav Fomichev "number of device queues is too low\n");
1022be43a6b2SStanislav Fomichev /* make sure can bind to multiple queues */
1023be43a6b2SStanislav Fomichev start_queue = num_queues / 2;
1024be43a6b2SStanislav Fomichev num_queues /= 2;
1025be43a6b2SStanislav Fomichev }
1026be43a6b2SStanislav Fomichev
1027be43a6b2SStanislav Fomichev if (start_queue < 0 || num_queues < 0)
1028be43a6b2SStanislav Fomichev error(1, 0, "Both -t and -q are required\n");
1029be43a6b2SStanislav Fomichev
1030be43a6b2SStanislav Fomichev run_devmem_tests();
1031be43a6b2SStanislav Fomichev return 0;
1032be43a6b2SStanislav Fomichev }
1033be43a6b2SStanislav Fomichev
1034be43a6b2SStanislav Fomichev if (start_queue < 0 && num_queues < 0) {
1035be43a6b2SStanislav Fomichev num_queues = rxq_num(ifindex);
1036be43a6b2SStanislav Fomichev if (num_queues < 2)
1037be43a6b2SStanislav Fomichev error(1, 0, "number of device queues is too low\n");
1038be43a6b2SStanislav Fomichev
1039be43a6b2SStanislav Fomichev num_queues = 1;
1040be43a6b2SStanislav Fomichev start_queue = rxq_num(ifindex) - num_queues;
1041be43a6b2SStanislav Fomichev
1042be43a6b2SStanislav Fomichev if (start_queue < 0)
1043be43a6b2SStanislav Fomichev error(1, 0, "couldn't detect number of queues\n");
1044be43a6b2SStanislav Fomichev
1045be43a6b2SStanislav Fomichev fprintf(stderr, "using queues %d..%d\n", start_queue, start_queue + num_queues);
1046be43a6b2SStanislav Fomichev }
1047be43a6b2SStanislav Fomichev
1048be43a6b2SStanislav Fomichev for (; optind < argc; optind++)
1049be43a6b2SStanislav Fomichev fprintf(stderr, "extra arguments: %s\n", argv[optind]);
1050be43a6b2SStanislav Fomichev
1051be43a6b2SStanislav Fomichev if (start_queue < 0)
1052be43a6b2SStanislav Fomichev error(1, 0, "Missing -t argument\n");
1053be43a6b2SStanislav Fomichev
1054be43a6b2SStanislav Fomichev if (num_queues < 0)
1055be43a6b2SStanislav Fomichev error(1, 0, "Missing -q argument\n");
1056be43a6b2SStanislav Fomichev
1057be43a6b2SStanislav Fomichev if (!server_ip)
1058be43a6b2SStanislav Fomichev error(1, 0, "Missing -s argument\n");
1059be43a6b2SStanislav Fomichev
1060be43a6b2SStanislav Fomichev if (!port)
1061be43a6b2SStanislav Fomichev error(1, 0, "Missing -p argument\n");
1062be43a6b2SStanislav Fomichev
1063be43a6b2SStanislav Fomichev mem = provider->alloc(getpagesize() * NUM_PAGES);
10642f1a805fSMina Almasry ret = is_server ? do_server(mem) : do_client(mem);
1065be43a6b2SStanislav Fomichev provider->free(mem);
1066be43a6b2SStanislav Fomichev
1067be43a6b2SStanislav Fomichev return ret;
1068be43a6b2SStanislav Fomichev }
1069