1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * tcpdevmem netcat. Works similarly to netcat but does device memory TCP
4 * instead of regular TCP. Uses udmabuf to mock a dmabuf provider.
5 *
6 * Usage:
7 *
8 * On server:
9 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201
10 *
11 * On client:
12 * echo -n "hello\nworld" | \
13 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1
14 *
15 * Note this is compatible with regular netcat. i.e. the sender or receiver can
16 * be replaced with regular netcat to test the RX or TX path in isolation.
17 *
18 * Test data validation (devmem TCP on RX only):
19 *
20 * On server:
21 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201 -v 7
22 *
23 * On client:
24 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06) | \
25 * head -c 1G | \
26 * nc <server IP> 5201 -p 5201
27 *
28 * Test data validation (devmem TCP on RX and TX, validation happens on RX):
29 *
30 * On server:
31 * ncdevmem -s <server IP> [-c <client IP>] -l -p 5201 -v 8 -f eth1
32 *
33 * On client:
34 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06\\x07) | \
35 * head -c 1M | \
36 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1
37 */
38 #define _GNU_SOURCE
39 #define __EXPORTED_HEADERS__
40
41 #include <linux/uio.h>
42 #include <stdarg.h>
43 #include <stdio.h>
44 #include <stdlib.h>
45 #include <unistd.h>
46 #include <stdbool.h>
47 #include <string.h>
48 #include <errno.h>
49 #define __iovec_defined
50 #include <fcntl.h>
51 #include <limits.h>
52 #include <malloc.h>
53 #include <error.h>
54 #include <poll.h>
55
56 #include <arpa/inet.h>
57 #include <sys/socket.h>
58 #include <sys/mman.h>
59 #include <sys/ioctl.h>
60 #include <sys/syscall.h>
61 #include <sys/time.h>
62
63 #include <linux/memfd.h>
64 #include <linux/dma-buf.h>
65 #include <linux/errqueue.h>
66 #include <linux/udmabuf.h>
67 #include <linux/types.h>
68 #include <linux/netlink.h>
69 #include <linux/genetlink.h>
70 #include <linux/netdev.h>
71 #include <linux/ethtool_netlink.h>
72 #include <time.h>
73 #include <net/if.h>
74
75 #include "netdev-user.h"
76 #include "ethtool-user.h"
77 #include <ynl.h>
78
79 #define PAGE_SHIFT 12
80 #define TEST_PREFIX "ncdevmem"
81 #define NUM_PAGES 16000
82
83 #ifndef MSG_SOCK_DEVMEM
84 #define MSG_SOCK_DEVMEM 0x2000000
85 #endif
86
87 #define MAX_IOV 1024
88
89 static size_t max_chunk;
90 static char *server_ip;
91 static char *client_ip;
92 static char *port;
93 static size_t do_validation;
94 static int start_queue = -1;
95 static int num_queues = -1;
96 static char *ifname;
97 static unsigned int ifindex;
98 static unsigned int dmabuf_id;
99 static uint32_t tx_dmabuf_id;
100 static int waittime_ms = 500;
101 static bool fail_on_linear;
102
103 /* System state loaded by current_config_load() */
104 #define MAX_FLOWS 8
105 static int ntuple_ids[MAX_FLOWS] = { -1, -1, -1, -1, -1, -1, -1, -1, };
106
107 struct memory_buffer {
108 int fd;
109 size_t size;
110
111 int devfd;
112 int memfd;
113 char *buf_mem;
114 };
115
116 struct memory_provider {
117 struct memory_buffer *(*alloc)(size_t size);
118 void (*free)(struct memory_buffer *ctx);
119 void (*memcpy_to_device)(struct memory_buffer *dst, size_t off,
120 void *src, int n);
121 void (*memcpy_from_device)(void *dst, struct memory_buffer *src,
122 size_t off, int n);
123 };
124
pr_err(const char * fmt,...)125 static void pr_err(const char *fmt, ...)
126 {
127 va_list args;
128
129 fprintf(stderr, "%s: ", TEST_PREFIX);
130
131 va_start(args, fmt);
132 vfprintf(stderr, fmt, args);
133 va_end(args);
134
135 if (errno != 0)
136 fprintf(stderr, ": %s", strerror(errno));
137 fprintf(stderr, "\n");
138 }
139
udmabuf_alloc(size_t size)140 static struct memory_buffer *udmabuf_alloc(size_t size)
141 {
142 struct udmabuf_create create;
143 struct memory_buffer *ctx;
144 int ret;
145
146 ctx = malloc(sizeof(*ctx));
147 if (!ctx)
148 return NULL;
149
150 ctx->size = size;
151
152 ctx->devfd = open("/dev/udmabuf", O_RDWR);
153 if (ctx->devfd < 0) {
154 pr_err("[skip,no-udmabuf: Unable to access DMA buffer device file]");
155 goto err_free_ctx;
156 }
157
158 ctx->memfd = memfd_create("udmabuf-test", MFD_ALLOW_SEALING);
159 if (ctx->memfd < 0) {
160 pr_err("[skip,no-memfd]");
161 goto err_close_dev;
162 }
163
164 ret = fcntl(ctx->memfd, F_ADD_SEALS, F_SEAL_SHRINK);
165 if (ret < 0) {
166 pr_err("[skip,fcntl-add-seals]");
167 goto err_close_memfd;
168 }
169
170 ret = ftruncate(ctx->memfd, size);
171 if (ret == -1) {
172 pr_err("[FAIL,memfd-truncate]");
173 goto err_close_memfd;
174 }
175
176 memset(&create, 0, sizeof(create));
177
178 create.memfd = ctx->memfd;
179 create.offset = 0;
180 create.size = size;
181 ctx->fd = ioctl(ctx->devfd, UDMABUF_CREATE, &create);
182 if (ctx->fd < 0) {
183 pr_err("[FAIL, create udmabuf]");
184 goto err_close_fd;
185 }
186
187 ctx->buf_mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED,
188 ctx->fd, 0);
189 if (ctx->buf_mem == MAP_FAILED) {
190 pr_err("[FAIL, map udmabuf]");
191 goto err_close_fd;
192 }
193
194 return ctx;
195
196 err_close_fd:
197 close(ctx->fd);
198 err_close_memfd:
199 close(ctx->memfd);
200 err_close_dev:
201 close(ctx->devfd);
202 err_free_ctx:
203 free(ctx);
204 return NULL;
205 }
206
udmabuf_free(struct memory_buffer * ctx)207 static void udmabuf_free(struct memory_buffer *ctx)
208 {
209 munmap(ctx->buf_mem, ctx->size);
210 close(ctx->fd);
211 close(ctx->memfd);
212 close(ctx->devfd);
213 free(ctx);
214 }
215
udmabuf_memcpy_to_device(struct memory_buffer * dst,size_t off,void * src,int n)216 static void udmabuf_memcpy_to_device(struct memory_buffer *dst, size_t off,
217 void *src, int n)
218 {
219 struct dma_buf_sync sync = {};
220
221 sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE;
222 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync);
223
224 memcpy(dst->buf_mem + off, src, n);
225
226 sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE;
227 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync);
228 }
229
udmabuf_memcpy_from_device(void * dst,struct memory_buffer * src,size_t off,int n)230 static void udmabuf_memcpy_from_device(void *dst, struct memory_buffer *src,
231 size_t off, int n)
232 {
233 struct dma_buf_sync sync = {};
234
235 sync.flags = DMA_BUF_SYNC_START;
236 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync);
237
238 memcpy(dst, src->buf_mem + off, n);
239
240 sync.flags = DMA_BUF_SYNC_END;
241 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync);
242 }
243
244 static struct memory_provider udmabuf_memory_provider = {
245 .alloc = udmabuf_alloc,
246 .free = udmabuf_free,
247 .memcpy_to_device = udmabuf_memcpy_to_device,
248 .memcpy_from_device = udmabuf_memcpy_from_device,
249 };
250
251 static struct memory_provider *provider = &udmabuf_memory_provider;
252
print_nonzero_bytes(void * ptr,size_t size)253 static void print_nonzero_bytes(void *ptr, size_t size)
254 {
255 unsigned char *p = ptr;
256 unsigned int i;
257
258 for (i = 0; i < size; i++)
259 putchar(p[i]);
260 }
261
validate_buffer(void * line,size_t size)262 int validate_buffer(void *line, size_t size)
263 {
264 static unsigned char seed = 1;
265 unsigned char *ptr = line;
266 unsigned char expected;
267 static int errors;
268 size_t i;
269
270 for (i = 0; i < size; i++) {
271 expected = seed ? seed : '\n';
272 if (ptr[i] != expected) {
273 fprintf(stderr,
274 "Failed validation: expected=%u, actual=%u, index=%lu\n",
275 expected, ptr[i], i);
276 errors++;
277 if (errors > 20) {
278 pr_err("validation failed");
279 return -1;
280 }
281 }
282 seed++;
283 if (seed == do_validation)
284 seed = 0;
285 }
286
287 fprintf(stdout, "Validated buffer\n");
288 return 0;
289 }
290
291 static int
__run_command(char * out,size_t outlen,const char * cmd,va_list args)292 __run_command(char *out, size_t outlen, const char *cmd, va_list args)
293 {
294 char command[256];
295 FILE *fp;
296
297 vsnprintf(command, sizeof(command), cmd, args);
298
299 fprintf(stderr, "Running: %s\n", command);
300 fp = popen(command, "r");
301 if (!fp)
302 return -1;
303 if (out) {
304 size_t len;
305
306 if (!fgets(out, outlen, fp))
307 return -1;
308
309 /* Remove trailing newline if present */
310 len = strlen(out);
311 if (len && out[len - 1] == '\n')
312 out[len - 1] = '\0';
313 }
314 return pclose(fp);
315 }
316
run_command(const char * cmd,...)317 static int run_command(const char *cmd, ...)
318 {
319 va_list args;
320 int ret;
321
322 va_start(args, cmd);
323 ret = __run_command(NULL, 0, cmd, args);
324 va_end(args);
325
326 return ret;
327 }
328
ethtool_add_flow(const char * format,...)329 static int ethtool_add_flow(const char *format, ...)
330 {
331 char local_output[256], cmd[256];
332 const char *id_start;
333 int flow_idx, ret;
334 char *endptr;
335 long flow_id;
336 va_list args;
337
338 for (flow_idx = 0; flow_idx < MAX_FLOWS; flow_idx++)
339 if (ntuple_ids[flow_idx] == -1)
340 break;
341 if (flow_idx == MAX_FLOWS) {
342 fprintf(stderr, "Error: too many flows\n");
343 return -1;
344 }
345
346 snprintf(cmd, sizeof(cmd), "ethtool -N %s %s", ifname, format);
347
348 va_start(args, format);
349 ret = __run_command(local_output, sizeof(local_output), cmd, args);
350 va_end(args);
351
352 if (ret != 0)
353 return ret;
354
355 /* Extract the ID from the output */
356 id_start = strstr(local_output, "Added rule with ID ");
357 if (!id_start)
358 return -1;
359 id_start += strlen("Added rule with ID ");
360
361 flow_id = strtol(id_start, &endptr, 10);
362 if (endptr == id_start || flow_id < 0 || flow_id > INT_MAX)
363 return -1;
364
365 fprintf(stderr, "Added flow rule with ID %ld\n", flow_id);
366 ntuple_ids[flow_idx] = flow_id;
367 return flow_id;
368 }
369
rxq_num(int ifindex)370 static int rxq_num(int ifindex)
371 {
372 struct ethtool_channels_get_req *req;
373 struct ethtool_channels_get_rsp *rsp;
374 struct ynl_error yerr;
375 struct ynl_sock *ys;
376 int num = -1;
377
378 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
379 if (!ys) {
380 fprintf(stderr, "YNL: %s\n", yerr.msg);
381 return -1;
382 }
383
384 req = ethtool_channels_get_req_alloc();
385 ethtool_channels_get_req_set_header_dev_index(req, ifindex);
386 rsp = ethtool_channels_get(ys, req);
387 if (rsp)
388 num = rsp->rx_count + rsp->combined_count;
389 ethtool_channels_get_req_free(req);
390 ethtool_channels_get_rsp_free(rsp);
391
392 ynl_sock_destroy(ys);
393
394 return num;
395 }
396
reset_flow_steering(void)397 static void reset_flow_steering(void)
398 {
399 int i;
400
401 for (i = 0; i < MAX_FLOWS; i++) {
402 if (ntuple_ids[i] == -1)
403 continue;
404 run_command("ethtool -N %s delete %d",
405 ifname, ntuple_ids[i]);
406 ntuple_ids[i] = -1;
407 }
408 }
409
tcp_data_split_str(int val)410 static const char *tcp_data_split_str(int val)
411 {
412 switch (val) {
413 case 0:
414 return "off";
415 case 1:
416 return "auto";
417 case 2:
418 return "on";
419 default:
420 return "?";
421 }
422 }
423
get_ring_config(void)424 static struct ethtool_rings_get_rsp *get_ring_config(void)
425 {
426 struct ethtool_rings_get_req *get_req;
427 struct ethtool_rings_get_rsp *get_rsp;
428 struct ynl_error yerr;
429 struct ynl_sock *ys;
430
431 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
432 if (!ys) {
433 fprintf(stderr, "YNL: %s\n", yerr.msg);
434 return NULL;
435 }
436
437 get_req = ethtool_rings_get_req_alloc();
438 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex);
439 get_rsp = ethtool_rings_get(ys, get_req);
440 ethtool_rings_get_req_free(get_req);
441
442 ynl_sock_destroy(ys);
443
444 return get_rsp;
445 }
446
restore_ring_config(const struct ethtool_rings_get_rsp * config)447 static void restore_ring_config(const struct ethtool_rings_get_rsp *config)
448 {
449 struct ethtool_rings_get_req *get_req;
450 struct ethtool_rings_get_rsp *get_rsp;
451 struct ethtool_rings_set_req *req;
452 struct ynl_error yerr;
453 struct ynl_sock *ys;
454 int ret;
455
456 if (!config)
457 return;
458
459 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
460 if (!ys) {
461 fprintf(stderr, "YNL: %s\n", yerr.msg);
462 return;
463 }
464
465 req = ethtool_rings_set_req_alloc();
466 ethtool_rings_set_req_set_header_dev_index(req, ifindex);
467 ethtool_rings_set_req_set_tcp_data_split(req,
468 ETHTOOL_TCP_DATA_SPLIT_UNKNOWN);
469 if (config->_present.hds_thresh)
470 ethtool_rings_set_req_set_hds_thresh(req, config->hds_thresh);
471
472 ret = ethtool_rings_set(ys, req);
473 if (ret < 0)
474 fprintf(stderr, "YNL restoring HDS cfg: %s\n", ys->err.msg);
475
476 get_req = ethtool_rings_get_req_alloc();
477 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex);
478 get_rsp = ethtool_rings_get(ys, get_req);
479 ethtool_rings_get_req_free(get_req);
480
481 /* use explicit value if UKNOWN didn't give us the previous */
482 if (get_rsp->tcp_data_split != config->tcp_data_split) {
483 ethtool_rings_set_req_set_tcp_data_split(req,
484 config->tcp_data_split);
485 ret = ethtool_rings_set(ys, req);
486 if (ret < 0)
487 fprintf(stderr, "YNL restoring expl HDS cfg: %s\n",
488 ys->err.msg);
489 }
490
491 ethtool_rings_get_rsp_free(get_rsp);
492 ethtool_rings_set_req_free(req);
493
494 ynl_sock_destroy(ys);
495 }
496
497 static int
configure_headersplit(const struct ethtool_rings_get_rsp * old,bool on)498 configure_headersplit(const struct ethtool_rings_get_rsp *old, bool on)
499 {
500 struct ethtool_rings_get_req *get_req;
501 struct ethtool_rings_get_rsp *get_rsp;
502 struct ethtool_rings_set_req *req;
503 struct ynl_error yerr;
504 struct ynl_sock *ys;
505 int ret;
506
507 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
508 if (!ys) {
509 fprintf(stderr, "YNL: %s\n", yerr.msg);
510 return -1;
511 }
512
513 req = ethtool_rings_set_req_alloc();
514 ethtool_rings_set_req_set_header_dev_index(req, ifindex);
515 if (on) {
516 ethtool_rings_set_req_set_tcp_data_split(req,
517 ETHTOOL_TCP_DATA_SPLIT_ENABLED);
518 if (old->_present.hds_thresh)
519 ethtool_rings_set_req_set_hds_thresh(req, 0);
520 } else {
521 ethtool_rings_set_req_set_tcp_data_split(req,
522 ETHTOOL_TCP_DATA_SPLIT_UNKNOWN);
523 }
524 ret = ethtool_rings_set(ys, req);
525 if (ret < 0)
526 fprintf(stderr, "YNL failed: %s\n", ys->err.msg);
527 ethtool_rings_set_req_free(req);
528
529 if (ret == 0) {
530 get_req = ethtool_rings_get_req_alloc();
531 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex);
532 get_rsp = ethtool_rings_get(ys, get_req);
533 ethtool_rings_get_req_free(get_req);
534 if (get_rsp)
535 fprintf(stderr, "TCP header split: %s\n",
536 tcp_data_split_str(get_rsp->tcp_data_split));
537 ethtool_rings_get_rsp_free(get_rsp);
538 }
539
540 ynl_sock_destroy(ys);
541
542 return ret;
543 }
544
configure_rss(void)545 static int configure_rss(void)
546 {
547 return run_command("ethtool -X %s equal %d >&2", ifname, start_queue);
548 }
549
reset_rss(void)550 static void reset_rss(void)
551 {
552 run_command("ethtool -X %s default >&2", ifname, start_queue);
553 }
554
check_changing_channels(unsigned int rx,unsigned int tx)555 static int check_changing_channels(unsigned int rx, unsigned int tx)
556 {
557 struct ethtool_channels_get_req *gchan;
558 struct ethtool_channels_set_req *schan;
559 struct ethtool_channels_get_rsp *chan;
560 struct ynl_error yerr;
561 struct ynl_sock *ys;
562 int ret;
563
564 fprintf(stderr, "setting channel count rx:%u tx:%u\n", rx, tx);
565
566 ys = ynl_sock_create(&ynl_ethtool_family, &yerr);
567 if (!ys) {
568 fprintf(stderr, "YNL: %s\n", yerr.msg);
569 return -1;
570 }
571
572 gchan = ethtool_channels_get_req_alloc();
573 if (!gchan) {
574 ret = -1;
575 goto exit_close_sock;
576 }
577
578 ethtool_channels_get_req_set_header_dev_index(gchan, ifindex);
579 chan = ethtool_channels_get(ys, gchan);
580 ethtool_channels_get_req_free(gchan);
581 if (!chan) {
582 fprintf(stderr, "YNL get channels: %s\n", ys->err.msg);
583 ret = -1;
584 goto exit_close_sock;
585 }
586
587 schan = ethtool_channels_set_req_alloc();
588 if (!schan) {
589 ret = -1;
590 goto exit_free_chan;
591 }
592
593 ethtool_channels_set_req_set_header_dev_index(schan, ifindex);
594
595 if (chan->_present.combined_count) {
596 if (chan->_present.rx_count || chan->_present.tx_count) {
597 ethtool_channels_set_req_set_rx_count(schan, 0);
598 ethtool_channels_set_req_set_tx_count(schan, 0);
599 }
600
601 if (rx == tx) {
602 ethtool_channels_set_req_set_combined_count(schan, rx);
603 } else if (rx > tx) {
604 ethtool_channels_set_req_set_combined_count(schan, tx);
605 ethtool_channels_set_req_set_rx_count(schan, rx - tx);
606 } else {
607 ethtool_channels_set_req_set_combined_count(schan, rx);
608 ethtool_channels_set_req_set_tx_count(schan, tx - rx);
609 }
610
611 } else if (chan->_present.rx_count) {
612 ethtool_channels_set_req_set_rx_count(schan, rx);
613 ethtool_channels_set_req_set_tx_count(schan, tx);
614 } else {
615 fprintf(stderr, "Error: device has neither combined nor rx channels\n");
616 ret = -1;
617 goto exit_free_schan;
618 }
619
620 ret = ethtool_channels_set(ys, schan);
621 if (ret) {
622 fprintf(stderr, "YNL set channels: %s\n", ys->err.msg);
623 } else {
624 /* We were expecting a failure, go back to previous settings */
625 ethtool_channels_set_req_set_combined_count(schan,
626 chan->combined_count);
627 ethtool_channels_set_req_set_rx_count(schan, chan->rx_count);
628 ethtool_channels_set_req_set_tx_count(schan, chan->tx_count);
629
630 ret = ethtool_channels_set(ys, schan);
631 if (ret)
632 fprintf(stderr, "YNL un-setting channels: %s\n",
633 ys->err.msg);
634 }
635
636 exit_free_schan:
637 ethtool_channels_set_req_free(schan);
638 exit_free_chan:
639 ethtool_channels_get_rsp_free(chan);
640 exit_close_sock:
641 ynl_sock_destroy(ys);
642
643 return ret;
644 }
645
configure_flow_steering(struct sockaddr_in6 * server_sin)646 static int configure_flow_steering(struct sockaddr_in6 *server_sin)
647 {
648 const char *type = "tcp6";
649 const char *server_addr;
650 char buf[40];
651 int flow_id;
652
653 inet_ntop(AF_INET6, &server_sin->sin6_addr, buf, sizeof(buf));
654 server_addr = buf;
655
656 if (IN6_IS_ADDR_V4MAPPED(&server_sin->sin6_addr)) {
657 type = "tcp4";
658 server_addr = strrchr(server_addr, ':') + 1;
659 }
660
661 /* Try configure 5-tuple */
662 flow_id = ethtool_add_flow("flow-type %s %s %s dst-ip %s %s %s dst-port %s queue %d",
663 type,
664 client_ip ? "src-ip" : "",
665 client_ip ?: "",
666 server_addr,
667 client_ip ? "src-port" : "",
668 client_ip ? port : "",
669 port, start_queue);
670 if (flow_id < 0) {
671 /* If that fails, try configure 3-tuple */
672 flow_id = ethtool_add_flow("flow-type %s dst-ip %s dst-port %s queue %d",
673 type, server_addr, port, start_queue);
674 if (flow_id < 0)
675 /* If that fails, return error */
676 return -1;
677 }
678
679 return 0;
680 }
681
bind_rx_queue(unsigned int ifindex,unsigned int dmabuf_fd,struct netdev_queue_id * queues,unsigned int n_queue_index,struct ynl_sock ** ys)682 static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd,
683 struct netdev_queue_id *queues,
684 unsigned int n_queue_index, struct ynl_sock **ys)
685 {
686 struct netdev_bind_rx_req *req = NULL;
687 struct netdev_bind_rx_rsp *rsp = NULL;
688 struct ynl_error yerr;
689
690 *ys = ynl_sock_create(&ynl_netdev_family, &yerr);
691 if (!*ys) {
692 netdev_queue_id_free(queues);
693 fprintf(stderr, "YNL: %s\n", yerr.msg);
694 return -1;
695 }
696
697 req = netdev_bind_rx_req_alloc();
698 netdev_bind_rx_req_set_ifindex(req, ifindex);
699 netdev_bind_rx_req_set_fd(req, dmabuf_fd);
700 __netdev_bind_rx_req_set_queues(req, queues, n_queue_index);
701
702 rsp = netdev_bind_rx(*ys, req);
703 if (!rsp) {
704 perror("netdev_bind_rx");
705 goto err_close;
706 }
707
708 if (!rsp->_present.id) {
709 perror("id not present");
710 goto err_close;
711 }
712
713 fprintf(stderr, "got dmabuf id=%d\n", rsp->id);
714 dmabuf_id = rsp->id;
715
716 netdev_bind_rx_req_free(req);
717 netdev_bind_rx_rsp_free(rsp);
718
719 return 0;
720
721 err_close:
722 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg);
723 netdev_bind_rx_req_free(req);
724 ynl_sock_destroy(*ys);
725 return -1;
726 }
727
bind_tx_queue(unsigned int ifindex,unsigned int dmabuf_fd,struct ynl_sock ** ys)728 static int bind_tx_queue(unsigned int ifindex, unsigned int dmabuf_fd,
729 struct ynl_sock **ys)
730 {
731 struct netdev_bind_tx_req *req = NULL;
732 struct netdev_bind_tx_rsp *rsp = NULL;
733 struct ynl_error yerr;
734
735 *ys = ynl_sock_create(&ynl_netdev_family, &yerr);
736 if (!*ys) {
737 fprintf(stderr, "YNL: %s\n", yerr.msg);
738 return -1;
739 }
740
741 req = netdev_bind_tx_req_alloc();
742 netdev_bind_tx_req_set_ifindex(req, ifindex);
743 netdev_bind_tx_req_set_fd(req, dmabuf_fd);
744
745 rsp = netdev_bind_tx(*ys, req);
746 if (!rsp) {
747 perror("netdev_bind_tx");
748 goto err_close;
749 }
750
751 if (!rsp->_present.id) {
752 perror("id not present");
753 goto err_close;
754 }
755
756 fprintf(stderr, "got tx dmabuf id=%d\n", rsp->id);
757 tx_dmabuf_id = rsp->id;
758
759 netdev_bind_tx_req_free(req);
760 netdev_bind_tx_rsp_free(rsp);
761
762 return 0;
763
764 err_close:
765 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg);
766 netdev_bind_tx_req_free(req);
767 ynl_sock_destroy(*ys);
768 return -1;
769 }
770
enable_reuseaddr(int fd)771 static int enable_reuseaddr(int fd)
772 {
773 int opt = 1;
774 int ret;
775
776 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt));
777 if (ret) {
778 pr_err("SO_REUSEPORT failed");
779 return -1;
780 }
781
782 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
783 if (ret) {
784 pr_err("SO_REUSEADDR failed");
785 return -1;
786 }
787
788 return 0;
789 }
790
parse_address(const char * str,int port,struct sockaddr_in6 * sin6)791 static int parse_address(const char *str, int port, struct sockaddr_in6 *sin6)
792 {
793 int ret;
794
795 sin6->sin6_family = AF_INET6;
796 sin6->sin6_port = htons(port);
797
798 ret = inet_pton(sin6->sin6_family, str, &sin6->sin6_addr);
799 if (ret != 1) {
800 /* fallback to plain IPv4 */
801 ret = inet_pton(AF_INET, str, &sin6->sin6_addr.s6_addr32[3]);
802 if (ret != 1)
803 return -1;
804
805 /* add ::ffff prefix */
806 sin6->sin6_addr.s6_addr32[0] = 0;
807 sin6->sin6_addr.s6_addr32[1] = 0;
808 sin6->sin6_addr.s6_addr16[4] = 0;
809 sin6->sin6_addr.s6_addr16[5] = 0xffff;
810 }
811
812 return 0;
813 }
814
create_queues(void)815 static struct netdev_queue_id *create_queues(void)
816 {
817 struct netdev_queue_id *queues;
818 size_t i = 0;
819
820 queues = netdev_queue_id_alloc(num_queues);
821 for (i = 0; i < num_queues; i++) {
822 netdev_queue_id_set_type(&queues[i], NETDEV_QUEUE_TYPE_RX);
823 netdev_queue_id_set_id(&queues[i], start_queue + i);
824 }
825
826 return queues;
827 }
828
do_server(struct memory_buffer * mem)829 static int do_server(struct memory_buffer *mem)
830 {
831 struct ethtool_rings_get_rsp *ring_config;
832 char ctrl_data[sizeof(int) * 20000];
833 size_t non_page_aligned_frags = 0;
834 struct sockaddr_in6 client_addr;
835 struct sockaddr_in6 server_sin;
836 size_t page_aligned_frags = 0;
837 size_t total_received = 0;
838 socklen_t client_addr_len;
839 bool is_devmem = false;
840 char *tmp_mem = NULL;
841 struct ynl_sock *ys;
842 char iobuf[819200];
843 int ret, err = -1;
844 char buffer[256];
845 int socket_fd;
846 int client_fd;
847
848 ret = parse_address(server_ip, atoi(port), &server_sin);
849 if (ret < 0) {
850 pr_err("parse server address");
851 return -1;
852 }
853
854 ring_config = get_ring_config();
855 if (!ring_config) {
856 pr_err("Failed to get current ring configuration");
857 return -1;
858 }
859
860 if (configure_headersplit(ring_config, 1)) {
861 pr_err("Failed to enable TCP header split");
862 goto err_free_ring_config;
863 }
864
865 /* Configure RSS to divert all traffic from our devmem queues */
866 if (configure_rss()) {
867 pr_err("Failed to configure rss");
868 goto err_reset_headersplit;
869 }
870
871 /* Flow steer our devmem flows to start_queue */
872 if (configure_flow_steering(&server_sin)) {
873 pr_err("Failed to configure flow steering");
874 goto err_reset_rss;
875 }
876
877 if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys)) {
878 pr_err("Failed to bind");
879 goto err_reset_flow_steering;
880 }
881
882 tmp_mem = malloc(mem->size);
883 if (!tmp_mem)
884 goto err_unbind;
885
886 socket_fd = socket(AF_INET6, SOCK_STREAM, 0);
887 if (socket_fd < 0) {
888 pr_err("Failed to create socket");
889 goto err_free_tmp;
890 }
891
892 if (enable_reuseaddr(socket_fd))
893 goto err_close_socket;
894
895 fprintf(stderr, "binding to address %s:%d\n", server_ip,
896 ntohs(server_sin.sin6_port));
897
898 ret = bind(socket_fd, &server_sin, sizeof(server_sin));
899 if (ret) {
900 pr_err("Failed to bind");
901 goto err_close_socket;
902 }
903
904 ret = listen(socket_fd, 1);
905 if (ret) {
906 pr_err("Failed to listen");
907 goto err_close_socket;
908 }
909
910 client_addr_len = sizeof(client_addr);
911
912 inet_ntop(AF_INET6, &server_sin.sin6_addr, buffer,
913 sizeof(buffer));
914 fprintf(stderr, "Waiting or connection on %s:%d\n", buffer,
915 ntohs(server_sin.sin6_port));
916 client_fd = accept(socket_fd, &client_addr, &client_addr_len);
917 if (client_fd < 0) {
918 pr_err("Failed to accept");
919 goto err_close_socket;
920 }
921
922 inet_ntop(AF_INET6, &client_addr.sin6_addr, buffer,
923 sizeof(buffer));
924 fprintf(stderr, "Got connection from %s:%d\n", buffer,
925 ntohs(client_addr.sin6_port));
926
927 while (1) {
928 struct iovec iov = { .iov_base = iobuf,
929 .iov_len = sizeof(iobuf) };
930 struct dmabuf_cmsg *dmabuf_cmsg = NULL;
931 struct cmsghdr *cm = NULL;
932 struct msghdr msg = { 0 };
933 struct dmabuf_token token;
934 ssize_t ret;
935
936 is_devmem = false;
937
938 msg.msg_iov = &iov;
939 msg.msg_iovlen = 1;
940 msg.msg_control = ctrl_data;
941 msg.msg_controllen = sizeof(ctrl_data);
942 ret = recvmsg(client_fd, &msg, MSG_SOCK_DEVMEM);
943 fprintf(stderr, "recvmsg ret=%ld\n", ret);
944 if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
945 continue;
946 if (ret < 0) {
947 perror("recvmsg");
948 if (errno == EFAULT) {
949 pr_err("received EFAULT, won't recover");
950 goto err_close_client;
951 }
952 continue;
953 }
954 if (ret == 0) {
955 errno = 0;
956 pr_err("client exited");
957 goto cleanup;
958 }
959
960 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {
961 if (cm->cmsg_level != SOL_SOCKET ||
962 (cm->cmsg_type != SCM_DEVMEM_DMABUF &&
963 cm->cmsg_type != SCM_DEVMEM_LINEAR)) {
964 fprintf(stderr, "skipping non-devmem cmsg\n");
965 continue;
966 }
967
968 dmabuf_cmsg = (struct dmabuf_cmsg *)CMSG_DATA(cm);
969 is_devmem = true;
970
971 if (cm->cmsg_type == SCM_DEVMEM_LINEAR) {
972 /* TODO: process data copied from skb's linear
973 * buffer.
974 */
975 fprintf(stderr,
976 "SCM_DEVMEM_LINEAR. dmabuf_cmsg->frag_size=%u\n",
977 dmabuf_cmsg->frag_size);
978
979 if (fail_on_linear) {
980 pr_err("received SCM_DEVMEM_LINEAR but --fail-on-linear (-L) set");
981 goto err_close_client;
982 }
983
984 continue;
985 }
986
987 token.token_start = dmabuf_cmsg->frag_token;
988 token.token_count = 1;
989
990 total_received += dmabuf_cmsg->frag_size;
991 fprintf(stderr,
992 "received frag_page=%llu, in_page_offset=%llu, frag_offset=%llu, frag_size=%u, token=%u, total_received=%lu, dmabuf_id=%u\n",
993 dmabuf_cmsg->frag_offset >> PAGE_SHIFT,
994 dmabuf_cmsg->frag_offset % getpagesize(),
995 dmabuf_cmsg->frag_offset,
996 dmabuf_cmsg->frag_size, dmabuf_cmsg->frag_token,
997 total_received, dmabuf_cmsg->dmabuf_id);
998
999 if (dmabuf_cmsg->dmabuf_id != dmabuf_id) {
1000 pr_err("received on wrong dmabuf_id: flow steering error");
1001 goto err_close_client;
1002 }
1003
1004 if (dmabuf_cmsg->frag_size % getpagesize())
1005 non_page_aligned_frags++;
1006 else
1007 page_aligned_frags++;
1008
1009 provider->memcpy_from_device(tmp_mem, mem,
1010 dmabuf_cmsg->frag_offset,
1011 dmabuf_cmsg->frag_size);
1012
1013 if (do_validation) {
1014 if (validate_buffer(tmp_mem,
1015 dmabuf_cmsg->frag_size))
1016 goto err_close_client;
1017 } else {
1018 print_nonzero_bytes(tmp_mem,
1019 dmabuf_cmsg->frag_size);
1020 }
1021
1022 ret = setsockopt(client_fd, SOL_SOCKET,
1023 SO_DEVMEM_DONTNEED, &token,
1024 sizeof(token));
1025 if (ret != 1) {
1026 pr_err("SO_DEVMEM_DONTNEED not enough tokens");
1027 goto err_close_client;
1028 }
1029 }
1030 if (!is_devmem) {
1031 pr_err("flow steering error");
1032 goto err_close_client;
1033 }
1034
1035 fprintf(stderr, "total_received=%lu\n", total_received);
1036 }
1037
1038 fprintf(stderr, "%s: ok\n", TEST_PREFIX);
1039
1040 fprintf(stderr, "page_aligned_frags=%lu, non_page_aligned_frags=%lu\n",
1041 page_aligned_frags, non_page_aligned_frags);
1042
1043 cleanup:
1044 err = 0;
1045
1046 err_close_client:
1047 close(client_fd);
1048 err_close_socket:
1049 close(socket_fd);
1050 err_free_tmp:
1051 free(tmp_mem);
1052 err_unbind:
1053 ynl_sock_destroy(ys);
1054 err_reset_flow_steering:
1055 reset_flow_steering();
1056 err_reset_rss:
1057 reset_rss();
1058 err_reset_headersplit:
1059 restore_ring_config(ring_config);
1060 err_free_ring_config:
1061 ethtool_rings_get_rsp_free(ring_config);
1062 return err;
1063 }
1064
run_devmem_tests(void)1065 int run_devmem_tests(void)
1066 {
1067 struct ethtool_rings_get_rsp *ring_config;
1068 struct netdev_queue_id *queues;
1069 struct memory_buffer *mem;
1070 struct ynl_sock *ys;
1071 int err = -1;
1072
1073 mem = provider->alloc(getpagesize() * NUM_PAGES);
1074 if (!mem) {
1075 pr_err("Failed to allocate memory buffer");
1076 return -1;
1077 }
1078
1079 ring_config = get_ring_config();
1080 if (!ring_config) {
1081 pr_err("Failed to get current ring configuration");
1082 goto err_free_mem;
1083 }
1084
1085 /* Configure RSS to divert all traffic from our devmem queues */
1086 if (configure_rss()) {
1087 pr_err("rss error");
1088 goto err_free_ring_config;
1089 }
1090
1091 if (configure_headersplit(ring_config, 1)) {
1092 pr_err("Failed to configure header split");
1093 goto err_reset_rss;
1094 }
1095
1096 queues = netdev_queue_id_alloc(num_queues);
1097 if (!queues) {
1098 pr_err("Failed to allocate empty queues array");
1099 goto err_reset_headersplit;
1100 }
1101
1102 if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) {
1103 pr_err("Binding empty queues array should have failed");
1104 goto err_unbind;
1105 }
1106
1107 if (configure_headersplit(ring_config, 0)) {
1108 pr_err("Failed to configure header split");
1109 goto err_reset_headersplit;
1110 }
1111
1112 queues = create_queues();
1113 if (!queues) {
1114 pr_err("Failed to create queues");
1115 goto err_reset_headersplit;
1116 }
1117
1118 if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) {
1119 pr_err("Configure dmabuf with header split off should have failed");
1120 goto err_unbind;
1121 }
1122
1123 if (configure_headersplit(ring_config, 1)) {
1124 pr_err("Failed to configure header split");
1125 goto err_reset_headersplit;
1126 }
1127
1128 queues = create_queues();
1129 if (!queues) {
1130 pr_err("Failed to create queues");
1131 goto err_reset_headersplit;
1132 }
1133
1134 if (bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) {
1135 pr_err("Failed to bind");
1136 goto err_reset_headersplit;
1137 }
1138
1139 /* Deactivating a bound queue should not be legal */
1140 if (!check_changing_channels(num_queues, num_queues)) {
1141 pr_err("Deactivating a bound queue should be illegal");
1142 goto err_unbind;
1143 }
1144
1145 err = 0;
1146 goto err_unbind;
1147
1148 err_unbind:
1149 ynl_sock_destroy(ys);
1150 err_reset_headersplit:
1151 restore_ring_config(ring_config);
1152 err_reset_rss:
1153 reset_rss();
1154 err_free_ring_config:
1155 ethtool_rings_get_rsp_free(ring_config);
1156 err_free_mem:
1157 provider->free(mem);
1158 return err;
1159 }
1160
gettimeofday_ms(void)1161 static uint64_t gettimeofday_ms(void)
1162 {
1163 struct timeval tv;
1164
1165 gettimeofday(&tv, NULL);
1166 return (tv.tv_sec * 1000ULL) + (tv.tv_usec / 1000ULL);
1167 }
1168
do_poll(int fd)1169 static int do_poll(int fd)
1170 {
1171 struct pollfd pfd;
1172 int ret;
1173
1174 pfd.revents = 0;
1175 pfd.fd = fd;
1176
1177 ret = poll(&pfd, 1, waittime_ms);
1178 if (ret == -1) {
1179 pr_err("poll");
1180 return -1;
1181 }
1182
1183 return ret && (pfd.revents & POLLERR);
1184 }
1185
wait_compl(int fd)1186 static int wait_compl(int fd)
1187 {
1188 int64_t tstop = gettimeofday_ms() + waittime_ms;
1189 char control[CMSG_SPACE(100)] = {};
1190 struct sock_extended_err *serr;
1191 struct msghdr msg = {};
1192 struct cmsghdr *cm;
1193 __u32 hi, lo;
1194 int ret;
1195
1196 msg.msg_control = control;
1197 msg.msg_controllen = sizeof(control);
1198
1199 while (gettimeofday_ms() < tstop) {
1200 ret = do_poll(fd);
1201 if (ret < 0)
1202 return ret;
1203 if (!ret)
1204 continue;
1205
1206 ret = recvmsg(fd, &msg, MSG_ERRQUEUE);
1207 if (ret < 0) {
1208 if (errno == EAGAIN)
1209 continue;
1210 pr_err("recvmsg(MSG_ERRQUEUE)");
1211 return -1;
1212 }
1213 if (msg.msg_flags & MSG_CTRUNC) {
1214 pr_err("MSG_CTRUNC");
1215 return -1;
1216 }
1217
1218 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {
1219 if (cm->cmsg_level != SOL_IP &&
1220 cm->cmsg_level != SOL_IPV6)
1221 continue;
1222 if (cm->cmsg_level == SOL_IP &&
1223 cm->cmsg_type != IP_RECVERR)
1224 continue;
1225 if (cm->cmsg_level == SOL_IPV6 &&
1226 cm->cmsg_type != IPV6_RECVERR)
1227 continue;
1228
1229 serr = (void *)CMSG_DATA(cm);
1230 if (serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY) {
1231 pr_err("wrong origin %u", serr->ee_origin);
1232 return -1;
1233 }
1234 if (serr->ee_errno != 0) {
1235 pr_err("wrong errno %d", serr->ee_errno);
1236 return -1;
1237 }
1238
1239 hi = serr->ee_data;
1240 lo = serr->ee_info;
1241
1242 fprintf(stderr, "tx complete [%d,%d]\n", lo, hi);
1243 return 0;
1244 }
1245 }
1246
1247 pr_err("did not receive tx completion");
1248 return -1;
1249 }
1250
do_client(struct memory_buffer * mem)1251 static int do_client(struct memory_buffer *mem)
1252 {
1253 char ctrl_data[CMSG_SPACE(sizeof(__u32))];
1254 struct sockaddr_in6 server_sin;
1255 struct sockaddr_in6 client_sin;
1256 struct ynl_sock *ys = NULL;
1257 struct iovec iov[MAX_IOV];
1258 struct msghdr msg = {};
1259 ssize_t line_size = 0;
1260 struct cmsghdr *cmsg;
1261 char *line = NULL;
1262 int ret, err = -1;
1263 size_t len = 0;
1264 int socket_fd;
1265 __u32 ddmabuf;
1266 int opt = 1;
1267
1268 ret = parse_address(server_ip, atoi(port), &server_sin);
1269 if (ret < 0) {
1270 pr_err("parse server address");
1271 return -1;
1272 }
1273
1274 if (client_ip) {
1275 ret = parse_address(client_ip, atoi(port), &client_sin);
1276 if (ret < 0) {
1277 pr_err("parse client address");
1278 return ret;
1279 }
1280 }
1281
1282 socket_fd = socket(AF_INET6, SOCK_STREAM, 0);
1283 if (socket_fd < 0) {
1284 pr_err("create socket");
1285 return -1;
1286 }
1287
1288 if (enable_reuseaddr(socket_fd))
1289 goto err_close_socket;
1290
1291 ret = setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, ifname,
1292 strlen(ifname) + 1);
1293 if (ret) {
1294 pr_err("bindtodevice");
1295 goto err_close_socket;
1296 }
1297
1298 if (bind_tx_queue(ifindex, mem->fd, &ys)) {
1299 pr_err("Failed to bind");
1300 goto err_close_socket;
1301 }
1302
1303 if (client_ip) {
1304 ret = bind(socket_fd, &client_sin, sizeof(client_sin));
1305 if (ret) {
1306 pr_err("bind");
1307 goto err_unbind;
1308 }
1309 }
1310
1311 ret = setsockopt(socket_fd, SOL_SOCKET, SO_ZEROCOPY, &opt, sizeof(opt));
1312 if (ret) {
1313 pr_err("set sock opt");
1314 goto err_unbind;
1315 }
1316
1317 fprintf(stderr, "Connect to %s %d (via %s)\n", server_ip,
1318 ntohs(server_sin.sin6_port), ifname);
1319
1320 ret = connect(socket_fd, &server_sin, sizeof(server_sin));
1321 if (ret) {
1322 pr_err("connect");
1323 goto err_unbind;
1324 }
1325
1326 while (1) {
1327 free(line);
1328 line = NULL;
1329 line_size = getline(&line, &len, stdin);
1330
1331 if (line_size < 0)
1332 break;
1333
1334 if (max_chunk) {
1335 msg.msg_iovlen =
1336 (line_size + max_chunk - 1) / max_chunk;
1337 if (msg.msg_iovlen > MAX_IOV) {
1338 pr_err("can't partition %zd bytes into maximum of %d chunks",
1339 line_size, MAX_IOV);
1340 goto err_free_line;
1341 }
1342
1343 for (int i = 0; i < msg.msg_iovlen; i++) {
1344 iov[i].iov_base = (void *)(i * max_chunk);
1345 iov[i].iov_len = max_chunk;
1346 }
1347
1348 iov[msg.msg_iovlen - 1].iov_len =
1349 line_size - (msg.msg_iovlen - 1) * max_chunk;
1350 } else {
1351 iov[0].iov_base = 0;
1352 iov[0].iov_len = line_size;
1353 msg.msg_iovlen = 1;
1354 }
1355
1356 msg.msg_iov = iov;
1357 provider->memcpy_to_device(mem, 0, line, line_size);
1358
1359 msg.msg_control = ctrl_data;
1360 msg.msg_controllen = sizeof(ctrl_data);
1361
1362 cmsg = CMSG_FIRSTHDR(&msg);
1363 cmsg->cmsg_level = SOL_SOCKET;
1364 cmsg->cmsg_type = SCM_DEVMEM_DMABUF;
1365 cmsg->cmsg_len = CMSG_LEN(sizeof(__u32));
1366
1367 ddmabuf = tx_dmabuf_id;
1368
1369 *((__u32 *)CMSG_DATA(cmsg)) = ddmabuf;
1370
1371 ret = sendmsg(socket_fd, &msg, MSG_ZEROCOPY);
1372 if (ret < 0) {
1373 pr_err("Failed sendmsg");
1374 goto err_free_line;
1375 }
1376
1377 fprintf(stderr, "sendmsg_ret=%d\n", ret);
1378
1379 if (ret != line_size) {
1380 pr_err("Did not send all bytes %d vs %zd", ret, line_size);
1381 goto err_free_line;
1382 }
1383
1384 if (wait_compl(socket_fd))
1385 goto err_free_line;
1386 }
1387
1388 fprintf(stderr, "%s: tx ok\n", TEST_PREFIX);
1389
1390 err = 0;
1391
1392 err_free_line:
1393 free(line);
1394 err_unbind:
1395 ynl_sock_destroy(ys);
1396 err_close_socket:
1397 close(socket_fd);
1398 return err;
1399 }
1400
main(int argc,char * argv[])1401 int main(int argc, char *argv[])
1402 {
1403 struct memory_buffer *mem;
1404 int is_server = 0, opt;
1405 int ret, err = 1;
1406
1407 while ((opt = getopt(argc, argv, "Lls:c:p:v:q:t:f:z:")) != -1) {
1408 switch (opt) {
1409 case 'L':
1410 fail_on_linear = true;
1411 break;
1412 case 'l':
1413 is_server = 1;
1414 break;
1415 case 's':
1416 server_ip = optarg;
1417 break;
1418 case 'c':
1419 client_ip = optarg;
1420 break;
1421 case 'p':
1422 port = optarg;
1423 break;
1424 case 'v':
1425 do_validation = atoll(optarg);
1426 break;
1427 case 'q':
1428 num_queues = atoi(optarg);
1429 break;
1430 case 't':
1431 start_queue = atoi(optarg);
1432 break;
1433 case 'f':
1434 ifname = optarg;
1435 break;
1436 case 'z':
1437 max_chunk = atoi(optarg);
1438 break;
1439 case '?':
1440 fprintf(stderr, "unknown option: %c\n", optopt);
1441 break;
1442 }
1443 }
1444
1445 if (!ifname) {
1446 pr_err("Missing -f argument");
1447 return 1;
1448 }
1449
1450 ifindex = if_nametoindex(ifname);
1451
1452 fprintf(stderr, "using ifindex=%u\n", ifindex);
1453
1454 if (!server_ip && !client_ip) {
1455 if (start_queue < 0 && num_queues < 0) {
1456 num_queues = rxq_num(ifindex);
1457 if (num_queues < 0) {
1458 pr_err("couldn't detect number of queues");
1459 return 1;
1460 }
1461 if (num_queues < 2) {
1462 pr_err("number of device queues is too low");
1463 return 1;
1464 }
1465 /* make sure can bind to multiple queues */
1466 start_queue = num_queues / 2;
1467 num_queues /= 2;
1468 }
1469
1470 if (start_queue < 0 || num_queues < 0) {
1471 pr_err("Both -t and -q are required");
1472 return 1;
1473 }
1474
1475 return run_devmem_tests();
1476 }
1477
1478 if (start_queue < 0 && num_queues < 0) {
1479 num_queues = rxq_num(ifindex);
1480 if (num_queues < 2) {
1481 pr_err("number of device queues is too low");
1482 return 1;
1483 }
1484
1485 num_queues = 1;
1486 start_queue = rxq_num(ifindex) - num_queues;
1487
1488 if (start_queue < 0) {
1489 pr_err("couldn't detect number of queues");
1490 return 1;
1491 }
1492
1493 fprintf(stderr, "using queues %d..%d\n", start_queue, start_queue + num_queues);
1494 }
1495
1496 for (; optind < argc; optind++)
1497 fprintf(stderr, "extra arguments: %s\n", argv[optind]);
1498
1499 if (start_queue < 0) {
1500 pr_err("Missing -t argument");
1501 return 1;
1502 }
1503
1504 if (num_queues < 0) {
1505 pr_err("Missing -q argument");
1506 return 1;
1507 }
1508
1509 if (!server_ip) {
1510 pr_err("Missing -s argument");
1511 return 1;
1512 }
1513
1514 if (!port) {
1515 pr_err("Missing -p argument");
1516 return 1;
1517 }
1518
1519 mem = provider->alloc(getpagesize() * NUM_PAGES);
1520 if (!mem) {
1521 pr_err("Failed to allocate memory buffer");
1522 return 1;
1523 }
1524
1525 ret = is_server ? do_server(mem) : do_client(mem);
1526 if (ret)
1527 goto err_free_mem;
1528
1529 err = 0;
1530
1531 err_free_mem:
1532 provider->free(mem);
1533 return err;
1534 }
1535