1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * tcpdevmem netcat. Works similarly to netcat but does device memory TCP 4 * instead of regular TCP. Uses udmabuf to mock a dmabuf provider. 5 * 6 * Usage: 7 * 8 * On server: 9 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201 10 * 11 * On client: 12 * echo -n "hello\nworld" | \ 13 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1 14 * 15 * Note this is compatible with regular netcat. i.e. the sender or receiver can 16 * be replaced with regular netcat to test the RX or TX path in isolation. 17 * 18 * Test data validation (devmem TCP on RX only): 19 * 20 * On server: 21 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201 -v 7 22 * 23 * On client: 24 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06) | \ 25 * head -c 1G | \ 26 * nc <server IP> 5201 -p 5201 27 * 28 * Test data validation (devmem TCP on RX and TX, validation happens on RX): 29 * 30 * On server: 31 * ncdevmem -s <server IP> [-c <client IP>] -l -p 5201 -v 8 -f eth1 32 * 33 * On client: 34 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06\\x07) | \ 35 * head -c 1M | \ 36 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1 37 */ 38 #define _GNU_SOURCE 39 #define __EXPORTED_HEADERS__ 40 41 #include <linux/uio.h> 42 #include <stdarg.h> 43 #include <stdio.h> 44 #include <stdlib.h> 45 #include <unistd.h> 46 #include <stdbool.h> 47 #include <string.h> 48 #include <errno.h> 49 #define __iovec_defined 50 #include <fcntl.h> 51 #include <limits.h> 52 #include <malloc.h> 53 #include <error.h> 54 #include <poll.h> 55 56 #include <arpa/inet.h> 57 #include <sys/socket.h> 58 #include <sys/mman.h> 59 #include <sys/ioctl.h> 60 #include <sys/syscall.h> 61 #include <sys/time.h> 62 63 #include <linux/memfd.h> 64 #include <linux/dma-buf.h> 65 #include <linux/errqueue.h> 66 #include <linux/udmabuf.h> 67 #include <linux/types.h> 68 #include <linux/netlink.h> 69 #include <linux/genetlink.h> 70 #include <linux/netdev.h> 71 #include <linux/ethtool_netlink.h> 72 #include <time.h> 73 #include <net/if.h> 74 75 #include "netdev-user.h" 76 #include "ethtool-user.h" 77 #include <ynl.h> 78 79 #define PAGE_SHIFT 12 80 #define TEST_PREFIX "ncdevmem" 81 #define NUM_PAGES 16000 82 83 #ifndef MSG_SOCK_DEVMEM 84 #define MSG_SOCK_DEVMEM 0x2000000 85 #endif 86 87 #define MAX_IOV 1024 88 89 static size_t max_chunk; 90 static char *server_ip; 91 static char *client_ip; 92 static char *port; 93 static size_t do_validation; 94 static int start_queue = -1; 95 static int num_queues = -1; 96 static char *ifname; 97 static unsigned int ifindex; 98 static unsigned int dmabuf_id; 99 static uint32_t tx_dmabuf_id; 100 static int waittime_ms = 500; 101 static bool fail_on_linear; 102 103 /* System state loaded by current_config_load() */ 104 #define MAX_FLOWS 8 105 static int ntuple_ids[MAX_FLOWS] = { -1, -1, -1, -1, -1, -1, -1, -1, }; 106 107 struct memory_buffer { 108 int fd; 109 size_t size; 110 111 int devfd; 112 int memfd; 113 char *buf_mem; 114 }; 115 116 struct memory_provider { 117 struct memory_buffer *(*alloc)(size_t size); 118 void (*free)(struct memory_buffer *ctx); 119 void (*memcpy_to_device)(struct memory_buffer *dst, size_t off, 120 void *src, int n); 121 void (*memcpy_from_device)(void *dst, struct memory_buffer *src, 122 size_t off, int n); 123 }; 124 125 static void pr_err(const char *fmt, ...) 126 { 127 va_list args; 128 129 fprintf(stderr, "%s: ", TEST_PREFIX); 130 131 va_start(args, fmt); 132 vfprintf(stderr, fmt, args); 133 va_end(args); 134 135 if (errno != 0) 136 fprintf(stderr, ": %s", strerror(errno)); 137 fprintf(stderr, "\n"); 138 } 139 140 static struct memory_buffer *udmabuf_alloc(size_t size) 141 { 142 struct udmabuf_create create; 143 struct memory_buffer *ctx; 144 int ret; 145 146 ctx = malloc(sizeof(*ctx)); 147 if (!ctx) 148 return NULL; 149 150 ctx->size = size; 151 152 ctx->devfd = open("/dev/udmabuf", O_RDWR); 153 if (ctx->devfd < 0) { 154 pr_err("[skip,no-udmabuf: Unable to access DMA buffer device file]"); 155 goto err_free_ctx; 156 } 157 158 ctx->memfd = memfd_create("udmabuf-test", MFD_ALLOW_SEALING); 159 if (ctx->memfd < 0) { 160 pr_err("[skip,no-memfd]"); 161 goto err_close_dev; 162 } 163 164 ret = fcntl(ctx->memfd, F_ADD_SEALS, F_SEAL_SHRINK); 165 if (ret < 0) { 166 pr_err("[skip,fcntl-add-seals]"); 167 goto err_close_memfd; 168 } 169 170 ret = ftruncate(ctx->memfd, size); 171 if (ret == -1) { 172 pr_err("[FAIL,memfd-truncate]"); 173 goto err_close_memfd; 174 } 175 176 memset(&create, 0, sizeof(create)); 177 178 create.memfd = ctx->memfd; 179 create.offset = 0; 180 create.size = size; 181 ctx->fd = ioctl(ctx->devfd, UDMABUF_CREATE, &create); 182 if (ctx->fd < 0) { 183 pr_err("[FAIL, create udmabuf]"); 184 goto err_close_fd; 185 } 186 187 ctx->buf_mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, 188 ctx->fd, 0); 189 if (ctx->buf_mem == MAP_FAILED) { 190 pr_err("[FAIL, map udmabuf]"); 191 goto err_close_fd; 192 } 193 194 return ctx; 195 196 err_close_fd: 197 close(ctx->fd); 198 err_close_memfd: 199 close(ctx->memfd); 200 err_close_dev: 201 close(ctx->devfd); 202 err_free_ctx: 203 free(ctx); 204 return NULL; 205 } 206 207 static void udmabuf_free(struct memory_buffer *ctx) 208 { 209 munmap(ctx->buf_mem, ctx->size); 210 close(ctx->fd); 211 close(ctx->memfd); 212 close(ctx->devfd); 213 free(ctx); 214 } 215 216 static void udmabuf_memcpy_to_device(struct memory_buffer *dst, size_t off, 217 void *src, int n) 218 { 219 struct dma_buf_sync sync = {}; 220 221 sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE; 222 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync); 223 224 memcpy(dst->buf_mem + off, src, n); 225 226 sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE; 227 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync); 228 } 229 230 static void udmabuf_memcpy_from_device(void *dst, struct memory_buffer *src, 231 size_t off, int n) 232 { 233 struct dma_buf_sync sync = {}; 234 235 sync.flags = DMA_BUF_SYNC_START; 236 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync); 237 238 memcpy(dst, src->buf_mem + off, n); 239 240 sync.flags = DMA_BUF_SYNC_END; 241 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync); 242 } 243 244 static struct memory_provider udmabuf_memory_provider = { 245 .alloc = udmabuf_alloc, 246 .free = udmabuf_free, 247 .memcpy_to_device = udmabuf_memcpy_to_device, 248 .memcpy_from_device = udmabuf_memcpy_from_device, 249 }; 250 251 static struct memory_provider *provider = &udmabuf_memory_provider; 252 253 static void print_nonzero_bytes(void *ptr, size_t size) 254 { 255 unsigned char *p = ptr; 256 unsigned int i; 257 258 for (i = 0; i < size; i++) 259 putchar(p[i]); 260 } 261 262 int validate_buffer(void *line, size_t size) 263 { 264 static unsigned char seed = 1; 265 unsigned char *ptr = line; 266 unsigned char expected; 267 static int errors; 268 size_t i; 269 270 for (i = 0; i < size; i++) { 271 expected = seed ? seed : '\n'; 272 if (ptr[i] != expected) { 273 fprintf(stderr, 274 "Failed validation: expected=%u, actual=%u, index=%lu\n", 275 expected, ptr[i], i); 276 errors++; 277 if (errors > 20) { 278 pr_err("validation failed"); 279 return -1; 280 } 281 } 282 seed++; 283 if (seed == do_validation) 284 seed = 0; 285 } 286 287 fprintf(stdout, "Validated buffer\n"); 288 return 0; 289 } 290 291 static int 292 __run_command(char *out, size_t outlen, const char *cmd, va_list args) 293 { 294 char command[256]; 295 FILE *fp; 296 297 vsnprintf(command, sizeof(command), cmd, args); 298 299 fprintf(stderr, "Running: %s\n", command); 300 fp = popen(command, "r"); 301 if (!fp) 302 return -1; 303 if (out) { 304 size_t len; 305 306 if (!fgets(out, outlen, fp)) 307 return -1; 308 309 /* Remove trailing newline if present */ 310 len = strlen(out); 311 if (len && out[len - 1] == '\n') 312 out[len - 1] = '\0'; 313 } 314 return pclose(fp); 315 } 316 317 static int run_command(const char *cmd, ...) 318 { 319 va_list args; 320 int ret; 321 322 va_start(args, cmd); 323 ret = __run_command(NULL, 0, cmd, args); 324 va_end(args); 325 326 return ret; 327 } 328 329 static int ethtool_add_flow(const char *format, ...) 330 { 331 char local_output[256], cmd[256]; 332 const char *id_start; 333 int flow_idx, ret; 334 char *endptr; 335 long flow_id; 336 va_list args; 337 338 for (flow_idx = 0; flow_idx < MAX_FLOWS; flow_idx++) 339 if (ntuple_ids[flow_idx] == -1) 340 break; 341 if (flow_idx == MAX_FLOWS) { 342 fprintf(stderr, "Error: too many flows\n"); 343 return -1; 344 } 345 346 snprintf(cmd, sizeof(cmd), "ethtool -N %s %s", ifname, format); 347 348 va_start(args, format); 349 ret = __run_command(local_output, sizeof(local_output), cmd, args); 350 va_end(args); 351 352 if (ret != 0) 353 return ret; 354 355 /* Extract the ID from the output */ 356 id_start = strstr(local_output, "Added rule with ID "); 357 if (!id_start) 358 return -1; 359 id_start += strlen("Added rule with ID "); 360 361 flow_id = strtol(id_start, &endptr, 10); 362 if (endptr == id_start || flow_id < 0 || flow_id > INT_MAX) 363 return -1; 364 365 fprintf(stderr, "Added flow rule with ID %ld\n", flow_id); 366 ntuple_ids[flow_idx] = flow_id; 367 return flow_id; 368 } 369 370 static int rxq_num(int ifindex) 371 { 372 struct ethtool_channels_get_req *req; 373 struct ethtool_channels_get_rsp *rsp; 374 struct ynl_error yerr; 375 struct ynl_sock *ys; 376 int num = -1; 377 378 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 379 if (!ys) { 380 fprintf(stderr, "YNL: %s\n", yerr.msg); 381 return -1; 382 } 383 384 req = ethtool_channels_get_req_alloc(); 385 ethtool_channels_get_req_set_header_dev_index(req, ifindex); 386 rsp = ethtool_channels_get(ys, req); 387 if (rsp) 388 num = rsp->rx_count + rsp->combined_count; 389 ethtool_channels_get_req_free(req); 390 ethtool_channels_get_rsp_free(rsp); 391 392 ynl_sock_destroy(ys); 393 394 return num; 395 } 396 397 static void reset_flow_steering(void) 398 { 399 int i; 400 401 for (i = 0; i < MAX_FLOWS; i++) { 402 if (ntuple_ids[i] == -1) 403 continue; 404 run_command("ethtool -N %s delete %d", 405 ifname, ntuple_ids[i]); 406 ntuple_ids[i] = -1; 407 } 408 } 409 410 static const char *tcp_data_split_str(int val) 411 { 412 switch (val) { 413 case 0: 414 return "off"; 415 case 1: 416 return "auto"; 417 case 2: 418 return "on"; 419 default: 420 return "?"; 421 } 422 } 423 424 static struct ethtool_rings_get_rsp *get_ring_config(void) 425 { 426 struct ethtool_rings_get_req *get_req; 427 struct ethtool_rings_get_rsp *get_rsp; 428 struct ynl_error yerr; 429 struct ynl_sock *ys; 430 431 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 432 if (!ys) { 433 fprintf(stderr, "YNL: %s\n", yerr.msg); 434 return NULL; 435 } 436 437 get_req = ethtool_rings_get_req_alloc(); 438 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex); 439 get_rsp = ethtool_rings_get(ys, get_req); 440 ethtool_rings_get_req_free(get_req); 441 442 ynl_sock_destroy(ys); 443 444 return get_rsp; 445 } 446 447 static void restore_ring_config(const struct ethtool_rings_get_rsp *config) 448 { 449 struct ethtool_rings_get_req *get_req; 450 struct ethtool_rings_get_rsp *get_rsp; 451 struct ethtool_rings_set_req *req; 452 struct ynl_error yerr; 453 struct ynl_sock *ys; 454 int ret; 455 456 if (!config) 457 return; 458 459 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 460 if (!ys) { 461 fprintf(stderr, "YNL: %s\n", yerr.msg); 462 return; 463 } 464 465 req = ethtool_rings_set_req_alloc(); 466 ethtool_rings_set_req_set_header_dev_index(req, ifindex); 467 ethtool_rings_set_req_set_tcp_data_split(req, 468 ETHTOOL_TCP_DATA_SPLIT_UNKNOWN); 469 if (config->_present.hds_thresh) 470 ethtool_rings_set_req_set_hds_thresh(req, config->hds_thresh); 471 472 ret = ethtool_rings_set(ys, req); 473 if (ret < 0) 474 fprintf(stderr, "YNL restoring HDS cfg: %s\n", ys->err.msg); 475 476 get_req = ethtool_rings_get_req_alloc(); 477 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex); 478 get_rsp = ethtool_rings_get(ys, get_req); 479 ethtool_rings_get_req_free(get_req); 480 481 /* use explicit value if UKNOWN didn't give us the previous */ 482 if (get_rsp->tcp_data_split != config->tcp_data_split) { 483 ethtool_rings_set_req_set_tcp_data_split(req, 484 config->tcp_data_split); 485 ret = ethtool_rings_set(ys, req); 486 if (ret < 0) 487 fprintf(stderr, "YNL restoring expl HDS cfg: %s\n", 488 ys->err.msg); 489 } 490 491 ethtool_rings_get_rsp_free(get_rsp); 492 ethtool_rings_set_req_free(req); 493 494 ynl_sock_destroy(ys); 495 } 496 497 static int 498 configure_headersplit(const struct ethtool_rings_get_rsp *old, bool on) 499 { 500 struct ethtool_rings_get_req *get_req; 501 struct ethtool_rings_get_rsp *get_rsp; 502 struct ethtool_rings_set_req *req; 503 struct ynl_error yerr; 504 struct ynl_sock *ys; 505 int ret; 506 507 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 508 if (!ys) { 509 fprintf(stderr, "YNL: %s\n", yerr.msg); 510 return -1; 511 } 512 513 req = ethtool_rings_set_req_alloc(); 514 ethtool_rings_set_req_set_header_dev_index(req, ifindex); 515 if (on) { 516 ethtool_rings_set_req_set_tcp_data_split(req, 517 ETHTOOL_TCP_DATA_SPLIT_ENABLED); 518 if (old->_present.hds_thresh) 519 ethtool_rings_set_req_set_hds_thresh(req, 0); 520 } else { 521 ethtool_rings_set_req_set_tcp_data_split(req, 522 ETHTOOL_TCP_DATA_SPLIT_UNKNOWN); 523 } 524 ret = ethtool_rings_set(ys, req); 525 if (ret < 0) 526 fprintf(stderr, "YNL failed: %s\n", ys->err.msg); 527 ethtool_rings_set_req_free(req); 528 529 if (ret == 0) { 530 get_req = ethtool_rings_get_req_alloc(); 531 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex); 532 get_rsp = ethtool_rings_get(ys, get_req); 533 ethtool_rings_get_req_free(get_req); 534 if (get_rsp) 535 fprintf(stderr, "TCP header split: %s\n", 536 tcp_data_split_str(get_rsp->tcp_data_split)); 537 ethtool_rings_get_rsp_free(get_rsp); 538 } 539 540 ynl_sock_destroy(ys); 541 542 return ret; 543 } 544 545 static int configure_rss(void) 546 { 547 return run_command("ethtool -X %s equal %d >&2", ifname, start_queue); 548 } 549 550 static void reset_rss(void) 551 { 552 run_command("ethtool -X %s default >&2", ifname, start_queue); 553 } 554 555 static int check_changing_channels(unsigned int rx, unsigned int tx) 556 { 557 struct ethtool_channels_get_req *gchan; 558 struct ethtool_channels_set_req *schan; 559 struct ethtool_channels_get_rsp *chan; 560 struct ynl_error yerr; 561 struct ynl_sock *ys; 562 int ret; 563 564 fprintf(stderr, "setting channel count rx:%u tx:%u\n", rx, tx); 565 566 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 567 if (!ys) { 568 fprintf(stderr, "YNL: %s\n", yerr.msg); 569 return -1; 570 } 571 572 gchan = ethtool_channels_get_req_alloc(); 573 if (!gchan) { 574 ret = -1; 575 goto exit_close_sock; 576 } 577 578 ethtool_channels_get_req_set_header_dev_index(gchan, ifindex); 579 chan = ethtool_channels_get(ys, gchan); 580 ethtool_channels_get_req_free(gchan); 581 if (!chan) { 582 fprintf(stderr, "YNL get channels: %s\n", ys->err.msg); 583 ret = -1; 584 goto exit_close_sock; 585 } 586 587 schan = ethtool_channels_set_req_alloc(); 588 if (!schan) { 589 ret = -1; 590 goto exit_free_chan; 591 } 592 593 ethtool_channels_set_req_set_header_dev_index(schan, ifindex); 594 595 if (chan->_present.combined_count) { 596 if (chan->_present.rx_count || chan->_present.tx_count) { 597 ethtool_channels_set_req_set_rx_count(schan, 0); 598 ethtool_channels_set_req_set_tx_count(schan, 0); 599 } 600 601 if (rx == tx) { 602 ethtool_channels_set_req_set_combined_count(schan, rx); 603 } else if (rx > tx) { 604 ethtool_channels_set_req_set_combined_count(schan, tx); 605 ethtool_channels_set_req_set_rx_count(schan, rx - tx); 606 } else { 607 ethtool_channels_set_req_set_combined_count(schan, rx); 608 ethtool_channels_set_req_set_tx_count(schan, tx - rx); 609 } 610 611 } else if (chan->_present.rx_count) { 612 ethtool_channels_set_req_set_rx_count(schan, rx); 613 ethtool_channels_set_req_set_tx_count(schan, tx); 614 } else { 615 fprintf(stderr, "Error: device has neither combined nor rx channels\n"); 616 ret = -1; 617 goto exit_free_schan; 618 } 619 620 ret = ethtool_channels_set(ys, schan); 621 if (ret) { 622 fprintf(stderr, "YNL set channels: %s\n", ys->err.msg); 623 } else { 624 /* We were expecting a failure, go back to previous settings */ 625 ethtool_channels_set_req_set_combined_count(schan, 626 chan->combined_count); 627 ethtool_channels_set_req_set_rx_count(schan, chan->rx_count); 628 ethtool_channels_set_req_set_tx_count(schan, chan->tx_count); 629 630 ret = ethtool_channels_set(ys, schan); 631 if (ret) 632 fprintf(stderr, "YNL un-setting channels: %s\n", 633 ys->err.msg); 634 } 635 636 exit_free_schan: 637 ethtool_channels_set_req_free(schan); 638 exit_free_chan: 639 ethtool_channels_get_rsp_free(chan); 640 exit_close_sock: 641 ynl_sock_destroy(ys); 642 643 return ret; 644 } 645 646 static int configure_flow_steering(struct sockaddr_in6 *server_sin) 647 { 648 const char *type = "tcp6"; 649 const char *server_addr; 650 char buf[40]; 651 int flow_id; 652 653 inet_ntop(AF_INET6, &server_sin->sin6_addr, buf, sizeof(buf)); 654 server_addr = buf; 655 656 if (IN6_IS_ADDR_V4MAPPED(&server_sin->sin6_addr)) { 657 type = "tcp4"; 658 server_addr = strrchr(server_addr, ':') + 1; 659 } 660 661 /* Try configure 5-tuple */ 662 flow_id = ethtool_add_flow("flow-type %s %s %s dst-ip %s %s %s dst-port %s queue %d", 663 type, 664 client_ip ? "src-ip" : "", 665 client_ip ?: "", 666 server_addr, 667 client_ip ? "src-port" : "", 668 client_ip ? port : "", 669 port, start_queue); 670 if (flow_id < 0) { 671 /* If that fails, try configure 3-tuple */ 672 flow_id = ethtool_add_flow("flow-type %s dst-ip %s dst-port %s queue %d", 673 type, server_addr, port, start_queue); 674 if (flow_id < 0) 675 /* If that fails, return error */ 676 return -1; 677 } 678 679 return 0; 680 } 681 682 static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd, 683 struct netdev_queue_id *queues, 684 unsigned int n_queue_index, struct ynl_sock **ys) 685 { 686 struct netdev_bind_rx_req *req = NULL; 687 struct netdev_bind_rx_rsp *rsp = NULL; 688 struct ynl_error yerr; 689 690 *ys = ynl_sock_create(&ynl_netdev_family, &yerr); 691 if (!*ys) { 692 netdev_queue_id_free(queues); 693 fprintf(stderr, "YNL: %s\n", yerr.msg); 694 return -1; 695 } 696 697 req = netdev_bind_rx_req_alloc(); 698 netdev_bind_rx_req_set_ifindex(req, ifindex); 699 netdev_bind_rx_req_set_fd(req, dmabuf_fd); 700 __netdev_bind_rx_req_set_queues(req, queues, n_queue_index); 701 702 rsp = netdev_bind_rx(*ys, req); 703 if (!rsp) { 704 perror("netdev_bind_rx"); 705 goto err_close; 706 } 707 708 if (!rsp->_present.id) { 709 perror("id not present"); 710 goto err_close; 711 } 712 713 fprintf(stderr, "got dmabuf id=%d\n", rsp->id); 714 dmabuf_id = rsp->id; 715 716 netdev_bind_rx_req_free(req); 717 netdev_bind_rx_rsp_free(rsp); 718 719 return 0; 720 721 err_close: 722 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg); 723 netdev_bind_rx_req_free(req); 724 ynl_sock_destroy(*ys); 725 return -1; 726 } 727 728 static int bind_tx_queue(unsigned int ifindex, unsigned int dmabuf_fd, 729 struct ynl_sock **ys) 730 { 731 struct netdev_bind_tx_req *req = NULL; 732 struct netdev_bind_tx_rsp *rsp = NULL; 733 struct ynl_error yerr; 734 735 *ys = ynl_sock_create(&ynl_netdev_family, &yerr); 736 if (!*ys) { 737 fprintf(stderr, "YNL: %s\n", yerr.msg); 738 return -1; 739 } 740 741 req = netdev_bind_tx_req_alloc(); 742 netdev_bind_tx_req_set_ifindex(req, ifindex); 743 netdev_bind_tx_req_set_fd(req, dmabuf_fd); 744 745 rsp = netdev_bind_tx(*ys, req); 746 if (!rsp) { 747 perror("netdev_bind_tx"); 748 goto err_close; 749 } 750 751 if (!rsp->_present.id) { 752 perror("id not present"); 753 goto err_close; 754 } 755 756 fprintf(stderr, "got tx dmabuf id=%d\n", rsp->id); 757 tx_dmabuf_id = rsp->id; 758 759 netdev_bind_tx_req_free(req); 760 netdev_bind_tx_rsp_free(rsp); 761 762 return 0; 763 764 err_close: 765 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg); 766 netdev_bind_tx_req_free(req); 767 ynl_sock_destroy(*ys); 768 return -1; 769 } 770 771 static int enable_reuseaddr(int fd) 772 { 773 int opt = 1; 774 int ret; 775 776 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)); 777 if (ret) { 778 pr_err("SO_REUSEPORT failed"); 779 return -1; 780 } 781 782 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); 783 if (ret) { 784 pr_err("SO_REUSEADDR failed"); 785 return -1; 786 } 787 788 return 0; 789 } 790 791 static int parse_address(const char *str, int port, struct sockaddr_in6 *sin6) 792 { 793 int ret; 794 795 sin6->sin6_family = AF_INET6; 796 sin6->sin6_port = htons(port); 797 798 ret = inet_pton(sin6->sin6_family, str, &sin6->sin6_addr); 799 if (ret != 1) { 800 /* fallback to plain IPv4 */ 801 ret = inet_pton(AF_INET, str, &sin6->sin6_addr.s6_addr32[3]); 802 if (ret != 1) 803 return -1; 804 805 /* add ::ffff prefix */ 806 sin6->sin6_addr.s6_addr32[0] = 0; 807 sin6->sin6_addr.s6_addr32[1] = 0; 808 sin6->sin6_addr.s6_addr16[4] = 0; 809 sin6->sin6_addr.s6_addr16[5] = 0xffff; 810 } 811 812 return 0; 813 } 814 815 static struct netdev_queue_id *create_queues(void) 816 { 817 struct netdev_queue_id *queues; 818 size_t i = 0; 819 820 queues = netdev_queue_id_alloc(num_queues); 821 for (i = 0; i < num_queues; i++) { 822 netdev_queue_id_set_type(&queues[i], NETDEV_QUEUE_TYPE_RX); 823 netdev_queue_id_set_id(&queues[i], start_queue + i); 824 } 825 826 return queues; 827 } 828 829 static int do_server(struct memory_buffer *mem) 830 { 831 struct ethtool_rings_get_rsp *ring_config; 832 char ctrl_data[sizeof(int) * 20000]; 833 size_t non_page_aligned_frags = 0; 834 struct sockaddr_in6 client_addr; 835 struct sockaddr_in6 server_sin; 836 size_t page_aligned_frags = 0; 837 size_t total_received = 0; 838 socklen_t client_addr_len; 839 bool is_devmem = false; 840 char *tmp_mem = NULL; 841 struct ynl_sock *ys; 842 char iobuf[819200]; 843 int ret, err = -1; 844 char buffer[256]; 845 int socket_fd; 846 int client_fd; 847 848 ret = parse_address(server_ip, atoi(port), &server_sin); 849 if (ret < 0) { 850 pr_err("parse server address"); 851 return -1; 852 } 853 854 ring_config = get_ring_config(); 855 if (!ring_config) { 856 pr_err("Failed to get current ring configuration"); 857 return -1; 858 } 859 860 if (configure_headersplit(ring_config, 1)) { 861 pr_err("Failed to enable TCP header split"); 862 goto err_free_ring_config; 863 } 864 865 /* Configure RSS to divert all traffic from our devmem queues */ 866 if (configure_rss()) { 867 pr_err("Failed to configure rss"); 868 goto err_reset_headersplit; 869 } 870 871 /* Flow steer our devmem flows to start_queue */ 872 if (configure_flow_steering(&server_sin)) { 873 pr_err("Failed to configure flow steering"); 874 goto err_reset_rss; 875 } 876 877 if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys)) { 878 pr_err("Failed to bind"); 879 goto err_reset_flow_steering; 880 } 881 882 tmp_mem = malloc(mem->size); 883 if (!tmp_mem) 884 goto err_unbind; 885 886 socket_fd = socket(AF_INET6, SOCK_STREAM, 0); 887 if (socket_fd < 0) { 888 pr_err("Failed to create socket"); 889 goto err_free_tmp; 890 } 891 892 if (enable_reuseaddr(socket_fd)) 893 goto err_close_socket; 894 895 fprintf(stderr, "binding to address %s:%d\n", server_ip, 896 ntohs(server_sin.sin6_port)); 897 898 ret = bind(socket_fd, &server_sin, sizeof(server_sin)); 899 if (ret) { 900 pr_err("Failed to bind"); 901 goto err_close_socket; 902 } 903 904 ret = listen(socket_fd, 1); 905 if (ret) { 906 pr_err("Failed to listen"); 907 goto err_close_socket; 908 } 909 910 client_addr_len = sizeof(client_addr); 911 912 inet_ntop(AF_INET6, &server_sin.sin6_addr, buffer, 913 sizeof(buffer)); 914 fprintf(stderr, "Waiting or connection on %s:%d\n", buffer, 915 ntohs(server_sin.sin6_port)); 916 client_fd = accept(socket_fd, &client_addr, &client_addr_len); 917 if (client_fd < 0) { 918 pr_err("Failed to accept"); 919 goto err_close_socket; 920 } 921 922 inet_ntop(AF_INET6, &client_addr.sin6_addr, buffer, 923 sizeof(buffer)); 924 fprintf(stderr, "Got connection from %s:%d\n", buffer, 925 ntohs(client_addr.sin6_port)); 926 927 while (1) { 928 struct iovec iov = { .iov_base = iobuf, 929 .iov_len = sizeof(iobuf) }; 930 struct dmabuf_cmsg *dmabuf_cmsg = NULL; 931 struct cmsghdr *cm = NULL; 932 struct msghdr msg = { 0 }; 933 struct dmabuf_token token; 934 ssize_t ret; 935 936 is_devmem = false; 937 938 msg.msg_iov = &iov; 939 msg.msg_iovlen = 1; 940 msg.msg_control = ctrl_data; 941 msg.msg_controllen = sizeof(ctrl_data); 942 ret = recvmsg(client_fd, &msg, MSG_SOCK_DEVMEM); 943 fprintf(stderr, "recvmsg ret=%ld\n", ret); 944 if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) 945 continue; 946 if (ret < 0) { 947 perror("recvmsg"); 948 if (errno == EFAULT) { 949 pr_err("received EFAULT, won't recover"); 950 goto err_close_client; 951 } 952 continue; 953 } 954 if (ret == 0) { 955 errno = 0; 956 pr_err("client exited"); 957 goto cleanup; 958 } 959 960 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) { 961 if (cm->cmsg_level != SOL_SOCKET || 962 (cm->cmsg_type != SCM_DEVMEM_DMABUF && 963 cm->cmsg_type != SCM_DEVMEM_LINEAR)) { 964 fprintf(stderr, "skipping non-devmem cmsg\n"); 965 continue; 966 } 967 968 dmabuf_cmsg = (struct dmabuf_cmsg *)CMSG_DATA(cm); 969 is_devmem = true; 970 971 if (cm->cmsg_type == SCM_DEVMEM_LINEAR) { 972 /* TODO: process data copied from skb's linear 973 * buffer. 974 */ 975 fprintf(stderr, 976 "SCM_DEVMEM_LINEAR. dmabuf_cmsg->frag_size=%u\n", 977 dmabuf_cmsg->frag_size); 978 979 if (fail_on_linear) { 980 pr_err("received SCM_DEVMEM_LINEAR but --fail-on-linear (-L) set"); 981 goto err_close_client; 982 } 983 984 continue; 985 } 986 987 token.token_start = dmabuf_cmsg->frag_token; 988 token.token_count = 1; 989 990 total_received += dmabuf_cmsg->frag_size; 991 fprintf(stderr, 992 "received frag_page=%llu, in_page_offset=%llu, frag_offset=%llu, frag_size=%u, token=%u, total_received=%lu, dmabuf_id=%u\n", 993 dmabuf_cmsg->frag_offset >> PAGE_SHIFT, 994 dmabuf_cmsg->frag_offset % getpagesize(), 995 dmabuf_cmsg->frag_offset, 996 dmabuf_cmsg->frag_size, dmabuf_cmsg->frag_token, 997 total_received, dmabuf_cmsg->dmabuf_id); 998 999 if (dmabuf_cmsg->dmabuf_id != dmabuf_id) { 1000 pr_err("received on wrong dmabuf_id: flow steering error"); 1001 goto err_close_client; 1002 } 1003 1004 if (dmabuf_cmsg->frag_size % getpagesize()) 1005 non_page_aligned_frags++; 1006 else 1007 page_aligned_frags++; 1008 1009 provider->memcpy_from_device(tmp_mem, mem, 1010 dmabuf_cmsg->frag_offset, 1011 dmabuf_cmsg->frag_size); 1012 1013 if (do_validation) { 1014 if (validate_buffer(tmp_mem, 1015 dmabuf_cmsg->frag_size)) 1016 goto err_close_client; 1017 } else { 1018 print_nonzero_bytes(tmp_mem, 1019 dmabuf_cmsg->frag_size); 1020 } 1021 1022 ret = setsockopt(client_fd, SOL_SOCKET, 1023 SO_DEVMEM_DONTNEED, &token, 1024 sizeof(token)); 1025 if (ret != 1) { 1026 pr_err("SO_DEVMEM_DONTNEED not enough tokens"); 1027 goto err_close_client; 1028 } 1029 } 1030 if (!is_devmem) { 1031 pr_err("flow steering error"); 1032 goto err_close_client; 1033 } 1034 1035 fprintf(stderr, "total_received=%lu\n", total_received); 1036 } 1037 1038 fprintf(stderr, "%s: ok\n", TEST_PREFIX); 1039 1040 fprintf(stderr, "page_aligned_frags=%lu, non_page_aligned_frags=%lu\n", 1041 page_aligned_frags, non_page_aligned_frags); 1042 1043 cleanup: 1044 err = 0; 1045 1046 err_close_client: 1047 close(client_fd); 1048 err_close_socket: 1049 close(socket_fd); 1050 err_free_tmp: 1051 free(tmp_mem); 1052 err_unbind: 1053 ynl_sock_destroy(ys); 1054 err_reset_flow_steering: 1055 reset_flow_steering(); 1056 err_reset_rss: 1057 reset_rss(); 1058 err_reset_headersplit: 1059 restore_ring_config(ring_config); 1060 err_free_ring_config: 1061 ethtool_rings_get_rsp_free(ring_config); 1062 return err; 1063 } 1064 1065 int run_devmem_tests(void) 1066 { 1067 struct ethtool_rings_get_rsp *ring_config; 1068 struct netdev_queue_id *queues; 1069 struct memory_buffer *mem; 1070 struct ynl_sock *ys; 1071 int err = -1; 1072 1073 mem = provider->alloc(getpagesize() * NUM_PAGES); 1074 if (!mem) { 1075 pr_err("Failed to allocate memory buffer"); 1076 return -1; 1077 } 1078 1079 ring_config = get_ring_config(); 1080 if (!ring_config) { 1081 pr_err("Failed to get current ring configuration"); 1082 goto err_free_mem; 1083 } 1084 1085 /* Configure RSS to divert all traffic from our devmem queues */ 1086 if (configure_rss()) { 1087 pr_err("rss error"); 1088 goto err_free_ring_config; 1089 } 1090 1091 if (configure_headersplit(ring_config, 1)) { 1092 pr_err("Failed to configure header split"); 1093 goto err_reset_rss; 1094 } 1095 1096 queues = netdev_queue_id_alloc(num_queues); 1097 if (!queues) { 1098 pr_err("Failed to allocate empty queues array"); 1099 goto err_reset_headersplit; 1100 } 1101 1102 if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { 1103 pr_err("Binding empty queues array should have failed"); 1104 goto err_unbind; 1105 } 1106 1107 if (configure_headersplit(ring_config, 0)) { 1108 pr_err("Failed to configure header split"); 1109 goto err_reset_headersplit; 1110 } 1111 1112 queues = create_queues(); 1113 if (!queues) { 1114 pr_err("Failed to create queues"); 1115 goto err_reset_headersplit; 1116 } 1117 1118 if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { 1119 pr_err("Configure dmabuf with header split off should have failed"); 1120 goto err_unbind; 1121 } 1122 1123 if (configure_headersplit(ring_config, 1)) { 1124 pr_err("Failed to configure header split"); 1125 goto err_reset_headersplit; 1126 } 1127 1128 queues = create_queues(); 1129 if (!queues) { 1130 pr_err("Failed to create queues"); 1131 goto err_reset_headersplit; 1132 } 1133 1134 if (bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { 1135 pr_err("Failed to bind"); 1136 goto err_reset_headersplit; 1137 } 1138 1139 /* Deactivating a bound queue should not be legal */ 1140 if (!check_changing_channels(num_queues, num_queues)) { 1141 pr_err("Deactivating a bound queue should be illegal"); 1142 goto err_unbind; 1143 } 1144 1145 err = 0; 1146 goto err_unbind; 1147 1148 err_unbind: 1149 ynl_sock_destroy(ys); 1150 err_reset_headersplit: 1151 restore_ring_config(ring_config); 1152 err_reset_rss: 1153 reset_rss(); 1154 err_free_ring_config: 1155 ethtool_rings_get_rsp_free(ring_config); 1156 err_free_mem: 1157 provider->free(mem); 1158 return err; 1159 } 1160 1161 static uint64_t gettimeofday_ms(void) 1162 { 1163 struct timeval tv; 1164 1165 gettimeofday(&tv, NULL); 1166 return (tv.tv_sec * 1000ULL) + (tv.tv_usec / 1000ULL); 1167 } 1168 1169 static int do_poll(int fd) 1170 { 1171 struct pollfd pfd; 1172 int ret; 1173 1174 pfd.revents = 0; 1175 pfd.fd = fd; 1176 1177 ret = poll(&pfd, 1, waittime_ms); 1178 if (ret == -1) { 1179 pr_err("poll"); 1180 return -1; 1181 } 1182 1183 return ret && (pfd.revents & POLLERR); 1184 } 1185 1186 static int wait_compl(int fd) 1187 { 1188 int64_t tstop = gettimeofday_ms() + waittime_ms; 1189 char control[CMSG_SPACE(100)] = {}; 1190 struct sock_extended_err *serr; 1191 struct msghdr msg = {}; 1192 struct cmsghdr *cm; 1193 __u32 hi, lo; 1194 int ret; 1195 1196 msg.msg_control = control; 1197 msg.msg_controllen = sizeof(control); 1198 1199 while (gettimeofday_ms() < tstop) { 1200 ret = do_poll(fd); 1201 if (ret < 0) 1202 return ret; 1203 if (!ret) 1204 continue; 1205 1206 ret = recvmsg(fd, &msg, MSG_ERRQUEUE); 1207 if (ret < 0) { 1208 if (errno == EAGAIN) 1209 continue; 1210 pr_err("recvmsg(MSG_ERRQUEUE)"); 1211 return -1; 1212 } 1213 if (msg.msg_flags & MSG_CTRUNC) { 1214 pr_err("MSG_CTRUNC"); 1215 return -1; 1216 } 1217 1218 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) { 1219 if (cm->cmsg_level != SOL_IP && 1220 cm->cmsg_level != SOL_IPV6) 1221 continue; 1222 if (cm->cmsg_level == SOL_IP && 1223 cm->cmsg_type != IP_RECVERR) 1224 continue; 1225 if (cm->cmsg_level == SOL_IPV6 && 1226 cm->cmsg_type != IPV6_RECVERR) 1227 continue; 1228 1229 serr = (void *)CMSG_DATA(cm); 1230 if (serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY) { 1231 pr_err("wrong origin %u", serr->ee_origin); 1232 return -1; 1233 } 1234 if (serr->ee_errno != 0) { 1235 pr_err("wrong errno %d", serr->ee_errno); 1236 return -1; 1237 } 1238 1239 hi = serr->ee_data; 1240 lo = serr->ee_info; 1241 1242 fprintf(stderr, "tx complete [%d,%d]\n", lo, hi); 1243 return 0; 1244 } 1245 } 1246 1247 pr_err("did not receive tx completion"); 1248 return -1; 1249 } 1250 1251 static int do_client(struct memory_buffer *mem) 1252 { 1253 char ctrl_data[CMSG_SPACE(sizeof(__u32))]; 1254 struct sockaddr_in6 server_sin; 1255 struct sockaddr_in6 client_sin; 1256 struct ynl_sock *ys = NULL; 1257 struct iovec iov[MAX_IOV]; 1258 struct msghdr msg = {}; 1259 ssize_t line_size = 0; 1260 struct cmsghdr *cmsg; 1261 char *line = NULL; 1262 int ret, err = -1; 1263 size_t len = 0; 1264 int socket_fd; 1265 __u32 ddmabuf; 1266 int opt = 1; 1267 1268 ret = parse_address(server_ip, atoi(port), &server_sin); 1269 if (ret < 0) { 1270 pr_err("parse server address"); 1271 return -1; 1272 } 1273 1274 if (client_ip) { 1275 ret = parse_address(client_ip, atoi(port), &client_sin); 1276 if (ret < 0) { 1277 pr_err("parse client address"); 1278 return ret; 1279 } 1280 } 1281 1282 socket_fd = socket(AF_INET6, SOCK_STREAM, 0); 1283 if (socket_fd < 0) { 1284 pr_err("create socket"); 1285 return -1; 1286 } 1287 1288 if (enable_reuseaddr(socket_fd)) 1289 goto err_close_socket; 1290 1291 ret = setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, ifname, 1292 strlen(ifname) + 1); 1293 if (ret) { 1294 pr_err("bindtodevice"); 1295 goto err_close_socket; 1296 } 1297 1298 if (bind_tx_queue(ifindex, mem->fd, &ys)) { 1299 pr_err("Failed to bind"); 1300 goto err_close_socket; 1301 } 1302 1303 if (client_ip) { 1304 ret = bind(socket_fd, &client_sin, sizeof(client_sin)); 1305 if (ret) { 1306 pr_err("bind"); 1307 goto err_unbind; 1308 } 1309 } 1310 1311 ret = setsockopt(socket_fd, SOL_SOCKET, SO_ZEROCOPY, &opt, sizeof(opt)); 1312 if (ret) { 1313 pr_err("set sock opt"); 1314 goto err_unbind; 1315 } 1316 1317 fprintf(stderr, "Connect to %s %d (via %s)\n", server_ip, 1318 ntohs(server_sin.sin6_port), ifname); 1319 1320 ret = connect(socket_fd, &server_sin, sizeof(server_sin)); 1321 if (ret) { 1322 pr_err("connect"); 1323 goto err_unbind; 1324 } 1325 1326 while (1) { 1327 free(line); 1328 line = NULL; 1329 line_size = getline(&line, &len, stdin); 1330 1331 if (line_size < 0) 1332 break; 1333 1334 if (max_chunk) { 1335 msg.msg_iovlen = 1336 (line_size + max_chunk - 1) / max_chunk; 1337 if (msg.msg_iovlen > MAX_IOV) { 1338 pr_err("can't partition %zd bytes into maximum of %d chunks", 1339 line_size, MAX_IOV); 1340 goto err_free_line; 1341 } 1342 1343 for (int i = 0; i < msg.msg_iovlen; i++) { 1344 iov[i].iov_base = (void *)(i * max_chunk); 1345 iov[i].iov_len = max_chunk; 1346 } 1347 1348 iov[msg.msg_iovlen - 1].iov_len = 1349 line_size - (msg.msg_iovlen - 1) * max_chunk; 1350 } else { 1351 iov[0].iov_base = 0; 1352 iov[0].iov_len = line_size; 1353 msg.msg_iovlen = 1; 1354 } 1355 1356 msg.msg_iov = iov; 1357 provider->memcpy_to_device(mem, 0, line, line_size); 1358 1359 msg.msg_control = ctrl_data; 1360 msg.msg_controllen = sizeof(ctrl_data); 1361 1362 cmsg = CMSG_FIRSTHDR(&msg); 1363 cmsg->cmsg_level = SOL_SOCKET; 1364 cmsg->cmsg_type = SCM_DEVMEM_DMABUF; 1365 cmsg->cmsg_len = CMSG_LEN(sizeof(__u32)); 1366 1367 ddmabuf = tx_dmabuf_id; 1368 1369 *((__u32 *)CMSG_DATA(cmsg)) = ddmabuf; 1370 1371 ret = sendmsg(socket_fd, &msg, MSG_ZEROCOPY); 1372 if (ret < 0) { 1373 pr_err("Failed sendmsg"); 1374 goto err_free_line; 1375 } 1376 1377 fprintf(stderr, "sendmsg_ret=%d\n", ret); 1378 1379 if (ret != line_size) { 1380 pr_err("Did not send all bytes %d vs %zd", ret, line_size); 1381 goto err_free_line; 1382 } 1383 1384 if (wait_compl(socket_fd)) 1385 goto err_free_line; 1386 } 1387 1388 fprintf(stderr, "%s: tx ok\n", TEST_PREFIX); 1389 1390 err = 0; 1391 1392 err_free_line: 1393 free(line); 1394 err_unbind: 1395 ynl_sock_destroy(ys); 1396 err_close_socket: 1397 close(socket_fd); 1398 return err; 1399 } 1400 1401 int main(int argc, char *argv[]) 1402 { 1403 struct memory_buffer *mem; 1404 int is_server = 0, opt; 1405 int ret, err = 1; 1406 1407 while ((opt = getopt(argc, argv, "Lls:c:p:v:q:t:f:z:")) != -1) { 1408 switch (opt) { 1409 case 'L': 1410 fail_on_linear = true; 1411 break; 1412 case 'l': 1413 is_server = 1; 1414 break; 1415 case 's': 1416 server_ip = optarg; 1417 break; 1418 case 'c': 1419 client_ip = optarg; 1420 break; 1421 case 'p': 1422 port = optarg; 1423 break; 1424 case 'v': 1425 do_validation = atoll(optarg); 1426 break; 1427 case 'q': 1428 num_queues = atoi(optarg); 1429 break; 1430 case 't': 1431 start_queue = atoi(optarg); 1432 break; 1433 case 'f': 1434 ifname = optarg; 1435 break; 1436 case 'z': 1437 max_chunk = atoi(optarg); 1438 break; 1439 case '?': 1440 fprintf(stderr, "unknown option: %c\n", optopt); 1441 break; 1442 } 1443 } 1444 1445 if (!ifname) { 1446 pr_err("Missing -f argument"); 1447 return 1; 1448 } 1449 1450 ifindex = if_nametoindex(ifname); 1451 1452 fprintf(stderr, "using ifindex=%u\n", ifindex); 1453 1454 if (!server_ip && !client_ip) { 1455 if (start_queue < 0 && num_queues < 0) { 1456 num_queues = rxq_num(ifindex); 1457 if (num_queues < 0) { 1458 pr_err("couldn't detect number of queues"); 1459 return 1; 1460 } 1461 if (num_queues < 2) { 1462 pr_err("number of device queues is too low"); 1463 return 1; 1464 } 1465 /* make sure can bind to multiple queues */ 1466 start_queue = num_queues / 2; 1467 num_queues /= 2; 1468 } 1469 1470 if (start_queue < 0 || num_queues < 0) { 1471 pr_err("Both -t and -q are required"); 1472 return 1; 1473 } 1474 1475 return run_devmem_tests(); 1476 } 1477 1478 if (start_queue < 0 && num_queues < 0) { 1479 num_queues = rxq_num(ifindex); 1480 if (num_queues < 2) { 1481 pr_err("number of device queues is too low"); 1482 return 1; 1483 } 1484 1485 num_queues = 1; 1486 start_queue = rxq_num(ifindex) - num_queues; 1487 1488 if (start_queue < 0) { 1489 pr_err("couldn't detect number of queues"); 1490 return 1; 1491 } 1492 1493 fprintf(stderr, "using queues %d..%d\n", start_queue, start_queue + num_queues); 1494 } 1495 1496 for (; optind < argc; optind++) 1497 fprintf(stderr, "extra arguments: %s\n", argv[optind]); 1498 1499 if (start_queue < 0) { 1500 pr_err("Missing -t argument"); 1501 return 1; 1502 } 1503 1504 if (num_queues < 0) { 1505 pr_err("Missing -q argument"); 1506 return 1; 1507 } 1508 1509 if (!server_ip) { 1510 pr_err("Missing -s argument"); 1511 return 1; 1512 } 1513 1514 if (!port) { 1515 pr_err("Missing -p argument"); 1516 return 1; 1517 } 1518 1519 mem = provider->alloc(getpagesize() * NUM_PAGES); 1520 if (!mem) { 1521 pr_err("Failed to allocate memory buffer"); 1522 return 1; 1523 } 1524 1525 ret = is_server ? do_server(mem) : do_client(mem); 1526 if (ret) 1527 goto err_free_mem; 1528 1529 err = 0; 1530 1531 err_free_mem: 1532 provider->free(mem); 1533 return err; 1534 } 1535