1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * tcpdevmem netcat. Works similarly to netcat but does device memory TCP 4 * instead of regular TCP. Uses udmabuf to mock a dmabuf provider. 5 * 6 * Usage: 7 * 8 * On server: 9 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201 10 * 11 * On client: 12 * echo -n "hello\nworld" | \ 13 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1 14 * 15 * Note this is compatible with regular netcat. i.e. the sender or receiver can 16 * be replaced with regular netcat to test the RX or TX path in isolation. 17 * 18 * Test data validation (devmem TCP on RX only): 19 * 20 * On server: 21 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201 -v 7 22 * 23 * On client: 24 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06) | \ 25 * head -c 1G | \ 26 * nc <server IP> 5201 -p 5201 27 * 28 * Test data validation (devmem TCP on RX and TX, validation happens on RX): 29 * 30 * On server: 31 * ncdevmem -s <server IP> [-c <client IP>] -l -p 5201 -v 8 -f eth1 32 * 33 * On client: 34 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06\\x07) | \ 35 * head -c 1M | \ 36 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1 37 */ 38 #define _GNU_SOURCE 39 #define __EXPORTED_HEADERS__ 40 41 #include <linux/uio.h> 42 #include <stdarg.h> 43 #include <stdint.h> 44 #include <stdio.h> 45 #include <stdlib.h> 46 #include <unistd.h> 47 #include <stdbool.h> 48 #include <string.h> 49 #include <errno.h> 50 #define __iovec_defined 51 #include <fcntl.h> 52 #include <limits.h> 53 #include <malloc.h> 54 #include <error.h> 55 #include <poll.h> 56 57 #include <arpa/inet.h> 58 #include <sys/socket.h> 59 #include <sys/mman.h> 60 #include <sys/ioctl.h> 61 #include <sys/syscall.h> 62 #include <sys/time.h> 63 64 #include <linux/memfd.h> 65 #include <sys/param.h> 66 #include <linux/dma-buf.h> 67 #include <linux/errqueue.h> 68 #include <linux/udmabuf.h> 69 #include <linux/types.h> 70 #include <linux/netlink.h> 71 #include <linux/genetlink.h> 72 #include <linux/netdev.h> 73 #include <linux/ethtool_netlink.h> 74 #include <time.h> 75 #include <net/if.h> 76 77 #include "netdev-user.h" 78 #include "ethtool-user.h" 79 #include <ynl.h> 80 81 #define PAGE_SHIFT 12 82 #define TEST_PREFIX "ncdevmem" 83 #define NUM_PAGES 16000 84 #define MB(x) ((x) << 20) 85 86 #ifndef MSG_SOCK_DEVMEM 87 #define MSG_SOCK_DEVMEM 0x2000000 88 #endif 89 90 #define MAX_IOV 1024 91 92 static size_t max_chunk; 93 static char *server_ip; 94 static char *client_ip; 95 static char *port; 96 static size_t do_validation; 97 static int start_queue = -1; 98 static int num_queues = -1; 99 static int skip_config; 100 static char *ifname; 101 static unsigned int ifindex; 102 static unsigned int dmabuf_id; 103 static uint32_t tx_dmabuf_id; 104 static int waittime_ms = 500; 105 static bool fail_on_linear; 106 static uint32_t rx_page_size; 107 108 /* System state loaded by current_config_load() */ 109 #define MAX_FLOWS 8 110 static int ntuple_ids[MAX_FLOWS] = { -1, -1, -1, -1, -1, -1, -1, -1, }; 111 112 struct memory_buffer { 113 int fd; 114 size_t size; 115 116 int devfd; 117 int memfd; 118 char *buf_mem; 119 }; 120 121 struct memory_provider { 122 struct memory_buffer *(*alloc)(size_t size); 123 void (*free)(struct memory_buffer *ctx); 124 void (*memcpy_to_device)(struct memory_buffer *dst, size_t off, 125 void *src, int n); 126 void (*memcpy_from_device)(void *dst, struct memory_buffer *src, 127 size_t off, int n); 128 }; 129 130 static void pr_err(const char *fmt, ...) 131 { 132 va_list args; 133 134 fprintf(stderr, "%s: ", TEST_PREFIX); 135 136 va_start(args, fmt); 137 vfprintf(stderr, fmt, args); 138 va_end(args); 139 140 if (errno != 0) 141 fprintf(stderr, ": %s", strerror(errno)); 142 fprintf(stderr, "\n"); 143 } 144 145 static struct memory_buffer *udmabuf_alloc(size_t size) 146 { 147 struct udmabuf_create create; 148 struct memory_buffer *ctx; 149 unsigned int memfd_flags; 150 int ret; 151 152 ctx = malloc(sizeof(*ctx)); 153 if (!ctx) 154 return NULL; 155 156 ctx->size = size; 157 158 ctx->devfd = open("/dev/udmabuf", O_RDONLY); 159 if (ctx->devfd < 0) { 160 pr_err("[skip,no-udmabuf: Unable to access DMA buffer device file]"); 161 goto err_free_ctx; 162 } 163 164 memfd_flags = MFD_ALLOW_SEALING; 165 if (rx_page_size > getpagesize()) 166 memfd_flags |= MFD_HUGETLB | MFD_HUGE_2MB; 167 168 ctx->memfd = memfd_create("udmabuf-test", memfd_flags); 169 if (ctx->memfd < 0) { 170 pr_err("[skip,no-memfd%s]", 171 (memfd_flags & MFD_HUGETLB) ? " (need hugepages)" : ""); 172 goto err_close_dev; 173 } 174 175 ret = fcntl(ctx->memfd, F_ADD_SEALS, F_SEAL_SHRINK); 176 if (ret < 0) { 177 pr_err("[skip,fcntl-add-seals]"); 178 goto err_close_memfd; 179 } 180 181 if (memfd_flags & MFD_HUGETLB) { 182 size = roundup(size, MB(2)); 183 ctx->size = size; 184 } 185 186 ret = ftruncate(ctx->memfd, size); 187 if (ret == -1) { 188 pr_err("[FAIL,memfd-truncate]"); 189 goto err_close_memfd; 190 } 191 192 memset(&create, 0, sizeof(create)); 193 194 create.memfd = ctx->memfd; 195 create.offset = 0; 196 create.size = size; 197 ctx->fd = ioctl(ctx->devfd, UDMABUF_CREATE, &create); 198 if (ctx->fd < 0) { 199 pr_err("[FAIL, create udmabuf]"); 200 goto err_close_fd; 201 } 202 203 ctx->buf_mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, 204 ctx->fd, 0); 205 if (ctx->buf_mem == MAP_FAILED) { 206 pr_err("[FAIL, map udmabuf]"); 207 goto err_close_fd; 208 } 209 210 return ctx; 211 212 err_close_fd: 213 close(ctx->fd); 214 err_close_memfd: 215 close(ctx->memfd); 216 err_close_dev: 217 close(ctx->devfd); 218 err_free_ctx: 219 free(ctx); 220 return NULL; 221 } 222 223 static void udmabuf_free(struct memory_buffer *ctx) 224 { 225 munmap(ctx->buf_mem, ctx->size); 226 close(ctx->fd); 227 close(ctx->memfd); 228 close(ctx->devfd); 229 free(ctx); 230 } 231 232 static void udmabuf_memcpy_to_device(struct memory_buffer *dst, size_t off, 233 void *src, int n) 234 { 235 struct dma_buf_sync sync = {}; 236 237 sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE; 238 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync); 239 240 memcpy(dst->buf_mem + off, src, n); 241 242 sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE; 243 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync); 244 } 245 246 static void udmabuf_memcpy_from_device(void *dst, struct memory_buffer *src, 247 size_t off, int n) 248 { 249 struct dma_buf_sync sync = {}; 250 251 sync.flags = DMA_BUF_SYNC_START; 252 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync); 253 254 memcpy(dst, src->buf_mem + off, n); 255 256 sync.flags = DMA_BUF_SYNC_END; 257 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync); 258 } 259 260 static struct memory_provider udmabuf_memory_provider = { 261 .alloc = udmabuf_alloc, 262 .free = udmabuf_free, 263 .memcpy_to_device = udmabuf_memcpy_to_device, 264 .memcpy_from_device = udmabuf_memcpy_from_device, 265 }; 266 267 static struct memory_provider *provider = &udmabuf_memory_provider; 268 269 static void print_nonzero_bytes(void *ptr, size_t size) 270 { 271 unsigned char *p = ptr; 272 unsigned int i; 273 274 for (i = 0; i < size; i++) 275 putchar(p[i]); 276 } 277 278 int validate_buffer(void *line, size_t size) 279 { 280 static unsigned char seed = 1; 281 unsigned char *ptr = line; 282 unsigned char expected; 283 static int errors; 284 size_t i; 285 286 for (i = 0; i < size; i++) { 287 expected = seed ? seed : '\n'; 288 if (ptr[i] != expected) { 289 fprintf(stderr, 290 "Failed validation: expected=%u, actual=%u, index=%lu\n", 291 expected, ptr[i], i); 292 errors++; 293 if (errors > 20) { 294 pr_err("validation failed"); 295 return -1; 296 } 297 } 298 seed++; 299 if (seed == do_validation) 300 seed = 0; 301 } 302 303 fprintf(stdout, "Validated buffer\n"); 304 return 0; 305 } 306 307 static int 308 __run_command(char *out, size_t outlen, const char *cmd, va_list args) 309 { 310 char command[256]; 311 FILE *fp; 312 313 vsnprintf(command, sizeof(command), cmd, args); 314 315 fprintf(stderr, "Running: %s\n", command); 316 fp = popen(command, "r"); 317 if (!fp) 318 return -1; 319 if (out) { 320 size_t len; 321 322 if (!fgets(out, outlen, fp)) 323 return -1; 324 325 /* Remove trailing newline if present */ 326 len = strlen(out); 327 if (len && out[len - 1] == '\n') 328 out[len - 1] = '\0'; 329 } 330 return pclose(fp); 331 } 332 333 static int run_command(const char *cmd, ...) 334 { 335 va_list args; 336 int ret; 337 338 va_start(args, cmd); 339 ret = __run_command(NULL, 0, cmd, args); 340 va_end(args); 341 342 return ret; 343 } 344 345 static int ethtool_add_flow(const char *format, ...) 346 { 347 char local_output[256], cmd[256]; 348 const char *id_start; 349 int flow_idx, ret; 350 char *endptr; 351 long flow_id; 352 va_list args; 353 354 for (flow_idx = 0; flow_idx < MAX_FLOWS; flow_idx++) 355 if (ntuple_ids[flow_idx] == -1) 356 break; 357 if (flow_idx == MAX_FLOWS) { 358 fprintf(stderr, "Error: too many flows\n"); 359 return -1; 360 } 361 362 snprintf(cmd, sizeof(cmd), "ethtool -N %s %s", ifname, format); 363 364 va_start(args, format); 365 ret = __run_command(local_output, sizeof(local_output), cmd, args); 366 va_end(args); 367 368 if (ret != 0) 369 return ret; 370 371 /* Extract the ID from the output */ 372 id_start = strstr(local_output, "Added rule with ID "); 373 if (!id_start) 374 return -1; 375 id_start += strlen("Added rule with ID "); 376 377 flow_id = strtol(id_start, &endptr, 10); 378 if (endptr == id_start || flow_id < 0 || flow_id > INT_MAX) 379 return -1; 380 381 fprintf(stderr, "Added flow rule with ID %ld\n", flow_id); 382 ntuple_ids[flow_idx] = flow_id; 383 return flow_id; 384 } 385 386 static int rxq_num(int ifindex) 387 { 388 struct ethtool_channels_get_req *req; 389 struct ethtool_channels_get_rsp *rsp; 390 struct ynl_error yerr; 391 struct ynl_sock *ys; 392 int num = -1; 393 394 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 395 if (!ys) { 396 fprintf(stderr, "YNL: %s\n", yerr.msg); 397 return -1; 398 } 399 400 req = ethtool_channels_get_req_alloc(); 401 ethtool_channels_get_req_set_header_dev_index(req, ifindex); 402 rsp = ethtool_channels_get(ys, req); 403 if (rsp) 404 num = rsp->rx_count + rsp->combined_count; 405 ethtool_channels_get_req_free(req); 406 ethtool_channels_get_rsp_free(rsp); 407 408 ynl_sock_destroy(ys); 409 410 return num; 411 } 412 413 static void reset_flow_steering(void) 414 { 415 int i; 416 417 for (i = 0; i < MAX_FLOWS; i++) { 418 if (ntuple_ids[i] == -1) 419 continue; 420 run_command("ethtool -N %s delete %d", 421 ifname, ntuple_ids[i]); 422 ntuple_ids[i] = -1; 423 } 424 } 425 426 static const char *tcp_data_split_str(int val) 427 { 428 switch (val) { 429 case 0: 430 return "off"; 431 case 1: 432 return "auto"; 433 case 2: 434 return "on"; 435 default: 436 return "?"; 437 } 438 } 439 440 static struct ethtool_rings_get_rsp *get_ring_config(void) 441 { 442 struct ethtool_rings_get_req *get_req; 443 struct ethtool_rings_get_rsp *get_rsp; 444 struct ynl_error yerr; 445 struct ynl_sock *ys; 446 447 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 448 if (!ys) { 449 fprintf(stderr, "YNL: %s\n", yerr.msg); 450 return NULL; 451 } 452 453 get_req = ethtool_rings_get_req_alloc(); 454 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex); 455 get_rsp = ethtool_rings_get(ys, get_req); 456 ethtool_rings_get_req_free(get_req); 457 458 ynl_sock_destroy(ys); 459 460 return get_rsp; 461 } 462 463 static void restore_ring_config(const struct ethtool_rings_get_rsp *config) 464 { 465 struct ethtool_rings_get_req *get_req; 466 struct ethtool_rings_get_rsp *get_rsp; 467 struct ethtool_rings_set_req *req; 468 struct ynl_error yerr; 469 struct ynl_sock *ys; 470 int ret; 471 472 if (!config) 473 return; 474 475 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 476 if (!ys) { 477 fprintf(stderr, "YNL: %s\n", yerr.msg); 478 return; 479 } 480 481 req = ethtool_rings_set_req_alloc(); 482 ethtool_rings_set_req_set_header_dev_index(req, ifindex); 483 ethtool_rings_set_req_set_tcp_data_split(req, 484 ETHTOOL_TCP_DATA_SPLIT_UNKNOWN); 485 if (config->_present.hds_thresh) 486 ethtool_rings_set_req_set_hds_thresh(req, config->hds_thresh); 487 488 ret = ethtool_rings_set(ys, req); 489 if (ret < 0) 490 fprintf(stderr, "YNL restoring HDS cfg: %s\n", ys->err.msg); 491 492 get_req = ethtool_rings_get_req_alloc(); 493 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex); 494 get_rsp = ethtool_rings_get(ys, get_req); 495 ethtool_rings_get_req_free(get_req); 496 497 /* use explicit value if UKNOWN didn't give us the previous */ 498 if (get_rsp->tcp_data_split != config->tcp_data_split) { 499 ethtool_rings_set_req_set_tcp_data_split(req, 500 config->tcp_data_split); 501 ret = ethtool_rings_set(ys, req); 502 if (ret < 0) 503 fprintf(stderr, "YNL restoring expl HDS cfg: %s\n", 504 ys->err.msg); 505 } 506 507 ethtool_rings_get_rsp_free(get_rsp); 508 ethtool_rings_set_req_free(req); 509 510 ynl_sock_destroy(ys); 511 } 512 513 static int 514 configure_headersplit(const struct ethtool_rings_get_rsp *old, bool on) 515 { 516 struct ethtool_rings_get_req *get_req; 517 struct ethtool_rings_get_rsp *get_rsp; 518 struct ethtool_rings_set_req *req; 519 struct ynl_error yerr; 520 struct ynl_sock *ys; 521 int ret; 522 523 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 524 if (!ys) { 525 fprintf(stderr, "YNL: %s\n", yerr.msg); 526 return -1; 527 } 528 529 req = ethtool_rings_set_req_alloc(); 530 ethtool_rings_set_req_set_header_dev_index(req, ifindex); 531 if (on) { 532 ethtool_rings_set_req_set_tcp_data_split(req, 533 ETHTOOL_TCP_DATA_SPLIT_ENABLED); 534 if (old->_present.hds_thresh) 535 ethtool_rings_set_req_set_hds_thresh(req, 0); 536 } else { 537 ethtool_rings_set_req_set_tcp_data_split(req, 538 ETHTOOL_TCP_DATA_SPLIT_UNKNOWN); 539 } 540 ret = ethtool_rings_set(ys, req); 541 if (ret < 0) 542 fprintf(stderr, "YNL failed: %s\n", ys->err.msg); 543 ethtool_rings_set_req_free(req); 544 545 if (ret == 0) { 546 get_req = ethtool_rings_get_req_alloc(); 547 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex); 548 get_rsp = ethtool_rings_get(ys, get_req); 549 ethtool_rings_get_req_free(get_req); 550 if (get_rsp) 551 fprintf(stderr, "TCP header split: %s\n", 552 tcp_data_split_str(get_rsp->tcp_data_split)); 553 ethtool_rings_get_rsp_free(get_rsp); 554 } 555 556 ynl_sock_destroy(ys); 557 558 return ret; 559 } 560 561 static int configure_rss(void) 562 { 563 return run_command("ethtool -X %s equal %d >&2", ifname, start_queue); 564 } 565 566 static void reset_rss(void) 567 { 568 run_command("ethtool -X %s default >&2", ifname, start_queue); 569 } 570 571 static int check_changing_channels(unsigned int rx, unsigned int tx) 572 { 573 struct ethtool_channels_get_req *gchan; 574 struct ethtool_channels_set_req *schan; 575 struct ethtool_channels_get_rsp *chan; 576 struct ynl_error yerr; 577 struct ynl_sock *ys; 578 int ret; 579 580 fprintf(stderr, "setting channel count rx:%u tx:%u\n", rx, tx); 581 582 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 583 if (!ys) { 584 fprintf(stderr, "YNL: %s\n", yerr.msg); 585 return -1; 586 } 587 588 gchan = ethtool_channels_get_req_alloc(); 589 if (!gchan) { 590 ret = -1; 591 goto exit_close_sock; 592 } 593 594 ethtool_channels_get_req_set_header_dev_index(gchan, ifindex); 595 chan = ethtool_channels_get(ys, gchan); 596 ethtool_channels_get_req_free(gchan); 597 if (!chan) { 598 fprintf(stderr, "YNL get channels: %s\n", ys->err.msg); 599 ret = -1; 600 goto exit_close_sock; 601 } 602 603 schan = ethtool_channels_set_req_alloc(); 604 if (!schan) { 605 ret = -1; 606 goto exit_free_chan; 607 } 608 609 ethtool_channels_set_req_set_header_dev_index(schan, ifindex); 610 611 if (chan->_present.combined_count) { 612 if (chan->_present.rx_count || chan->_present.tx_count) { 613 ethtool_channels_set_req_set_rx_count(schan, 0); 614 ethtool_channels_set_req_set_tx_count(schan, 0); 615 } 616 617 if (rx == tx) { 618 ethtool_channels_set_req_set_combined_count(schan, rx); 619 } else if (rx > tx) { 620 ethtool_channels_set_req_set_combined_count(schan, tx); 621 ethtool_channels_set_req_set_rx_count(schan, rx - tx); 622 } else { 623 ethtool_channels_set_req_set_combined_count(schan, rx); 624 ethtool_channels_set_req_set_tx_count(schan, tx - rx); 625 } 626 627 } else if (chan->_present.rx_count) { 628 ethtool_channels_set_req_set_rx_count(schan, rx); 629 ethtool_channels_set_req_set_tx_count(schan, tx); 630 } else { 631 fprintf(stderr, "Error: device has neither combined nor rx channels\n"); 632 ret = -1; 633 goto exit_free_schan; 634 } 635 636 ret = ethtool_channels_set(ys, schan); 637 if (ret) { 638 fprintf(stderr, "YNL set channels: %s\n", ys->err.msg); 639 } else { 640 /* We were expecting a failure, go back to previous settings */ 641 ethtool_channels_set_req_set_combined_count(schan, 642 chan->combined_count); 643 ethtool_channels_set_req_set_rx_count(schan, chan->rx_count); 644 ethtool_channels_set_req_set_tx_count(schan, chan->tx_count); 645 646 ret = ethtool_channels_set(ys, schan); 647 if (ret) 648 fprintf(stderr, "YNL un-setting channels: %s\n", 649 ys->err.msg); 650 } 651 652 exit_free_schan: 653 ethtool_channels_set_req_free(schan); 654 exit_free_chan: 655 ethtool_channels_get_rsp_free(chan); 656 exit_close_sock: 657 ynl_sock_destroy(ys); 658 659 return ret; 660 } 661 662 static int configure_flow_steering(struct sockaddr_in6 *server_sin) 663 { 664 const char *type = "tcp6"; 665 const char *server_addr; 666 char buf[40]; 667 int flow_id; 668 669 inet_ntop(AF_INET6, &server_sin->sin6_addr, buf, sizeof(buf)); 670 server_addr = buf; 671 672 if (IN6_IS_ADDR_V4MAPPED(&server_sin->sin6_addr)) { 673 type = "tcp4"; 674 server_addr = strrchr(server_addr, ':') + 1; 675 } 676 677 /* Try configure 5-tuple */ 678 flow_id = ethtool_add_flow("flow-type %s %s %s dst-ip %s %s %s dst-port %s queue %d", 679 type, 680 client_ip ? "src-ip" : "", 681 client_ip ?: "", 682 server_addr, 683 client_ip ? "src-port" : "", 684 client_ip ? port : "", 685 port, start_queue); 686 if (flow_id < 0) { 687 /* If that fails, try configure 3-tuple */ 688 flow_id = ethtool_add_flow("flow-type %s dst-ip %s dst-port %s queue %d", 689 type, server_addr, port, start_queue); 690 if (flow_id < 0) 691 /* If that fails, return error */ 692 return -1; 693 } 694 695 return 0; 696 } 697 698 static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd, 699 struct netdev_queue_id *queues, 700 unsigned int n_queue_index, struct ynl_sock **ys) 701 { 702 struct netdev_bind_rx_req *req = NULL; 703 struct netdev_bind_rx_rsp *rsp = NULL; 704 struct ynl_error yerr; 705 706 *ys = ynl_sock_create(&ynl_netdev_family, &yerr); 707 if (!*ys) { 708 netdev_queue_id_free(queues); 709 fprintf(stderr, "YNL: %s\n", yerr.msg); 710 return -1; 711 } 712 713 req = netdev_bind_rx_req_alloc(); 714 netdev_bind_rx_req_set_ifindex(req, ifindex); 715 netdev_bind_rx_req_set_fd(req, dmabuf_fd); 716 __netdev_bind_rx_req_set_queues(req, queues, n_queue_index); 717 if (rx_page_size) 718 netdev_bind_rx_req_set_rx_page_size(req, rx_page_size); 719 720 rsp = netdev_bind_rx(*ys, req); 721 if (!rsp) { 722 perror("netdev_bind_rx"); 723 goto err_close; 724 } 725 726 if (!rsp->_present.id) { 727 perror("id not present"); 728 goto err_close; 729 } 730 731 fprintf(stderr, "got dmabuf id=%d\n", rsp->id); 732 dmabuf_id = rsp->id; 733 734 netdev_bind_rx_req_free(req); 735 netdev_bind_rx_rsp_free(rsp); 736 737 return 0; 738 739 err_close: 740 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg); 741 netdev_bind_rx_req_free(req); 742 ynl_sock_destroy(*ys); 743 return -1; 744 } 745 746 static int bind_tx_queue(unsigned int ifindex, unsigned int dmabuf_fd, 747 struct ynl_sock **ys) 748 { 749 struct netdev_bind_tx_req *req = NULL; 750 struct netdev_bind_tx_rsp *rsp = NULL; 751 struct ynl_error yerr; 752 753 *ys = ynl_sock_create(&ynl_netdev_family, &yerr); 754 if (!*ys) { 755 fprintf(stderr, "YNL: %s\n", yerr.msg); 756 return -1; 757 } 758 759 req = netdev_bind_tx_req_alloc(); 760 netdev_bind_tx_req_set_ifindex(req, ifindex); 761 netdev_bind_tx_req_set_fd(req, dmabuf_fd); 762 763 rsp = netdev_bind_tx(*ys, req); 764 if (!rsp) { 765 perror("netdev_bind_tx"); 766 goto err_close; 767 } 768 769 if (!rsp->_present.id) { 770 perror("id not present"); 771 goto err_close; 772 } 773 774 fprintf(stderr, "got tx dmabuf id=%d\n", rsp->id); 775 tx_dmabuf_id = rsp->id; 776 777 netdev_bind_tx_req_free(req); 778 netdev_bind_tx_rsp_free(rsp); 779 780 return 0; 781 782 err_close: 783 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg); 784 netdev_bind_tx_req_free(req); 785 ynl_sock_destroy(*ys); 786 return -1; 787 } 788 789 static int enable_reuseaddr(int fd) 790 { 791 int opt = 1; 792 int ret; 793 794 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)); 795 if (ret) { 796 pr_err("SO_REUSEPORT failed"); 797 return -1; 798 } 799 800 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); 801 if (ret) { 802 pr_err("SO_REUSEADDR failed"); 803 return -1; 804 } 805 806 return 0; 807 } 808 809 static int parse_address(const char *str, int port, struct sockaddr_in6 *sin6) 810 { 811 int ret; 812 813 sin6->sin6_family = AF_INET6; 814 sin6->sin6_port = htons(port); 815 816 ret = inet_pton(sin6->sin6_family, str, &sin6->sin6_addr); 817 if (ret != 1) { 818 /* fallback to plain IPv4 */ 819 ret = inet_pton(AF_INET, str, &sin6->sin6_addr.s6_addr32[3]); 820 if (ret != 1) 821 return -1; 822 823 /* add ::ffff prefix */ 824 sin6->sin6_addr.s6_addr32[0] = 0; 825 sin6->sin6_addr.s6_addr32[1] = 0; 826 sin6->sin6_addr.s6_addr16[4] = 0; 827 sin6->sin6_addr.s6_addr16[5] = 0xffff; 828 } 829 830 return 0; 831 } 832 833 static struct netdev_queue_id *create_queues(void) 834 { 835 struct netdev_queue_id *queues; 836 size_t i = 0; 837 838 queues = netdev_queue_id_alloc(num_queues); 839 for (i = 0; i < num_queues; i++) { 840 netdev_queue_id_set_type(&queues[i], NETDEV_QUEUE_TYPE_RX); 841 netdev_queue_id_set_id(&queues[i], start_queue + i); 842 } 843 844 return queues; 845 } 846 847 static int do_server(struct memory_buffer *mem) 848 { 849 struct ethtool_rings_get_rsp *ring_config = NULL; 850 char ctrl_data[sizeof(int) * 20000]; 851 size_t non_page_aligned_frags = 0; 852 struct sockaddr_in6 client_addr; 853 struct sockaddr_in6 server_sin; 854 size_t page_aligned_frags = 0; 855 size_t total_received = 0; 856 socklen_t client_addr_len; 857 bool is_devmem = false; 858 char *tmp_mem = NULL; 859 struct ynl_sock *ys; 860 char iobuf[819200]; 861 int ret, err = -1; 862 char buffer[256]; 863 int socket_fd; 864 int client_fd; 865 866 ret = parse_address(server_ip, atoi(port), &server_sin); 867 if (ret < 0) { 868 pr_err("parse server address"); 869 return -1; 870 } 871 872 if (!skip_config) { 873 ring_config = get_ring_config(); 874 if (!ring_config) { 875 pr_err("Failed to get current ring configuration"); 876 return -1; 877 } 878 879 if (configure_headersplit(ring_config, 1)) { 880 pr_err("Failed to enable TCP header split"); 881 goto err_free_ring_config; 882 } 883 884 /* Configure RSS to divert all traffic from our devmem queues */ 885 if (configure_rss()) { 886 pr_err("Failed to configure rss"); 887 goto err_reset_headersplit; 888 } 889 890 /* Flow steer our devmem flows to start_queue */ 891 if (configure_flow_steering(&server_sin)) { 892 pr_err("Failed to configure flow steering"); 893 goto err_reset_rss; 894 } 895 } 896 897 if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys)) { 898 pr_err("Failed to bind"); 899 goto err_reset_flow_steering; 900 } 901 902 tmp_mem = malloc(mem->size); 903 if (!tmp_mem) 904 goto err_unbind; 905 906 socket_fd = socket(AF_INET6, SOCK_STREAM, 0); 907 if (socket_fd < 0) { 908 pr_err("Failed to create socket"); 909 goto err_free_tmp; 910 } 911 912 if (enable_reuseaddr(socket_fd)) 913 goto err_close_socket; 914 915 fprintf(stderr, "binding to address %s:%d\n", server_ip, 916 ntohs(server_sin.sin6_port)); 917 918 ret = bind(socket_fd, &server_sin, sizeof(server_sin)); 919 if (ret) { 920 pr_err("Failed to bind"); 921 goto err_close_socket; 922 } 923 924 ret = listen(socket_fd, 1); 925 if (ret) { 926 pr_err("Failed to listen"); 927 goto err_close_socket; 928 } 929 930 client_addr_len = sizeof(client_addr); 931 932 inet_ntop(AF_INET6, &server_sin.sin6_addr, buffer, 933 sizeof(buffer)); 934 fprintf(stderr, "Waiting or connection on %s:%d\n", buffer, 935 ntohs(server_sin.sin6_port)); 936 client_fd = accept(socket_fd, &client_addr, &client_addr_len); 937 if (client_fd < 0) { 938 pr_err("Failed to accept"); 939 goto err_close_socket; 940 } 941 942 inet_ntop(AF_INET6, &client_addr.sin6_addr, buffer, 943 sizeof(buffer)); 944 fprintf(stderr, "Got connection from %s:%d\n", buffer, 945 ntohs(client_addr.sin6_port)); 946 947 while (1) { 948 struct iovec iov = { .iov_base = iobuf, 949 .iov_len = sizeof(iobuf) }; 950 struct dmabuf_cmsg *dmabuf_cmsg = NULL; 951 struct cmsghdr *cm = NULL; 952 struct msghdr msg = { 0 }; 953 struct dmabuf_token token; 954 ssize_t ret; 955 956 is_devmem = false; 957 958 msg.msg_iov = &iov; 959 msg.msg_iovlen = 1; 960 msg.msg_control = ctrl_data; 961 msg.msg_controllen = sizeof(ctrl_data); 962 ret = recvmsg(client_fd, &msg, MSG_SOCK_DEVMEM); 963 fprintf(stderr, "recvmsg ret=%ld\n", ret); 964 if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) 965 continue; 966 if (ret < 0) { 967 perror("recvmsg"); 968 if (errno == EFAULT) { 969 pr_err("received EFAULT, won't recover"); 970 goto err_close_client; 971 } 972 continue; 973 } 974 if (ret == 0) { 975 errno = 0; 976 pr_err("client exited"); 977 goto cleanup; 978 } 979 980 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) { 981 if (cm->cmsg_level != SOL_SOCKET || 982 (cm->cmsg_type != SCM_DEVMEM_DMABUF && 983 cm->cmsg_type != SCM_DEVMEM_LINEAR)) { 984 fprintf(stderr, "skipping non-devmem cmsg\n"); 985 continue; 986 } 987 988 dmabuf_cmsg = (struct dmabuf_cmsg *)CMSG_DATA(cm); 989 is_devmem = true; 990 991 if (cm->cmsg_type == SCM_DEVMEM_LINEAR) { 992 /* TODO: process data copied from skb's linear 993 * buffer. 994 */ 995 fprintf(stderr, 996 "SCM_DEVMEM_LINEAR. dmabuf_cmsg->frag_size=%u\n", 997 dmabuf_cmsg->frag_size); 998 999 if (fail_on_linear) { 1000 pr_err("received SCM_DEVMEM_LINEAR but --fail-on-linear (-L) set"); 1001 goto err_close_client; 1002 } 1003 1004 continue; 1005 } 1006 1007 token.token_start = dmabuf_cmsg->frag_token; 1008 token.token_count = 1; 1009 1010 total_received += dmabuf_cmsg->frag_size; 1011 fprintf(stderr, 1012 "received frag_page=%llu, in_page_offset=%llu, frag_offset=%llu, frag_size=%u, token=%u, total_received=%lu, dmabuf_id=%u\n", 1013 dmabuf_cmsg->frag_offset >> PAGE_SHIFT, 1014 dmabuf_cmsg->frag_offset % getpagesize(), 1015 dmabuf_cmsg->frag_offset, 1016 dmabuf_cmsg->frag_size, dmabuf_cmsg->frag_token, 1017 total_received, dmabuf_cmsg->dmabuf_id); 1018 1019 if (dmabuf_cmsg->dmabuf_id != dmabuf_id) { 1020 pr_err("received on wrong dmabuf_id: flow steering error"); 1021 goto err_close_client; 1022 } 1023 1024 if (dmabuf_cmsg->frag_size % getpagesize()) 1025 non_page_aligned_frags++; 1026 else 1027 page_aligned_frags++; 1028 1029 provider->memcpy_from_device(tmp_mem, mem, 1030 dmabuf_cmsg->frag_offset, 1031 dmabuf_cmsg->frag_size); 1032 1033 if (do_validation) { 1034 if (validate_buffer(tmp_mem, 1035 dmabuf_cmsg->frag_size)) 1036 goto err_close_client; 1037 } else { 1038 print_nonzero_bytes(tmp_mem, 1039 dmabuf_cmsg->frag_size); 1040 } 1041 1042 ret = setsockopt(client_fd, SOL_SOCKET, 1043 SO_DEVMEM_DONTNEED, &token, 1044 sizeof(token)); 1045 if (ret != 1) { 1046 pr_err("SO_DEVMEM_DONTNEED not enough tokens"); 1047 goto err_close_client; 1048 } 1049 } 1050 if (!is_devmem) { 1051 pr_err("flow steering error"); 1052 goto err_close_client; 1053 } 1054 1055 fprintf(stderr, "total_received=%lu\n", total_received); 1056 } 1057 1058 fprintf(stderr, "%s: ok\n", TEST_PREFIX); 1059 1060 fprintf(stderr, "page_aligned_frags=%lu, non_page_aligned_frags=%lu\n", 1061 page_aligned_frags, non_page_aligned_frags); 1062 1063 cleanup: 1064 err = 0; 1065 1066 err_close_client: 1067 close(client_fd); 1068 err_close_socket: 1069 close(socket_fd); 1070 err_free_tmp: 1071 free(tmp_mem); 1072 err_unbind: 1073 ynl_sock_destroy(ys); 1074 err_reset_flow_steering: 1075 if (!skip_config) 1076 reset_flow_steering(); 1077 err_reset_rss: 1078 if (!skip_config) 1079 reset_rss(); 1080 err_reset_headersplit: 1081 if (!skip_config) 1082 restore_ring_config(ring_config); 1083 err_free_ring_config: 1084 if (!skip_config) 1085 ethtool_rings_get_rsp_free(ring_config); 1086 return err; 1087 } 1088 1089 int run_devmem_tests(void) 1090 { 1091 struct ethtool_rings_get_rsp *ring_config; 1092 struct netdev_queue_id *queues; 1093 struct memory_buffer *mem; 1094 struct ynl_sock *ys; 1095 int err = -1; 1096 1097 mem = provider->alloc(getpagesize() * NUM_PAGES); 1098 if (!mem) { 1099 pr_err("Failed to allocate memory buffer"); 1100 return -1; 1101 } 1102 1103 ring_config = get_ring_config(); 1104 if (!ring_config) { 1105 pr_err("Failed to get current ring configuration"); 1106 goto err_free_mem; 1107 } 1108 1109 /* Configure RSS to divert all traffic from our devmem queues */ 1110 if (configure_rss()) { 1111 pr_err("rss error"); 1112 goto err_free_ring_config; 1113 } 1114 1115 if (configure_headersplit(ring_config, 1)) { 1116 pr_err("Failed to configure header split"); 1117 goto err_reset_rss; 1118 } 1119 1120 queues = netdev_queue_id_alloc(num_queues); 1121 if (!queues) { 1122 pr_err("Failed to allocate empty queues array"); 1123 goto err_reset_headersplit; 1124 } 1125 1126 if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { 1127 pr_err("Binding empty queues array should have failed"); 1128 goto err_unbind; 1129 } 1130 1131 if (configure_headersplit(ring_config, 0)) { 1132 pr_err("Failed to configure header split"); 1133 goto err_reset_headersplit; 1134 } 1135 1136 queues = create_queues(); 1137 if (!queues) { 1138 pr_err("Failed to create queues"); 1139 goto err_reset_headersplit; 1140 } 1141 1142 if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { 1143 pr_err("Configure dmabuf with header split off should have failed"); 1144 goto err_unbind; 1145 } 1146 1147 if (configure_headersplit(ring_config, 1)) { 1148 pr_err("Failed to configure header split"); 1149 goto err_reset_headersplit; 1150 } 1151 1152 queues = create_queues(); 1153 if (!queues) { 1154 pr_err("Failed to create queues"); 1155 goto err_reset_headersplit; 1156 } 1157 1158 if (bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { 1159 pr_err("Failed to bind"); 1160 goto err_reset_headersplit; 1161 } 1162 1163 /* Deactivating a bound queue should not be legal */ 1164 if (!check_changing_channels(num_queues, num_queues)) { 1165 pr_err("Deactivating a bound queue should be illegal"); 1166 goto err_unbind; 1167 } 1168 1169 err = 0; 1170 goto err_unbind; 1171 1172 err_unbind: 1173 ynl_sock_destroy(ys); 1174 err_reset_headersplit: 1175 restore_ring_config(ring_config); 1176 err_reset_rss: 1177 reset_rss(); 1178 err_free_ring_config: 1179 ethtool_rings_get_rsp_free(ring_config); 1180 err_free_mem: 1181 provider->free(mem); 1182 return err; 1183 } 1184 1185 static uint64_t gettimeofday_ms(void) 1186 { 1187 struct timeval tv; 1188 1189 gettimeofday(&tv, NULL); 1190 return (tv.tv_sec * 1000ULL) + (tv.tv_usec / 1000ULL); 1191 } 1192 1193 static int do_poll(int fd) 1194 { 1195 struct pollfd pfd; 1196 int ret; 1197 1198 pfd.revents = 0; 1199 pfd.fd = fd; 1200 1201 ret = poll(&pfd, 1, waittime_ms); 1202 if (ret == -1) { 1203 pr_err("poll"); 1204 return -1; 1205 } 1206 1207 return ret && (pfd.revents & POLLERR); 1208 } 1209 1210 static int wait_compl(int fd) 1211 { 1212 int64_t tstop = gettimeofday_ms() + waittime_ms; 1213 char control[CMSG_SPACE(100)] = {}; 1214 struct sock_extended_err *serr; 1215 struct msghdr msg = {}; 1216 struct cmsghdr *cm; 1217 __u32 hi, lo; 1218 int ret; 1219 1220 msg.msg_control = control; 1221 msg.msg_controllen = sizeof(control); 1222 1223 while (gettimeofday_ms() < tstop) { 1224 ret = do_poll(fd); 1225 if (ret < 0) 1226 return ret; 1227 if (!ret) 1228 continue; 1229 1230 ret = recvmsg(fd, &msg, MSG_ERRQUEUE); 1231 if (ret < 0) { 1232 if (errno == EAGAIN) 1233 continue; 1234 pr_err("recvmsg(MSG_ERRQUEUE)"); 1235 return -1; 1236 } 1237 if (msg.msg_flags & MSG_CTRUNC) { 1238 pr_err("MSG_CTRUNC"); 1239 return -1; 1240 } 1241 1242 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) { 1243 if (cm->cmsg_level != SOL_IP && 1244 cm->cmsg_level != SOL_IPV6) 1245 continue; 1246 if (cm->cmsg_level == SOL_IP && 1247 cm->cmsg_type != IP_RECVERR) 1248 continue; 1249 if (cm->cmsg_level == SOL_IPV6 && 1250 cm->cmsg_type != IPV6_RECVERR) 1251 continue; 1252 1253 serr = (void *)CMSG_DATA(cm); 1254 if (serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY) { 1255 pr_err("wrong origin %u", serr->ee_origin); 1256 return -1; 1257 } 1258 if (serr->ee_errno != 0) { 1259 pr_err("wrong errno %d", serr->ee_errno); 1260 return -1; 1261 } 1262 1263 hi = serr->ee_data; 1264 lo = serr->ee_info; 1265 1266 fprintf(stderr, "tx complete [%d,%d]\n", lo, hi); 1267 return 0; 1268 } 1269 } 1270 1271 pr_err("did not receive tx completion"); 1272 return -1; 1273 } 1274 1275 static int do_client(struct memory_buffer *mem) 1276 { 1277 char ctrl_data[CMSG_SPACE(sizeof(__u32))]; 1278 struct sockaddr_in6 server_sin; 1279 struct sockaddr_in6 client_sin; 1280 struct ynl_sock *ys = NULL; 1281 struct iovec iov[MAX_IOV]; 1282 struct msghdr msg = {}; 1283 ssize_t line_size = 0; 1284 struct cmsghdr *cmsg; 1285 char *line = NULL; 1286 int ret, err = -1; 1287 size_t len = 0; 1288 int socket_fd; 1289 __u32 ddmabuf; 1290 int opt = 1; 1291 1292 ret = parse_address(server_ip, atoi(port), &server_sin); 1293 if (ret < 0) { 1294 pr_err("parse server address"); 1295 return -1; 1296 } 1297 1298 if (client_ip) { 1299 ret = parse_address(client_ip, atoi(port), &client_sin); 1300 if (ret < 0) { 1301 pr_err("parse client address"); 1302 return ret; 1303 } 1304 } 1305 1306 socket_fd = socket(AF_INET6, SOCK_STREAM, 0); 1307 if (socket_fd < 0) { 1308 pr_err("create socket"); 1309 return -1; 1310 } 1311 1312 if (enable_reuseaddr(socket_fd)) 1313 goto err_close_socket; 1314 1315 ret = setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, ifname, 1316 strlen(ifname) + 1); 1317 if (ret) { 1318 pr_err("bindtodevice"); 1319 goto err_close_socket; 1320 } 1321 1322 if (bind_tx_queue(ifindex, mem->fd, &ys)) { 1323 pr_err("Failed to bind"); 1324 goto err_close_socket; 1325 } 1326 1327 if (client_ip) { 1328 ret = bind(socket_fd, &client_sin, sizeof(client_sin)); 1329 if (ret) { 1330 pr_err("bind"); 1331 goto err_unbind; 1332 } 1333 } 1334 1335 ret = setsockopt(socket_fd, SOL_SOCKET, SO_ZEROCOPY, &opt, sizeof(opt)); 1336 if (ret) { 1337 pr_err("set sock opt"); 1338 goto err_unbind; 1339 } 1340 1341 fprintf(stderr, "Connect to %s %d (via %s)\n", server_ip, 1342 ntohs(server_sin.sin6_port), ifname); 1343 1344 ret = connect(socket_fd, &server_sin, sizeof(server_sin)); 1345 if (ret) { 1346 pr_err("connect"); 1347 goto err_unbind; 1348 } 1349 1350 while (1) { 1351 free(line); 1352 line = NULL; 1353 line_size = getline(&line, &len, stdin); 1354 1355 if (line_size < 0) 1356 break; 1357 1358 if (max_chunk) { 1359 msg.msg_iovlen = 1360 (line_size + max_chunk - 1) / max_chunk; 1361 if (msg.msg_iovlen > MAX_IOV) { 1362 pr_err("can't partition %zd bytes into maximum of %d chunks", 1363 line_size, MAX_IOV); 1364 goto err_free_line; 1365 } 1366 1367 for (int i = 0; i < msg.msg_iovlen; i++) { 1368 iov[i].iov_base = (void *)(i * max_chunk); 1369 iov[i].iov_len = max_chunk; 1370 } 1371 1372 iov[msg.msg_iovlen - 1].iov_len = 1373 line_size - (msg.msg_iovlen - 1) * max_chunk; 1374 } else { 1375 iov[0].iov_base = 0; 1376 iov[0].iov_len = line_size; 1377 msg.msg_iovlen = 1; 1378 } 1379 1380 msg.msg_iov = iov; 1381 provider->memcpy_to_device(mem, 0, line, line_size); 1382 1383 msg.msg_control = ctrl_data; 1384 msg.msg_controllen = sizeof(ctrl_data); 1385 1386 cmsg = CMSG_FIRSTHDR(&msg); 1387 cmsg->cmsg_level = SOL_SOCKET; 1388 cmsg->cmsg_type = SCM_DEVMEM_DMABUF; 1389 cmsg->cmsg_len = CMSG_LEN(sizeof(__u32)); 1390 1391 ddmabuf = tx_dmabuf_id; 1392 1393 *((__u32 *)CMSG_DATA(cmsg)) = ddmabuf; 1394 1395 ret = sendmsg(socket_fd, &msg, MSG_ZEROCOPY); 1396 if (ret < 0) { 1397 pr_err("Failed sendmsg"); 1398 goto err_free_line; 1399 } 1400 1401 fprintf(stderr, "sendmsg_ret=%d\n", ret); 1402 1403 if (ret != line_size) { 1404 pr_err("Did not send all bytes %d vs %zd", ret, line_size); 1405 goto err_free_line; 1406 } 1407 1408 if (wait_compl(socket_fd)) 1409 goto err_free_line; 1410 } 1411 1412 fprintf(stderr, "%s: tx ok\n", TEST_PREFIX); 1413 1414 err = 0; 1415 1416 err_free_line: 1417 free(line); 1418 err_unbind: 1419 ynl_sock_destroy(ys); 1420 err_close_socket: 1421 close(socket_fd); 1422 return err; 1423 } 1424 1425 int main(int argc, char *argv[]) 1426 { 1427 struct memory_buffer *mem; 1428 int is_server = 0, opt; 1429 int ret, err = 1; 1430 1431 while ((opt = getopt(argc, argv, "Lls:c:p:v:q:t:f:z:nb:")) != -1) { 1432 switch (opt) { 1433 case 'L': 1434 fail_on_linear = true; 1435 break; 1436 case 'l': 1437 is_server = 1; 1438 break; 1439 case 's': 1440 server_ip = optarg; 1441 break; 1442 case 'c': 1443 client_ip = optarg; 1444 break; 1445 case 'p': 1446 port = optarg; 1447 break; 1448 case 'v': 1449 do_validation = atoll(optarg); 1450 break; 1451 case 'q': 1452 num_queues = atoi(optarg); 1453 break; 1454 case 't': 1455 start_queue = atoi(optarg); 1456 break; 1457 case 'f': 1458 ifname = optarg; 1459 break; 1460 case 'z': 1461 max_chunk = atoi(optarg); 1462 break; 1463 case 'n': 1464 skip_config = 1; 1465 break; 1466 case 'b': { 1467 unsigned long val; 1468 1469 errno = 0; 1470 val = strtoul(optarg, NULL, 0); 1471 if ((val == ULONG_MAX && errno == ERANGE) || 1472 val > UINT32_MAX) { 1473 pr_err("invalid rx_page_size: %s", optarg); 1474 return 1; 1475 } 1476 rx_page_size = val; 1477 break; 1478 } 1479 case '?': 1480 fprintf(stderr, "unknown option: %c\n", optopt); 1481 break; 1482 } 1483 } 1484 1485 if (!ifname) { 1486 pr_err("Missing -f argument"); 1487 return 1; 1488 } 1489 1490 ifindex = if_nametoindex(ifname); 1491 1492 fprintf(stderr, "using ifindex=%u\n", ifindex); 1493 1494 if (!server_ip && !client_ip) { 1495 if (start_queue < 0 && num_queues < 0) { 1496 num_queues = rxq_num(ifindex); 1497 if (num_queues < 0) { 1498 pr_err("couldn't detect number of queues"); 1499 return 1; 1500 } 1501 if (num_queues < 2) { 1502 pr_err("number of device queues is too low"); 1503 return 1; 1504 } 1505 /* make sure can bind to multiple queues */ 1506 start_queue = num_queues / 2; 1507 num_queues /= 2; 1508 } 1509 1510 if (start_queue < 0 || num_queues < 0) { 1511 pr_err("Both -t and -q are required"); 1512 return 1; 1513 } 1514 1515 return run_devmem_tests(); 1516 } 1517 1518 if (start_queue < 0 && num_queues < 0) { 1519 num_queues = rxq_num(ifindex); 1520 if (num_queues < 2) { 1521 pr_err("number of device queues is too low"); 1522 return 1; 1523 } 1524 1525 num_queues = 1; 1526 start_queue = rxq_num(ifindex) - num_queues; 1527 1528 if (start_queue < 0) { 1529 pr_err("couldn't detect number of queues"); 1530 return 1; 1531 } 1532 1533 fprintf(stderr, "using queues %d..%d\n", start_queue, start_queue + num_queues); 1534 } 1535 1536 for (; optind < argc; optind++) 1537 fprintf(stderr, "extra arguments: %s\n", argv[optind]); 1538 1539 if (start_queue < 0) { 1540 pr_err("Missing -t argument"); 1541 return 1; 1542 } 1543 1544 if (num_queues < 0) { 1545 pr_err("Missing -q argument"); 1546 return 1; 1547 } 1548 1549 if (!server_ip) { 1550 pr_err("Missing -s argument"); 1551 return 1; 1552 } 1553 1554 if (!port) { 1555 pr_err("Missing -p argument"); 1556 return 1; 1557 } 1558 1559 mem = provider->alloc(getpagesize() * NUM_PAGES); 1560 if (!mem) { 1561 pr_err("Failed to allocate memory buffer"); 1562 return 1; 1563 } 1564 1565 ret = is_server ? do_server(mem) : do_client(mem); 1566 if (ret) 1567 goto err_free_mem; 1568 1569 err = 0; 1570 1571 err_free_mem: 1572 provider->free(mem); 1573 return err; 1574 } 1575