1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * tcpdevmem netcat. Works similarly to netcat but does device memory TCP 4 * instead of regular TCP. Uses udmabuf to mock a dmabuf provider. 5 * 6 * Usage: 7 * 8 * On server: 9 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201 10 * 11 * On client: 12 * echo -n "hello\nworld" | \ 13 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1 14 * 15 * Note this is compatible with regular netcat. i.e. the sender or receiver can 16 * be replaced with regular netcat to test the RX or TX path in isolation. 17 * 18 * Test data validation (devmem TCP on RX only): 19 * 20 * On server: 21 * ncdevmem -s <server IP> [-c <client IP>] -f eth1 -l -p 5201 -v 7 22 * 23 * On client: 24 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06) | \ 25 * head -c 1G | \ 26 * nc <server IP> 5201 -p 5201 27 * 28 * Test data validation (devmem TCP on RX and TX, validation happens on RX): 29 * 30 * On server: 31 * ncdevmem -s <server IP> [-c <client IP>] -l -p 5201 -v 8 -f eth1 32 * 33 * On client: 34 * yes $(echo -e \\x01\\x02\\x03\\x04\\x05\\x06\\x07) | \ 35 * head -c 1M | \ 36 * ncdevmem -s <server IP> [-c <client IP>] -p 5201 -f eth1 37 */ 38 #define _GNU_SOURCE 39 #define __EXPORTED_HEADERS__ 40 41 #include <linux/uio.h> 42 #include <stdarg.h> 43 #include <stdio.h> 44 #include <stdlib.h> 45 #include <unistd.h> 46 #include <stdbool.h> 47 #include <string.h> 48 #include <errno.h> 49 #define __iovec_defined 50 #include <fcntl.h> 51 #include <limits.h> 52 #include <malloc.h> 53 #include <error.h> 54 #include <poll.h> 55 56 #include <arpa/inet.h> 57 #include <sys/socket.h> 58 #include <sys/mman.h> 59 #include <sys/ioctl.h> 60 #include <sys/syscall.h> 61 #include <sys/time.h> 62 63 #include <linux/memfd.h> 64 #include <linux/dma-buf.h> 65 #include <linux/errqueue.h> 66 #include <linux/udmabuf.h> 67 #include <linux/types.h> 68 #include <linux/netlink.h> 69 #include <linux/genetlink.h> 70 #include <linux/netdev.h> 71 #include <linux/ethtool_netlink.h> 72 #include <time.h> 73 #include <net/if.h> 74 75 #include "netdev-user.h" 76 #include "ethtool-user.h" 77 #include <ynl.h> 78 79 #define PAGE_SHIFT 12 80 #define TEST_PREFIX "ncdevmem" 81 #define NUM_PAGES 16000 82 83 #ifndef MSG_SOCK_DEVMEM 84 #define MSG_SOCK_DEVMEM 0x2000000 85 #endif 86 87 #define MAX_IOV 1024 88 89 static size_t max_chunk; 90 static char *server_ip; 91 static char *client_ip; 92 static char *port; 93 static size_t do_validation; 94 static int start_queue = -1; 95 static int num_queues = -1; 96 static int skip_config; 97 static char *ifname; 98 static unsigned int ifindex; 99 static unsigned int dmabuf_id; 100 static uint32_t tx_dmabuf_id; 101 static int waittime_ms = 500; 102 static bool fail_on_linear; 103 104 /* System state loaded by current_config_load() */ 105 #define MAX_FLOWS 8 106 static int ntuple_ids[MAX_FLOWS] = { -1, -1, -1, -1, -1, -1, -1, -1, }; 107 108 struct memory_buffer { 109 int fd; 110 size_t size; 111 112 int devfd; 113 int memfd; 114 char *buf_mem; 115 }; 116 117 struct memory_provider { 118 struct memory_buffer *(*alloc)(size_t size); 119 void (*free)(struct memory_buffer *ctx); 120 void (*memcpy_to_device)(struct memory_buffer *dst, size_t off, 121 void *src, int n); 122 void (*memcpy_from_device)(void *dst, struct memory_buffer *src, 123 size_t off, int n); 124 }; 125 126 static void pr_err(const char *fmt, ...) 127 { 128 va_list args; 129 130 fprintf(stderr, "%s: ", TEST_PREFIX); 131 132 va_start(args, fmt); 133 vfprintf(stderr, fmt, args); 134 va_end(args); 135 136 if (errno != 0) 137 fprintf(stderr, ": %s", strerror(errno)); 138 fprintf(stderr, "\n"); 139 } 140 141 static struct memory_buffer *udmabuf_alloc(size_t size) 142 { 143 struct udmabuf_create create; 144 struct memory_buffer *ctx; 145 int ret; 146 147 ctx = malloc(sizeof(*ctx)); 148 if (!ctx) 149 return NULL; 150 151 ctx->size = size; 152 153 ctx->devfd = open("/dev/udmabuf", O_RDWR); 154 if (ctx->devfd < 0) { 155 pr_err("[skip,no-udmabuf: Unable to access DMA buffer device file]"); 156 goto err_free_ctx; 157 } 158 159 ctx->memfd = memfd_create("udmabuf-test", MFD_ALLOW_SEALING); 160 if (ctx->memfd < 0) { 161 pr_err("[skip,no-memfd]"); 162 goto err_close_dev; 163 } 164 165 ret = fcntl(ctx->memfd, F_ADD_SEALS, F_SEAL_SHRINK); 166 if (ret < 0) { 167 pr_err("[skip,fcntl-add-seals]"); 168 goto err_close_memfd; 169 } 170 171 ret = ftruncate(ctx->memfd, size); 172 if (ret == -1) { 173 pr_err("[FAIL,memfd-truncate]"); 174 goto err_close_memfd; 175 } 176 177 memset(&create, 0, sizeof(create)); 178 179 create.memfd = ctx->memfd; 180 create.offset = 0; 181 create.size = size; 182 ctx->fd = ioctl(ctx->devfd, UDMABUF_CREATE, &create); 183 if (ctx->fd < 0) { 184 pr_err("[FAIL, create udmabuf]"); 185 goto err_close_fd; 186 } 187 188 ctx->buf_mem = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, 189 ctx->fd, 0); 190 if (ctx->buf_mem == MAP_FAILED) { 191 pr_err("[FAIL, map udmabuf]"); 192 goto err_close_fd; 193 } 194 195 return ctx; 196 197 err_close_fd: 198 close(ctx->fd); 199 err_close_memfd: 200 close(ctx->memfd); 201 err_close_dev: 202 close(ctx->devfd); 203 err_free_ctx: 204 free(ctx); 205 return NULL; 206 } 207 208 static void udmabuf_free(struct memory_buffer *ctx) 209 { 210 munmap(ctx->buf_mem, ctx->size); 211 close(ctx->fd); 212 close(ctx->memfd); 213 close(ctx->devfd); 214 free(ctx); 215 } 216 217 static void udmabuf_memcpy_to_device(struct memory_buffer *dst, size_t off, 218 void *src, int n) 219 { 220 struct dma_buf_sync sync = {}; 221 222 sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE; 223 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync); 224 225 memcpy(dst->buf_mem + off, src, n); 226 227 sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE; 228 ioctl(dst->fd, DMA_BUF_IOCTL_SYNC, &sync); 229 } 230 231 static void udmabuf_memcpy_from_device(void *dst, struct memory_buffer *src, 232 size_t off, int n) 233 { 234 struct dma_buf_sync sync = {}; 235 236 sync.flags = DMA_BUF_SYNC_START; 237 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync); 238 239 memcpy(dst, src->buf_mem + off, n); 240 241 sync.flags = DMA_BUF_SYNC_END; 242 ioctl(src->fd, DMA_BUF_IOCTL_SYNC, &sync); 243 } 244 245 static struct memory_provider udmabuf_memory_provider = { 246 .alloc = udmabuf_alloc, 247 .free = udmabuf_free, 248 .memcpy_to_device = udmabuf_memcpy_to_device, 249 .memcpy_from_device = udmabuf_memcpy_from_device, 250 }; 251 252 static struct memory_provider *provider = &udmabuf_memory_provider; 253 254 static void print_nonzero_bytes(void *ptr, size_t size) 255 { 256 unsigned char *p = ptr; 257 unsigned int i; 258 259 for (i = 0; i < size; i++) 260 putchar(p[i]); 261 } 262 263 int validate_buffer(void *line, size_t size) 264 { 265 static unsigned char seed = 1; 266 unsigned char *ptr = line; 267 unsigned char expected; 268 static int errors; 269 size_t i; 270 271 for (i = 0; i < size; i++) { 272 expected = seed ? seed : '\n'; 273 if (ptr[i] != expected) { 274 fprintf(stderr, 275 "Failed validation: expected=%u, actual=%u, index=%lu\n", 276 expected, ptr[i], i); 277 errors++; 278 if (errors > 20) { 279 pr_err("validation failed"); 280 return -1; 281 } 282 } 283 seed++; 284 if (seed == do_validation) 285 seed = 0; 286 } 287 288 fprintf(stdout, "Validated buffer\n"); 289 return 0; 290 } 291 292 static int 293 __run_command(char *out, size_t outlen, const char *cmd, va_list args) 294 { 295 char command[256]; 296 FILE *fp; 297 298 vsnprintf(command, sizeof(command), cmd, args); 299 300 fprintf(stderr, "Running: %s\n", command); 301 fp = popen(command, "r"); 302 if (!fp) 303 return -1; 304 if (out) { 305 size_t len; 306 307 if (!fgets(out, outlen, fp)) 308 return -1; 309 310 /* Remove trailing newline if present */ 311 len = strlen(out); 312 if (len && out[len - 1] == '\n') 313 out[len - 1] = '\0'; 314 } 315 return pclose(fp); 316 } 317 318 static int run_command(const char *cmd, ...) 319 { 320 va_list args; 321 int ret; 322 323 va_start(args, cmd); 324 ret = __run_command(NULL, 0, cmd, args); 325 va_end(args); 326 327 return ret; 328 } 329 330 static int ethtool_add_flow(const char *format, ...) 331 { 332 char local_output[256], cmd[256]; 333 const char *id_start; 334 int flow_idx, ret; 335 char *endptr; 336 long flow_id; 337 va_list args; 338 339 for (flow_idx = 0; flow_idx < MAX_FLOWS; flow_idx++) 340 if (ntuple_ids[flow_idx] == -1) 341 break; 342 if (flow_idx == MAX_FLOWS) { 343 fprintf(stderr, "Error: too many flows\n"); 344 return -1; 345 } 346 347 snprintf(cmd, sizeof(cmd), "ethtool -N %s %s", ifname, format); 348 349 va_start(args, format); 350 ret = __run_command(local_output, sizeof(local_output), cmd, args); 351 va_end(args); 352 353 if (ret != 0) 354 return ret; 355 356 /* Extract the ID from the output */ 357 id_start = strstr(local_output, "Added rule with ID "); 358 if (!id_start) 359 return -1; 360 id_start += strlen("Added rule with ID "); 361 362 flow_id = strtol(id_start, &endptr, 10); 363 if (endptr == id_start || flow_id < 0 || flow_id > INT_MAX) 364 return -1; 365 366 fprintf(stderr, "Added flow rule with ID %ld\n", flow_id); 367 ntuple_ids[flow_idx] = flow_id; 368 return flow_id; 369 } 370 371 static int rxq_num(int ifindex) 372 { 373 struct ethtool_channels_get_req *req; 374 struct ethtool_channels_get_rsp *rsp; 375 struct ynl_error yerr; 376 struct ynl_sock *ys; 377 int num = -1; 378 379 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 380 if (!ys) { 381 fprintf(stderr, "YNL: %s\n", yerr.msg); 382 return -1; 383 } 384 385 req = ethtool_channels_get_req_alloc(); 386 ethtool_channels_get_req_set_header_dev_index(req, ifindex); 387 rsp = ethtool_channels_get(ys, req); 388 if (rsp) 389 num = rsp->rx_count + rsp->combined_count; 390 ethtool_channels_get_req_free(req); 391 ethtool_channels_get_rsp_free(rsp); 392 393 ynl_sock_destroy(ys); 394 395 return num; 396 } 397 398 static void reset_flow_steering(void) 399 { 400 int i; 401 402 for (i = 0; i < MAX_FLOWS; i++) { 403 if (ntuple_ids[i] == -1) 404 continue; 405 run_command("ethtool -N %s delete %d", 406 ifname, ntuple_ids[i]); 407 ntuple_ids[i] = -1; 408 } 409 } 410 411 static const char *tcp_data_split_str(int val) 412 { 413 switch (val) { 414 case 0: 415 return "off"; 416 case 1: 417 return "auto"; 418 case 2: 419 return "on"; 420 default: 421 return "?"; 422 } 423 } 424 425 static struct ethtool_rings_get_rsp *get_ring_config(void) 426 { 427 struct ethtool_rings_get_req *get_req; 428 struct ethtool_rings_get_rsp *get_rsp; 429 struct ynl_error yerr; 430 struct ynl_sock *ys; 431 432 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 433 if (!ys) { 434 fprintf(stderr, "YNL: %s\n", yerr.msg); 435 return NULL; 436 } 437 438 get_req = ethtool_rings_get_req_alloc(); 439 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex); 440 get_rsp = ethtool_rings_get(ys, get_req); 441 ethtool_rings_get_req_free(get_req); 442 443 ynl_sock_destroy(ys); 444 445 return get_rsp; 446 } 447 448 static void restore_ring_config(const struct ethtool_rings_get_rsp *config) 449 { 450 struct ethtool_rings_get_req *get_req; 451 struct ethtool_rings_get_rsp *get_rsp; 452 struct ethtool_rings_set_req *req; 453 struct ynl_error yerr; 454 struct ynl_sock *ys; 455 int ret; 456 457 if (!config) 458 return; 459 460 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 461 if (!ys) { 462 fprintf(stderr, "YNL: %s\n", yerr.msg); 463 return; 464 } 465 466 req = ethtool_rings_set_req_alloc(); 467 ethtool_rings_set_req_set_header_dev_index(req, ifindex); 468 ethtool_rings_set_req_set_tcp_data_split(req, 469 ETHTOOL_TCP_DATA_SPLIT_UNKNOWN); 470 if (config->_present.hds_thresh) 471 ethtool_rings_set_req_set_hds_thresh(req, config->hds_thresh); 472 473 ret = ethtool_rings_set(ys, req); 474 if (ret < 0) 475 fprintf(stderr, "YNL restoring HDS cfg: %s\n", ys->err.msg); 476 477 get_req = ethtool_rings_get_req_alloc(); 478 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex); 479 get_rsp = ethtool_rings_get(ys, get_req); 480 ethtool_rings_get_req_free(get_req); 481 482 /* use explicit value if UKNOWN didn't give us the previous */ 483 if (get_rsp->tcp_data_split != config->tcp_data_split) { 484 ethtool_rings_set_req_set_tcp_data_split(req, 485 config->tcp_data_split); 486 ret = ethtool_rings_set(ys, req); 487 if (ret < 0) 488 fprintf(stderr, "YNL restoring expl HDS cfg: %s\n", 489 ys->err.msg); 490 } 491 492 ethtool_rings_get_rsp_free(get_rsp); 493 ethtool_rings_set_req_free(req); 494 495 ynl_sock_destroy(ys); 496 } 497 498 static int 499 configure_headersplit(const struct ethtool_rings_get_rsp *old, bool on) 500 { 501 struct ethtool_rings_get_req *get_req; 502 struct ethtool_rings_get_rsp *get_rsp; 503 struct ethtool_rings_set_req *req; 504 struct ynl_error yerr; 505 struct ynl_sock *ys; 506 int ret; 507 508 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 509 if (!ys) { 510 fprintf(stderr, "YNL: %s\n", yerr.msg); 511 return -1; 512 } 513 514 req = ethtool_rings_set_req_alloc(); 515 ethtool_rings_set_req_set_header_dev_index(req, ifindex); 516 if (on) { 517 ethtool_rings_set_req_set_tcp_data_split(req, 518 ETHTOOL_TCP_DATA_SPLIT_ENABLED); 519 if (old->_present.hds_thresh) 520 ethtool_rings_set_req_set_hds_thresh(req, 0); 521 } else { 522 ethtool_rings_set_req_set_tcp_data_split(req, 523 ETHTOOL_TCP_DATA_SPLIT_UNKNOWN); 524 } 525 ret = ethtool_rings_set(ys, req); 526 if (ret < 0) 527 fprintf(stderr, "YNL failed: %s\n", ys->err.msg); 528 ethtool_rings_set_req_free(req); 529 530 if (ret == 0) { 531 get_req = ethtool_rings_get_req_alloc(); 532 ethtool_rings_get_req_set_header_dev_index(get_req, ifindex); 533 get_rsp = ethtool_rings_get(ys, get_req); 534 ethtool_rings_get_req_free(get_req); 535 if (get_rsp) 536 fprintf(stderr, "TCP header split: %s\n", 537 tcp_data_split_str(get_rsp->tcp_data_split)); 538 ethtool_rings_get_rsp_free(get_rsp); 539 } 540 541 ynl_sock_destroy(ys); 542 543 return ret; 544 } 545 546 static int configure_rss(void) 547 { 548 return run_command("ethtool -X %s equal %d >&2", ifname, start_queue); 549 } 550 551 static void reset_rss(void) 552 { 553 run_command("ethtool -X %s default >&2", ifname, start_queue); 554 } 555 556 static int check_changing_channels(unsigned int rx, unsigned int tx) 557 { 558 struct ethtool_channels_get_req *gchan; 559 struct ethtool_channels_set_req *schan; 560 struct ethtool_channels_get_rsp *chan; 561 struct ynl_error yerr; 562 struct ynl_sock *ys; 563 int ret; 564 565 fprintf(stderr, "setting channel count rx:%u tx:%u\n", rx, tx); 566 567 ys = ynl_sock_create(&ynl_ethtool_family, &yerr); 568 if (!ys) { 569 fprintf(stderr, "YNL: %s\n", yerr.msg); 570 return -1; 571 } 572 573 gchan = ethtool_channels_get_req_alloc(); 574 if (!gchan) { 575 ret = -1; 576 goto exit_close_sock; 577 } 578 579 ethtool_channels_get_req_set_header_dev_index(gchan, ifindex); 580 chan = ethtool_channels_get(ys, gchan); 581 ethtool_channels_get_req_free(gchan); 582 if (!chan) { 583 fprintf(stderr, "YNL get channels: %s\n", ys->err.msg); 584 ret = -1; 585 goto exit_close_sock; 586 } 587 588 schan = ethtool_channels_set_req_alloc(); 589 if (!schan) { 590 ret = -1; 591 goto exit_free_chan; 592 } 593 594 ethtool_channels_set_req_set_header_dev_index(schan, ifindex); 595 596 if (chan->_present.combined_count) { 597 if (chan->_present.rx_count || chan->_present.tx_count) { 598 ethtool_channels_set_req_set_rx_count(schan, 0); 599 ethtool_channels_set_req_set_tx_count(schan, 0); 600 } 601 602 if (rx == tx) { 603 ethtool_channels_set_req_set_combined_count(schan, rx); 604 } else if (rx > tx) { 605 ethtool_channels_set_req_set_combined_count(schan, tx); 606 ethtool_channels_set_req_set_rx_count(schan, rx - tx); 607 } else { 608 ethtool_channels_set_req_set_combined_count(schan, rx); 609 ethtool_channels_set_req_set_tx_count(schan, tx - rx); 610 } 611 612 } else if (chan->_present.rx_count) { 613 ethtool_channels_set_req_set_rx_count(schan, rx); 614 ethtool_channels_set_req_set_tx_count(schan, tx); 615 } else { 616 fprintf(stderr, "Error: device has neither combined nor rx channels\n"); 617 ret = -1; 618 goto exit_free_schan; 619 } 620 621 ret = ethtool_channels_set(ys, schan); 622 if (ret) { 623 fprintf(stderr, "YNL set channels: %s\n", ys->err.msg); 624 } else { 625 /* We were expecting a failure, go back to previous settings */ 626 ethtool_channels_set_req_set_combined_count(schan, 627 chan->combined_count); 628 ethtool_channels_set_req_set_rx_count(schan, chan->rx_count); 629 ethtool_channels_set_req_set_tx_count(schan, chan->tx_count); 630 631 ret = ethtool_channels_set(ys, schan); 632 if (ret) 633 fprintf(stderr, "YNL un-setting channels: %s\n", 634 ys->err.msg); 635 } 636 637 exit_free_schan: 638 ethtool_channels_set_req_free(schan); 639 exit_free_chan: 640 ethtool_channels_get_rsp_free(chan); 641 exit_close_sock: 642 ynl_sock_destroy(ys); 643 644 return ret; 645 } 646 647 static int configure_flow_steering(struct sockaddr_in6 *server_sin) 648 { 649 const char *type = "tcp6"; 650 const char *server_addr; 651 char buf[40]; 652 int flow_id; 653 654 inet_ntop(AF_INET6, &server_sin->sin6_addr, buf, sizeof(buf)); 655 server_addr = buf; 656 657 if (IN6_IS_ADDR_V4MAPPED(&server_sin->sin6_addr)) { 658 type = "tcp4"; 659 server_addr = strrchr(server_addr, ':') + 1; 660 } 661 662 /* Try configure 5-tuple */ 663 flow_id = ethtool_add_flow("flow-type %s %s %s dst-ip %s %s %s dst-port %s queue %d", 664 type, 665 client_ip ? "src-ip" : "", 666 client_ip ?: "", 667 server_addr, 668 client_ip ? "src-port" : "", 669 client_ip ? port : "", 670 port, start_queue); 671 if (flow_id < 0) { 672 /* If that fails, try configure 3-tuple */ 673 flow_id = ethtool_add_flow("flow-type %s dst-ip %s dst-port %s queue %d", 674 type, server_addr, port, start_queue); 675 if (flow_id < 0) 676 /* If that fails, return error */ 677 return -1; 678 } 679 680 return 0; 681 } 682 683 static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd, 684 struct netdev_queue_id *queues, 685 unsigned int n_queue_index, struct ynl_sock **ys) 686 { 687 struct netdev_bind_rx_req *req = NULL; 688 struct netdev_bind_rx_rsp *rsp = NULL; 689 struct ynl_error yerr; 690 691 *ys = ynl_sock_create(&ynl_netdev_family, &yerr); 692 if (!*ys) { 693 netdev_queue_id_free(queues); 694 fprintf(stderr, "YNL: %s\n", yerr.msg); 695 return -1; 696 } 697 698 req = netdev_bind_rx_req_alloc(); 699 netdev_bind_rx_req_set_ifindex(req, ifindex); 700 netdev_bind_rx_req_set_fd(req, dmabuf_fd); 701 __netdev_bind_rx_req_set_queues(req, queues, n_queue_index); 702 703 rsp = netdev_bind_rx(*ys, req); 704 if (!rsp) { 705 perror("netdev_bind_rx"); 706 goto err_close; 707 } 708 709 if (!rsp->_present.id) { 710 perror("id not present"); 711 goto err_close; 712 } 713 714 fprintf(stderr, "got dmabuf id=%d\n", rsp->id); 715 dmabuf_id = rsp->id; 716 717 netdev_bind_rx_req_free(req); 718 netdev_bind_rx_rsp_free(rsp); 719 720 return 0; 721 722 err_close: 723 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg); 724 netdev_bind_rx_req_free(req); 725 ynl_sock_destroy(*ys); 726 return -1; 727 } 728 729 static int bind_tx_queue(unsigned int ifindex, unsigned int dmabuf_fd, 730 struct ynl_sock **ys) 731 { 732 struct netdev_bind_tx_req *req = NULL; 733 struct netdev_bind_tx_rsp *rsp = NULL; 734 struct ynl_error yerr; 735 736 *ys = ynl_sock_create(&ynl_netdev_family, &yerr); 737 if (!*ys) { 738 fprintf(stderr, "YNL: %s\n", yerr.msg); 739 return -1; 740 } 741 742 req = netdev_bind_tx_req_alloc(); 743 netdev_bind_tx_req_set_ifindex(req, ifindex); 744 netdev_bind_tx_req_set_fd(req, dmabuf_fd); 745 746 rsp = netdev_bind_tx(*ys, req); 747 if (!rsp) { 748 perror("netdev_bind_tx"); 749 goto err_close; 750 } 751 752 if (!rsp->_present.id) { 753 perror("id not present"); 754 goto err_close; 755 } 756 757 fprintf(stderr, "got tx dmabuf id=%d\n", rsp->id); 758 tx_dmabuf_id = rsp->id; 759 760 netdev_bind_tx_req_free(req); 761 netdev_bind_tx_rsp_free(rsp); 762 763 return 0; 764 765 err_close: 766 fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg); 767 netdev_bind_tx_req_free(req); 768 ynl_sock_destroy(*ys); 769 return -1; 770 } 771 772 static int enable_reuseaddr(int fd) 773 { 774 int opt = 1; 775 int ret; 776 777 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)); 778 if (ret) { 779 pr_err("SO_REUSEPORT failed"); 780 return -1; 781 } 782 783 ret = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); 784 if (ret) { 785 pr_err("SO_REUSEADDR failed"); 786 return -1; 787 } 788 789 return 0; 790 } 791 792 static int parse_address(const char *str, int port, struct sockaddr_in6 *sin6) 793 { 794 int ret; 795 796 sin6->sin6_family = AF_INET6; 797 sin6->sin6_port = htons(port); 798 799 ret = inet_pton(sin6->sin6_family, str, &sin6->sin6_addr); 800 if (ret != 1) { 801 /* fallback to plain IPv4 */ 802 ret = inet_pton(AF_INET, str, &sin6->sin6_addr.s6_addr32[3]); 803 if (ret != 1) 804 return -1; 805 806 /* add ::ffff prefix */ 807 sin6->sin6_addr.s6_addr32[0] = 0; 808 sin6->sin6_addr.s6_addr32[1] = 0; 809 sin6->sin6_addr.s6_addr16[4] = 0; 810 sin6->sin6_addr.s6_addr16[5] = 0xffff; 811 } 812 813 return 0; 814 } 815 816 static struct netdev_queue_id *create_queues(void) 817 { 818 struct netdev_queue_id *queues; 819 size_t i = 0; 820 821 queues = netdev_queue_id_alloc(num_queues); 822 for (i = 0; i < num_queues; i++) { 823 netdev_queue_id_set_type(&queues[i], NETDEV_QUEUE_TYPE_RX); 824 netdev_queue_id_set_id(&queues[i], start_queue + i); 825 } 826 827 return queues; 828 } 829 830 static int do_server(struct memory_buffer *mem) 831 { 832 struct ethtool_rings_get_rsp *ring_config = NULL; 833 char ctrl_data[sizeof(int) * 20000]; 834 size_t non_page_aligned_frags = 0; 835 struct sockaddr_in6 client_addr; 836 struct sockaddr_in6 server_sin; 837 size_t page_aligned_frags = 0; 838 size_t total_received = 0; 839 socklen_t client_addr_len; 840 bool is_devmem = false; 841 char *tmp_mem = NULL; 842 struct ynl_sock *ys; 843 char iobuf[819200]; 844 int ret, err = -1; 845 char buffer[256]; 846 int socket_fd; 847 int client_fd; 848 849 ret = parse_address(server_ip, atoi(port), &server_sin); 850 if (ret < 0) { 851 pr_err("parse server address"); 852 return -1; 853 } 854 855 if (!skip_config) { 856 ring_config = get_ring_config(); 857 if (!ring_config) { 858 pr_err("Failed to get current ring configuration"); 859 return -1; 860 } 861 862 if (configure_headersplit(ring_config, 1)) { 863 pr_err("Failed to enable TCP header split"); 864 goto err_free_ring_config; 865 } 866 867 /* Configure RSS to divert all traffic from our devmem queues */ 868 if (configure_rss()) { 869 pr_err("Failed to configure rss"); 870 goto err_reset_headersplit; 871 } 872 873 /* Flow steer our devmem flows to start_queue */ 874 if (configure_flow_steering(&server_sin)) { 875 pr_err("Failed to configure flow steering"); 876 goto err_reset_rss; 877 } 878 } 879 880 if (bind_rx_queue(ifindex, mem->fd, create_queues(), num_queues, &ys)) { 881 pr_err("Failed to bind"); 882 goto err_reset_flow_steering; 883 } 884 885 tmp_mem = malloc(mem->size); 886 if (!tmp_mem) 887 goto err_unbind; 888 889 socket_fd = socket(AF_INET6, SOCK_STREAM, 0); 890 if (socket_fd < 0) { 891 pr_err("Failed to create socket"); 892 goto err_free_tmp; 893 } 894 895 if (enable_reuseaddr(socket_fd)) 896 goto err_close_socket; 897 898 fprintf(stderr, "binding to address %s:%d\n", server_ip, 899 ntohs(server_sin.sin6_port)); 900 901 ret = bind(socket_fd, &server_sin, sizeof(server_sin)); 902 if (ret) { 903 pr_err("Failed to bind"); 904 goto err_close_socket; 905 } 906 907 ret = listen(socket_fd, 1); 908 if (ret) { 909 pr_err("Failed to listen"); 910 goto err_close_socket; 911 } 912 913 client_addr_len = sizeof(client_addr); 914 915 inet_ntop(AF_INET6, &server_sin.sin6_addr, buffer, 916 sizeof(buffer)); 917 fprintf(stderr, "Waiting or connection on %s:%d\n", buffer, 918 ntohs(server_sin.sin6_port)); 919 client_fd = accept(socket_fd, &client_addr, &client_addr_len); 920 if (client_fd < 0) { 921 pr_err("Failed to accept"); 922 goto err_close_socket; 923 } 924 925 inet_ntop(AF_INET6, &client_addr.sin6_addr, buffer, 926 sizeof(buffer)); 927 fprintf(stderr, "Got connection from %s:%d\n", buffer, 928 ntohs(client_addr.sin6_port)); 929 930 while (1) { 931 struct iovec iov = { .iov_base = iobuf, 932 .iov_len = sizeof(iobuf) }; 933 struct dmabuf_cmsg *dmabuf_cmsg = NULL; 934 struct cmsghdr *cm = NULL; 935 struct msghdr msg = { 0 }; 936 struct dmabuf_token token; 937 ssize_t ret; 938 939 is_devmem = false; 940 941 msg.msg_iov = &iov; 942 msg.msg_iovlen = 1; 943 msg.msg_control = ctrl_data; 944 msg.msg_controllen = sizeof(ctrl_data); 945 ret = recvmsg(client_fd, &msg, MSG_SOCK_DEVMEM); 946 fprintf(stderr, "recvmsg ret=%ld\n", ret); 947 if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) 948 continue; 949 if (ret < 0) { 950 perror("recvmsg"); 951 if (errno == EFAULT) { 952 pr_err("received EFAULT, won't recover"); 953 goto err_close_client; 954 } 955 continue; 956 } 957 if (ret == 0) { 958 errno = 0; 959 pr_err("client exited"); 960 goto cleanup; 961 } 962 963 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) { 964 if (cm->cmsg_level != SOL_SOCKET || 965 (cm->cmsg_type != SCM_DEVMEM_DMABUF && 966 cm->cmsg_type != SCM_DEVMEM_LINEAR)) { 967 fprintf(stderr, "skipping non-devmem cmsg\n"); 968 continue; 969 } 970 971 dmabuf_cmsg = (struct dmabuf_cmsg *)CMSG_DATA(cm); 972 is_devmem = true; 973 974 if (cm->cmsg_type == SCM_DEVMEM_LINEAR) { 975 /* TODO: process data copied from skb's linear 976 * buffer. 977 */ 978 fprintf(stderr, 979 "SCM_DEVMEM_LINEAR. dmabuf_cmsg->frag_size=%u\n", 980 dmabuf_cmsg->frag_size); 981 982 if (fail_on_linear) { 983 pr_err("received SCM_DEVMEM_LINEAR but --fail-on-linear (-L) set"); 984 goto err_close_client; 985 } 986 987 continue; 988 } 989 990 token.token_start = dmabuf_cmsg->frag_token; 991 token.token_count = 1; 992 993 total_received += dmabuf_cmsg->frag_size; 994 fprintf(stderr, 995 "received frag_page=%llu, in_page_offset=%llu, frag_offset=%llu, frag_size=%u, token=%u, total_received=%lu, dmabuf_id=%u\n", 996 dmabuf_cmsg->frag_offset >> PAGE_SHIFT, 997 dmabuf_cmsg->frag_offset % getpagesize(), 998 dmabuf_cmsg->frag_offset, 999 dmabuf_cmsg->frag_size, dmabuf_cmsg->frag_token, 1000 total_received, dmabuf_cmsg->dmabuf_id); 1001 1002 if (dmabuf_cmsg->dmabuf_id != dmabuf_id) { 1003 pr_err("received on wrong dmabuf_id: flow steering error"); 1004 goto err_close_client; 1005 } 1006 1007 if (dmabuf_cmsg->frag_size % getpagesize()) 1008 non_page_aligned_frags++; 1009 else 1010 page_aligned_frags++; 1011 1012 provider->memcpy_from_device(tmp_mem, mem, 1013 dmabuf_cmsg->frag_offset, 1014 dmabuf_cmsg->frag_size); 1015 1016 if (do_validation) { 1017 if (validate_buffer(tmp_mem, 1018 dmabuf_cmsg->frag_size)) 1019 goto err_close_client; 1020 } else { 1021 print_nonzero_bytes(tmp_mem, 1022 dmabuf_cmsg->frag_size); 1023 } 1024 1025 ret = setsockopt(client_fd, SOL_SOCKET, 1026 SO_DEVMEM_DONTNEED, &token, 1027 sizeof(token)); 1028 if (ret != 1) { 1029 pr_err("SO_DEVMEM_DONTNEED not enough tokens"); 1030 goto err_close_client; 1031 } 1032 } 1033 if (!is_devmem) { 1034 pr_err("flow steering error"); 1035 goto err_close_client; 1036 } 1037 1038 fprintf(stderr, "total_received=%lu\n", total_received); 1039 } 1040 1041 fprintf(stderr, "%s: ok\n", TEST_PREFIX); 1042 1043 fprintf(stderr, "page_aligned_frags=%lu, non_page_aligned_frags=%lu\n", 1044 page_aligned_frags, non_page_aligned_frags); 1045 1046 cleanup: 1047 err = 0; 1048 1049 err_close_client: 1050 close(client_fd); 1051 err_close_socket: 1052 close(socket_fd); 1053 err_free_tmp: 1054 free(tmp_mem); 1055 err_unbind: 1056 ynl_sock_destroy(ys); 1057 err_reset_flow_steering: 1058 if (!skip_config) 1059 reset_flow_steering(); 1060 err_reset_rss: 1061 if (!skip_config) 1062 reset_rss(); 1063 err_reset_headersplit: 1064 if (!skip_config) 1065 restore_ring_config(ring_config); 1066 err_free_ring_config: 1067 if (!skip_config) 1068 ethtool_rings_get_rsp_free(ring_config); 1069 return err; 1070 } 1071 1072 int run_devmem_tests(void) 1073 { 1074 struct ethtool_rings_get_rsp *ring_config; 1075 struct netdev_queue_id *queues; 1076 struct memory_buffer *mem; 1077 struct ynl_sock *ys; 1078 int err = -1; 1079 1080 mem = provider->alloc(getpagesize() * NUM_PAGES); 1081 if (!mem) { 1082 pr_err("Failed to allocate memory buffer"); 1083 return -1; 1084 } 1085 1086 ring_config = get_ring_config(); 1087 if (!ring_config) { 1088 pr_err("Failed to get current ring configuration"); 1089 goto err_free_mem; 1090 } 1091 1092 /* Configure RSS to divert all traffic from our devmem queues */ 1093 if (configure_rss()) { 1094 pr_err("rss error"); 1095 goto err_free_ring_config; 1096 } 1097 1098 if (configure_headersplit(ring_config, 1)) { 1099 pr_err("Failed to configure header split"); 1100 goto err_reset_rss; 1101 } 1102 1103 queues = netdev_queue_id_alloc(num_queues); 1104 if (!queues) { 1105 pr_err("Failed to allocate empty queues array"); 1106 goto err_reset_headersplit; 1107 } 1108 1109 if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { 1110 pr_err("Binding empty queues array should have failed"); 1111 goto err_unbind; 1112 } 1113 1114 if (configure_headersplit(ring_config, 0)) { 1115 pr_err("Failed to configure header split"); 1116 goto err_reset_headersplit; 1117 } 1118 1119 queues = create_queues(); 1120 if (!queues) { 1121 pr_err("Failed to create queues"); 1122 goto err_reset_headersplit; 1123 } 1124 1125 if (!bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { 1126 pr_err("Configure dmabuf with header split off should have failed"); 1127 goto err_unbind; 1128 } 1129 1130 if (configure_headersplit(ring_config, 1)) { 1131 pr_err("Failed to configure header split"); 1132 goto err_reset_headersplit; 1133 } 1134 1135 queues = create_queues(); 1136 if (!queues) { 1137 pr_err("Failed to create queues"); 1138 goto err_reset_headersplit; 1139 } 1140 1141 if (bind_rx_queue(ifindex, mem->fd, queues, num_queues, &ys)) { 1142 pr_err("Failed to bind"); 1143 goto err_reset_headersplit; 1144 } 1145 1146 /* Deactivating a bound queue should not be legal */ 1147 if (!check_changing_channels(num_queues, num_queues)) { 1148 pr_err("Deactivating a bound queue should be illegal"); 1149 goto err_unbind; 1150 } 1151 1152 err = 0; 1153 goto err_unbind; 1154 1155 err_unbind: 1156 ynl_sock_destroy(ys); 1157 err_reset_headersplit: 1158 restore_ring_config(ring_config); 1159 err_reset_rss: 1160 reset_rss(); 1161 err_free_ring_config: 1162 ethtool_rings_get_rsp_free(ring_config); 1163 err_free_mem: 1164 provider->free(mem); 1165 return err; 1166 } 1167 1168 static uint64_t gettimeofday_ms(void) 1169 { 1170 struct timeval tv; 1171 1172 gettimeofday(&tv, NULL); 1173 return (tv.tv_sec * 1000ULL) + (tv.tv_usec / 1000ULL); 1174 } 1175 1176 static int do_poll(int fd) 1177 { 1178 struct pollfd pfd; 1179 int ret; 1180 1181 pfd.revents = 0; 1182 pfd.fd = fd; 1183 1184 ret = poll(&pfd, 1, waittime_ms); 1185 if (ret == -1) { 1186 pr_err("poll"); 1187 return -1; 1188 } 1189 1190 return ret && (pfd.revents & POLLERR); 1191 } 1192 1193 static int wait_compl(int fd) 1194 { 1195 int64_t tstop = gettimeofday_ms() + waittime_ms; 1196 char control[CMSG_SPACE(100)] = {}; 1197 struct sock_extended_err *serr; 1198 struct msghdr msg = {}; 1199 struct cmsghdr *cm; 1200 __u32 hi, lo; 1201 int ret; 1202 1203 msg.msg_control = control; 1204 msg.msg_controllen = sizeof(control); 1205 1206 while (gettimeofday_ms() < tstop) { 1207 ret = do_poll(fd); 1208 if (ret < 0) 1209 return ret; 1210 if (!ret) 1211 continue; 1212 1213 ret = recvmsg(fd, &msg, MSG_ERRQUEUE); 1214 if (ret < 0) { 1215 if (errno == EAGAIN) 1216 continue; 1217 pr_err("recvmsg(MSG_ERRQUEUE)"); 1218 return -1; 1219 } 1220 if (msg.msg_flags & MSG_CTRUNC) { 1221 pr_err("MSG_CTRUNC"); 1222 return -1; 1223 } 1224 1225 for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) { 1226 if (cm->cmsg_level != SOL_IP && 1227 cm->cmsg_level != SOL_IPV6) 1228 continue; 1229 if (cm->cmsg_level == SOL_IP && 1230 cm->cmsg_type != IP_RECVERR) 1231 continue; 1232 if (cm->cmsg_level == SOL_IPV6 && 1233 cm->cmsg_type != IPV6_RECVERR) 1234 continue; 1235 1236 serr = (void *)CMSG_DATA(cm); 1237 if (serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY) { 1238 pr_err("wrong origin %u", serr->ee_origin); 1239 return -1; 1240 } 1241 if (serr->ee_errno != 0) { 1242 pr_err("wrong errno %d", serr->ee_errno); 1243 return -1; 1244 } 1245 1246 hi = serr->ee_data; 1247 lo = serr->ee_info; 1248 1249 fprintf(stderr, "tx complete [%d,%d]\n", lo, hi); 1250 return 0; 1251 } 1252 } 1253 1254 pr_err("did not receive tx completion"); 1255 return -1; 1256 } 1257 1258 static int do_client(struct memory_buffer *mem) 1259 { 1260 char ctrl_data[CMSG_SPACE(sizeof(__u32))]; 1261 struct sockaddr_in6 server_sin; 1262 struct sockaddr_in6 client_sin; 1263 struct ynl_sock *ys = NULL; 1264 struct iovec iov[MAX_IOV]; 1265 struct msghdr msg = {}; 1266 ssize_t line_size = 0; 1267 struct cmsghdr *cmsg; 1268 char *line = NULL; 1269 int ret, err = -1; 1270 size_t len = 0; 1271 int socket_fd; 1272 __u32 ddmabuf; 1273 int opt = 1; 1274 1275 ret = parse_address(server_ip, atoi(port), &server_sin); 1276 if (ret < 0) { 1277 pr_err("parse server address"); 1278 return -1; 1279 } 1280 1281 if (client_ip) { 1282 ret = parse_address(client_ip, atoi(port), &client_sin); 1283 if (ret < 0) { 1284 pr_err("parse client address"); 1285 return ret; 1286 } 1287 } 1288 1289 socket_fd = socket(AF_INET6, SOCK_STREAM, 0); 1290 if (socket_fd < 0) { 1291 pr_err("create socket"); 1292 return -1; 1293 } 1294 1295 if (enable_reuseaddr(socket_fd)) 1296 goto err_close_socket; 1297 1298 ret = setsockopt(socket_fd, SOL_SOCKET, SO_BINDTODEVICE, ifname, 1299 strlen(ifname) + 1); 1300 if (ret) { 1301 pr_err("bindtodevice"); 1302 goto err_close_socket; 1303 } 1304 1305 if (bind_tx_queue(ifindex, mem->fd, &ys)) { 1306 pr_err("Failed to bind"); 1307 goto err_close_socket; 1308 } 1309 1310 if (client_ip) { 1311 ret = bind(socket_fd, &client_sin, sizeof(client_sin)); 1312 if (ret) { 1313 pr_err("bind"); 1314 goto err_unbind; 1315 } 1316 } 1317 1318 ret = setsockopt(socket_fd, SOL_SOCKET, SO_ZEROCOPY, &opt, sizeof(opt)); 1319 if (ret) { 1320 pr_err("set sock opt"); 1321 goto err_unbind; 1322 } 1323 1324 fprintf(stderr, "Connect to %s %d (via %s)\n", server_ip, 1325 ntohs(server_sin.sin6_port), ifname); 1326 1327 ret = connect(socket_fd, &server_sin, sizeof(server_sin)); 1328 if (ret) { 1329 pr_err("connect"); 1330 goto err_unbind; 1331 } 1332 1333 while (1) { 1334 free(line); 1335 line = NULL; 1336 line_size = getline(&line, &len, stdin); 1337 1338 if (line_size < 0) 1339 break; 1340 1341 if (max_chunk) { 1342 msg.msg_iovlen = 1343 (line_size + max_chunk - 1) / max_chunk; 1344 if (msg.msg_iovlen > MAX_IOV) { 1345 pr_err("can't partition %zd bytes into maximum of %d chunks", 1346 line_size, MAX_IOV); 1347 goto err_free_line; 1348 } 1349 1350 for (int i = 0; i < msg.msg_iovlen; i++) { 1351 iov[i].iov_base = (void *)(i * max_chunk); 1352 iov[i].iov_len = max_chunk; 1353 } 1354 1355 iov[msg.msg_iovlen - 1].iov_len = 1356 line_size - (msg.msg_iovlen - 1) * max_chunk; 1357 } else { 1358 iov[0].iov_base = 0; 1359 iov[0].iov_len = line_size; 1360 msg.msg_iovlen = 1; 1361 } 1362 1363 msg.msg_iov = iov; 1364 provider->memcpy_to_device(mem, 0, line, line_size); 1365 1366 msg.msg_control = ctrl_data; 1367 msg.msg_controllen = sizeof(ctrl_data); 1368 1369 cmsg = CMSG_FIRSTHDR(&msg); 1370 cmsg->cmsg_level = SOL_SOCKET; 1371 cmsg->cmsg_type = SCM_DEVMEM_DMABUF; 1372 cmsg->cmsg_len = CMSG_LEN(sizeof(__u32)); 1373 1374 ddmabuf = tx_dmabuf_id; 1375 1376 *((__u32 *)CMSG_DATA(cmsg)) = ddmabuf; 1377 1378 ret = sendmsg(socket_fd, &msg, MSG_ZEROCOPY); 1379 if (ret < 0) { 1380 pr_err("Failed sendmsg"); 1381 goto err_free_line; 1382 } 1383 1384 fprintf(stderr, "sendmsg_ret=%d\n", ret); 1385 1386 if (ret != line_size) { 1387 pr_err("Did not send all bytes %d vs %zd", ret, line_size); 1388 goto err_free_line; 1389 } 1390 1391 if (wait_compl(socket_fd)) 1392 goto err_free_line; 1393 } 1394 1395 fprintf(stderr, "%s: tx ok\n", TEST_PREFIX); 1396 1397 err = 0; 1398 1399 err_free_line: 1400 free(line); 1401 err_unbind: 1402 ynl_sock_destroy(ys); 1403 err_close_socket: 1404 close(socket_fd); 1405 return err; 1406 } 1407 1408 int main(int argc, char *argv[]) 1409 { 1410 struct memory_buffer *mem; 1411 int is_server = 0, opt; 1412 int ret, err = 1; 1413 1414 while ((opt = getopt(argc, argv, "Lls:c:p:v:q:t:f:z:n")) != -1) { 1415 switch (opt) { 1416 case 'L': 1417 fail_on_linear = true; 1418 break; 1419 case 'l': 1420 is_server = 1; 1421 break; 1422 case 's': 1423 server_ip = optarg; 1424 break; 1425 case 'c': 1426 client_ip = optarg; 1427 break; 1428 case 'p': 1429 port = optarg; 1430 break; 1431 case 'v': 1432 do_validation = atoll(optarg); 1433 break; 1434 case 'q': 1435 num_queues = atoi(optarg); 1436 break; 1437 case 't': 1438 start_queue = atoi(optarg); 1439 break; 1440 case 'f': 1441 ifname = optarg; 1442 break; 1443 case 'z': 1444 max_chunk = atoi(optarg); 1445 break; 1446 case 'n': 1447 skip_config = 1; 1448 break; 1449 case '?': 1450 fprintf(stderr, "unknown option: %c\n", optopt); 1451 break; 1452 } 1453 } 1454 1455 if (!ifname) { 1456 pr_err("Missing -f argument"); 1457 return 1; 1458 } 1459 1460 ifindex = if_nametoindex(ifname); 1461 1462 fprintf(stderr, "using ifindex=%u\n", ifindex); 1463 1464 if (!server_ip && !client_ip) { 1465 if (start_queue < 0 && num_queues < 0) { 1466 num_queues = rxq_num(ifindex); 1467 if (num_queues < 0) { 1468 pr_err("couldn't detect number of queues"); 1469 return 1; 1470 } 1471 if (num_queues < 2) { 1472 pr_err("number of device queues is too low"); 1473 return 1; 1474 } 1475 /* make sure can bind to multiple queues */ 1476 start_queue = num_queues / 2; 1477 num_queues /= 2; 1478 } 1479 1480 if (start_queue < 0 || num_queues < 0) { 1481 pr_err("Both -t and -q are required"); 1482 return 1; 1483 } 1484 1485 return run_devmem_tests(); 1486 } 1487 1488 if (start_queue < 0 && num_queues < 0) { 1489 num_queues = rxq_num(ifindex); 1490 if (num_queues < 2) { 1491 pr_err("number of device queues is too low"); 1492 return 1; 1493 } 1494 1495 num_queues = 1; 1496 start_queue = rxq_num(ifindex) - num_queues; 1497 1498 if (start_queue < 0) { 1499 pr_err("couldn't detect number of queues"); 1500 return 1; 1501 } 1502 1503 fprintf(stderr, "using queues %d..%d\n", start_queue, start_queue + num_queues); 1504 } 1505 1506 for (; optind < argc; optind++) 1507 fprintf(stderr, "extra arguments: %s\n", argv[optind]); 1508 1509 if (start_queue < 0) { 1510 pr_err("Missing -t argument"); 1511 return 1; 1512 } 1513 1514 if (num_queues < 0) { 1515 pr_err("Missing -q argument"); 1516 return 1; 1517 } 1518 1519 if (!server_ip) { 1520 pr_err("Missing -s argument"); 1521 return 1; 1522 } 1523 1524 if (!port) { 1525 pr_err("Missing -p argument"); 1526 return 1; 1527 } 1528 1529 mem = provider->alloc(getpagesize() * NUM_PAGES); 1530 if (!mem) { 1531 pr_err("Failed to allocate memory buffer"); 1532 return 1; 1533 } 1534 1535 ret = is_server ? do_server(mem) : do_client(mem); 1536 if (ret) 1537 goto err_free_mem; 1538 1539 err = 0; 1540 1541 err_free_mem: 1542 provider->free(mem); 1543 return err; 1544 } 1545