1 // SPDX-License-Identifier: GPL-2.0 2 /* Toeplitz test 3 * 4 * 1. Read packets and their rx_hash using PF_PACKET/TPACKET_V3 5 * 2. Compute the rx_hash in software based on the packet contents 6 * 3. Compare the two 7 * 8 * Optionally, either '-C $rx_irq_cpu_list' or '-r $rps_bitmap' may be given. 9 * 10 * If '-C $rx_irq_cpu_list' is given, also 11 * 12 * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU 13 * 5. Compute the rxqueue that RSS would select based on this rx_hash 14 * 6. Using the $rx_irq_cpu_list map, identify the arriving cpu based on rxq irq 15 * 7. Compare the cpus from 4 and 6 16 * 17 * Else if '-r $rps_bitmap' is given, also 18 * 19 * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU 20 * 5. Compute the cpu that RPS should select based on rx_hash and $rps_bitmap 21 * 6. Compare the cpus from 4 and 5 22 */ 23 24 #define _GNU_SOURCE 25 26 #include <arpa/inet.h> 27 #include <errno.h> 28 #include <error.h> 29 #include <fcntl.h> 30 #include <getopt.h> 31 #include <linux/filter.h> 32 #include <linux/if_ether.h> 33 #include <linux/if_packet.h> 34 #include <net/if.h> 35 #include <netdb.h> 36 #include <netinet/ip.h> 37 #include <netinet/ip6.h> 38 #include <netinet/tcp.h> 39 #include <netinet/udp.h> 40 #include <poll.h> 41 #include <stdbool.h> 42 #include <stddef.h> 43 #include <stdint.h> 44 #include <stdio.h> 45 #include <stdlib.h> 46 #include <string.h> 47 #include <sys/mman.h> 48 #include <sys/socket.h> 49 #include <sys/stat.h> 50 #include <sys/sysinfo.h> 51 #include <sys/time.h> 52 #include <sys/types.h> 53 #include <unistd.h> 54 55 #include <ynl.h> 56 #include "ethtool-user.h" 57 58 #include "../../../kselftest.h" 59 #include "../../../net/lib/ksft.h" 60 61 #define TOEPLITZ_KEY_MIN_LEN 40 62 #define TOEPLITZ_KEY_MAX_LEN 60 63 64 #define TOEPLITZ_STR_LEN(K) (((K) * 3) - 1) /* hex encoded: AA:BB:CC:...:ZZ */ 65 #define TOEPLITZ_STR_MIN_LEN TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MIN_LEN) 66 #define TOEPLITZ_STR_MAX_LEN TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MAX_LEN) 67 68 #define FOUR_TUPLE_MAX_LEN ((sizeof(struct in6_addr) * 2) + (sizeof(uint16_t) * 2)) 69 70 #define RSS_MAX_CPUS (1 << 16) /* real constraint is PACKET_FANOUT_MAX */ 71 #define RSS_MAX_INDIR (1 << 16) 72 73 #define RPS_MAX_CPUS 16UL /* must be a power of 2 */ 74 75 /* configuration options (cmdline arguments) */ 76 static uint16_t cfg_dport = 8000; 77 static int cfg_family = AF_INET6; 78 static char *cfg_ifname = "eth0"; 79 static int cfg_num_queues; 80 static int cfg_num_rps_cpus; 81 static bool cfg_sink; 82 static int cfg_type = SOCK_STREAM; 83 static int cfg_timeout_msec = 1000; 84 static bool cfg_verbose; 85 86 /* global vars */ 87 static int num_cpus; 88 static int ring_block_nr; 89 static int ring_block_sz; 90 91 /* stats */ 92 static int frames_received; 93 static int frames_nohash; 94 static int frames_error; 95 96 #define log_verbose(args...) do { if (cfg_verbose) fprintf(stderr, args); } while (0) 97 98 /* tpacket ring */ 99 struct ring_state { 100 int fd; 101 char *mmap; 102 int idx; 103 int cpu; 104 }; 105 106 static unsigned int rx_irq_cpus[RSS_MAX_CPUS]; /* map from rxq to cpu */ 107 static int rps_silo_to_cpu[RPS_MAX_CPUS]; 108 static unsigned char toeplitz_key[TOEPLITZ_KEY_MAX_LEN]; 109 static unsigned int rss_indir_tbl[RSS_MAX_INDIR]; 110 static unsigned int rss_indir_tbl_size; 111 static struct ring_state rings[RSS_MAX_CPUS]; 112 113 static inline uint32_t toeplitz(const unsigned char *four_tuple, 114 const unsigned char *key) 115 { 116 int i, bit, ret = 0; 117 uint32_t key32; 118 119 key32 = ntohl(*((uint32_t *)key)); 120 key += 4; 121 122 for (i = 0; i < FOUR_TUPLE_MAX_LEN; i++) { 123 for (bit = 7; bit >= 0; bit--) { 124 if (four_tuple[i] & (1 << bit)) 125 ret ^= key32; 126 127 key32 <<= 1; 128 key32 |= !!(key[0] & (1 << bit)); 129 } 130 key++; 131 } 132 133 return ret; 134 } 135 136 /* Compare computed cpu with arrival cpu from packet_fanout_cpu */ 137 static void verify_rss(uint32_t rx_hash, int cpu) 138 { 139 int queue; 140 141 if (rss_indir_tbl_size) 142 queue = rss_indir_tbl[rx_hash % rss_indir_tbl_size]; 143 else 144 queue = rx_hash % cfg_num_queues; 145 146 log_verbose(" rxq %d (cpu %d)", queue, rx_irq_cpus[queue]); 147 if (rx_irq_cpus[queue] != cpu) { 148 log_verbose(". error: rss cpu mismatch (%d)", cpu); 149 frames_error++; 150 } 151 } 152 153 static void verify_rps(uint64_t rx_hash, int cpu) 154 { 155 int silo = (rx_hash * cfg_num_rps_cpus) >> 32; 156 157 log_verbose(" silo %d (cpu %d)", silo, rps_silo_to_cpu[silo]); 158 if (rps_silo_to_cpu[silo] != cpu) { 159 log_verbose(". error: rps cpu mismatch (%d)", cpu); 160 frames_error++; 161 } 162 } 163 164 static void log_rxhash(int cpu, uint32_t rx_hash, 165 const char *addrs, int addr_len) 166 { 167 char saddr[INET6_ADDRSTRLEN], daddr[INET6_ADDRSTRLEN]; 168 uint16_t *ports; 169 170 if (!inet_ntop(cfg_family, addrs, saddr, sizeof(saddr)) || 171 !inet_ntop(cfg_family, addrs + addr_len, daddr, sizeof(daddr))) 172 error(1, 0, "address parse error"); 173 174 ports = (void *)addrs + (addr_len * 2); 175 log_verbose("cpu %d: rx_hash 0x%08x [saddr %s daddr %s sport %02hu dport %02hu]", 176 cpu, rx_hash, saddr, daddr, 177 ntohs(ports[0]), ntohs(ports[1])); 178 } 179 180 /* Compare computed rxhash with rxhash received from tpacket_v3 */ 181 static void verify_rxhash(const char *pkt, uint32_t rx_hash, int cpu) 182 { 183 unsigned char four_tuple[FOUR_TUPLE_MAX_LEN] = {0}; 184 uint32_t rx_hash_sw; 185 const char *addrs; 186 int addr_len; 187 188 if (cfg_family == AF_INET) { 189 addr_len = sizeof(struct in_addr); 190 addrs = pkt + offsetof(struct iphdr, saddr); 191 } else { 192 addr_len = sizeof(struct in6_addr); 193 addrs = pkt + offsetof(struct ip6_hdr, ip6_src); 194 } 195 196 memcpy(four_tuple, addrs, (addr_len * 2) + (sizeof(uint16_t) * 2)); 197 rx_hash_sw = toeplitz(four_tuple, toeplitz_key); 198 199 if (cfg_verbose) 200 log_rxhash(cpu, rx_hash, addrs, addr_len); 201 202 if (rx_hash != rx_hash_sw) { 203 log_verbose(" != expected 0x%x\n", rx_hash_sw); 204 frames_error++; 205 return; 206 } 207 208 log_verbose(" OK"); 209 if (cfg_num_queues) 210 verify_rss(rx_hash, cpu); 211 else if (cfg_num_rps_cpus) 212 verify_rps(rx_hash, cpu); 213 log_verbose("\n"); 214 } 215 216 static char *recv_frame(const struct ring_state *ring, char *frame) 217 { 218 struct tpacket3_hdr *hdr = (void *)frame; 219 220 if (hdr->hv1.tp_rxhash) 221 verify_rxhash(frame + hdr->tp_net, hdr->hv1.tp_rxhash, 222 ring->cpu); 223 else 224 frames_nohash++; 225 226 return frame + hdr->tp_next_offset; 227 } 228 229 /* A single TPACKET_V3 block can hold multiple frames */ 230 static bool recv_block(struct ring_state *ring) 231 { 232 struct tpacket_block_desc *block; 233 char *frame; 234 int i; 235 236 block = (void *)(ring->mmap + ring->idx * ring_block_sz); 237 if (!(block->hdr.bh1.block_status & TP_STATUS_USER)) 238 return false; 239 240 frame = (char *)block; 241 frame += block->hdr.bh1.offset_to_first_pkt; 242 243 for (i = 0; i < block->hdr.bh1.num_pkts; i++) { 244 frame = recv_frame(ring, frame); 245 frames_received++; 246 } 247 248 block->hdr.bh1.block_status = TP_STATUS_KERNEL; 249 ring->idx = (ring->idx + 1) % ring_block_nr; 250 251 return true; 252 } 253 254 /* simple test: sleep once unconditionally and then process all rings */ 255 static void process_rings(void) 256 { 257 int i; 258 259 usleep(1000 * cfg_timeout_msec); 260 261 for (i = 0; i < num_cpus; i++) 262 do {} while (recv_block(&rings[i])); 263 264 fprintf(stderr, "count: pass=%u nohash=%u fail=%u\n", 265 frames_received - frames_nohash - frames_error, 266 frames_nohash, frames_error); 267 } 268 269 static char *setup_ring(int fd) 270 { 271 struct tpacket_req3 req3 = {0}; 272 void *ring; 273 274 req3.tp_retire_blk_tov = cfg_timeout_msec / 8; 275 req3.tp_feature_req_word = TP_FT_REQ_FILL_RXHASH; 276 277 req3.tp_frame_size = 2048; 278 req3.tp_frame_nr = 1 << 10; 279 req3.tp_block_nr = 16; 280 281 req3.tp_block_size = req3.tp_frame_size * req3.tp_frame_nr; 282 req3.tp_block_size /= req3.tp_block_nr; 283 284 if (setsockopt(fd, SOL_PACKET, PACKET_RX_RING, &req3, sizeof(req3))) 285 error(1, errno, "setsockopt PACKET_RX_RING"); 286 287 ring_block_sz = req3.tp_block_size; 288 ring_block_nr = req3.tp_block_nr; 289 290 ring = mmap(0, req3.tp_block_size * req3.tp_block_nr, 291 PROT_READ | PROT_WRITE, 292 MAP_SHARED | MAP_LOCKED | MAP_POPULATE, fd, 0); 293 if (ring == MAP_FAILED) 294 error(1, 0, "mmap failed"); 295 296 return ring; 297 } 298 299 static void __set_filter(int fd, int off_proto, uint8_t proto, int off_dport) 300 { 301 struct sock_filter filter[] = { 302 BPF_STMT(BPF_LD + BPF_B + BPF_ABS, SKF_AD_OFF + SKF_AD_PKTTYPE), 303 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, PACKET_HOST, 0, 4), 304 BPF_STMT(BPF_LD + BPF_B + BPF_ABS, off_proto), 305 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, proto, 0, 2), 306 BPF_STMT(BPF_LD + BPF_H + BPF_ABS, off_dport), 307 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, cfg_dport, 1, 0), 308 BPF_STMT(BPF_RET + BPF_K, 0), 309 BPF_STMT(BPF_RET + BPF_K, 0xFFFF), 310 }; 311 struct sock_fprog prog = {}; 312 313 prog.filter = filter; 314 prog.len = ARRAY_SIZE(filter); 315 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog))) 316 error(1, errno, "setsockopt filter"); 317 } 318 319 /* filter on transport protocol and destination port */ 320 static void set_filter(int fd) 321 { 322 const int off_dport = offsetof(struct tcphdr, dest); /* same for udp */ 323 uint8_t proto; 324 325 proto = cfg_type == SOCK_STREAM ? IPPROTO_TCP : IPPROTO_UDP; 326 if (cfg_family == AF_INET) 327 __set_filter(fd, offsetof(struct iphdr, protocol), proto, 328 sizeof(struct iphdr) + off_dport); 329 else 330 __set_filter(fd, offsetof(struct ip6_hdr, ip6_nxt), proto, 331 sizeof(struct ip6_hdr) + off_dport); 332 } 333 334 /* drop everything: used temporarily during setup */ 335 static void set_filter_null(int fd) 336 { 337 struct sock_filter filter[] = { 338 BPF_STMT(BPF_RET + BPF_K, 0), 339 }; 340 struct sock_fprog prog = {}; 341 342 prog.filter = filter; 343 prog.len = ARRAY_SIZE(filter); 344 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog))) 345 error(1, errno, "setsockopt filter"); 346 } 347 348 static int create_ring(char **ring) 349 { 350 struct fanout_args args = { 351 .id = 1, 352 .type_flags = PACKET_FANOUT_CPU, 353 .max_num_members = RSS_MAX_CPUS 354 }; 355 struct sockaddr_ll ll = { 0 }; 356 int fd, val; 357 358 fd = socket(PF_PACKET, SOCK_DGRAM, 0); 359 if (fd == -1) 360 error(1, errno, "socket creation failed"); 361 362 val = TPACKET_V3; 363 if (setsockopt(fd, SOL_PACKET, PACKET_VERSION, &val, sizeof(val))) 364 error(1, errno, "setsockopt PACKET_VERSION"); 365 *ring = setup_ring(fd); 366 367 /* block packets until all rings are added to the fanout group: 368 * else packets can arrive during setup and get misclassified 369 */ 370 set_filter_null(fd); 371 372 ll.sll_family = AF_PACKET; 373 ll.sll_ifindex = if_nametoindex(cfg_ifname); 374 ll.sll_protocol = cfg_family == AF_INET ? htons(ETH_P_IP) : 375 htons(ETH_P_IPV6); 376 if (bind(fd, (void *)&ll, sizeof(ll))) 377 error(1, errno, "bind"); 378 379 /* must come after bind: verifies all programs in group match */ 380 if (setsockopt(fd, SOL_PACKET, PACKET_FANOUT, &args, sizeof(args))) { 381 /* on failure, retry using old API if that is sufficient: 382 * it has a hard limit of 256 sockets, so only try if 383 * (a) only testing rxhash, not RSS or (b) <= 256 cpus. 384 * in this API, the third argument is left implicit. 385 */ 386 if (cfg_num_queues || num_cpus > 256 || 387 setsockopt(fd, SOL_PACKET, PACKET_FANOUT, 388 &args, sizeof(uint32_t))) 389 error(1, errno, "setsockopt PACKET_FANOUT cpu"); 390 } 391 392 return fd; 393 } 394 395 /* setup inet(6) socket to blackhole the test traffic, if arg '-s' */ 396 static int setup_sink(void) 397 { 398 int fd, val; 399 400 fd = socket(cfg_family, cfg_type, 0); 401 if (fd == -1) 402 error(1, errno, "socket %d.%d", cfg_family, cfg_type); 403 404 val = 1 << 20; 405 if (setsockopt(fd, SOL_SOCKET, SO_RCVBUFFORCE, &val, sizeof(val))) 406 error(1, errno, "setsockopt rcvbuf"); 407 408 return fd; 409 } 410 411 static void setup_rings(void) 412 { 413 int i; 414 415 for (i = 0; i < num_cpus; i++) { 416 rings[i].cpu = i; 417 rings[i].fd = create_ring(&rings[i].mmap); 418 } 419 420 /* accept packets once all rings in the fanout group are up */ 421 for (i = 0; i < num_cpus; i++) 422 set_filter(rings[i].fd); 423 } 424 425 static void cleanup_rings(void) 426 { 427 int i; 428 429 for (i = 0; i < num_cpus; i++) { 430 if (munmap(rings[i].mmap, ring_block_nr * ring_block_sz)) 431 error(1, errno, "munmap"); 432 if (close(rings[i].fd)) 433 error(1, errno, "close"); 434 } 435 } 436 437 static void parse_cpulist(const char *arg) 438 { 439 do { 440 rx_irq_cpus[cfg_num_queues++] = strtol(arg, NULL, 10); 441 442 arg = strchr(arg, ','); 443 if (!arg) 444 break; 445 arg++; // skip ',' 446 } while (1); 447 } 448 449 static void show_cpulist(void) 450 { 451 int i; 452 453 for (i = 0; i < cfg_num_queues; i++) 454 fprintf(stderr, "rxq %d: cpu %d\n", i, rx_irq_cpus[i]); 455 } 456 457 static void show_silos(void) 458 { 459 int i; 460 461 for (i = 0; i < cfg_num_rps_cpus; i++) 462 fprintf(stderr, "silo %d: cpu %d\n", i, rps_silo_to_cpu[i]); 463 } 464 465 static void parse_toeplitz_key(const char *str, int slen, unsigned char *key) 466 { 467 int i, ret, off; 468 469 if (slen < TOEPLITZ_STR_MIN_LEN || 470 slen > TOEPLITZ_STR_MAX_LEN + 1) 471 error(1, 0, "invalid toeplitz key"); 472 473 for (i = 0, off = 0; off < slen; i++, off += 3) { 474 ret = sscanf(str + off, "%hhx", &key[i]); 475 if (ret != 1) 476 error(1, 0, "key parse error at %d off %d len %d", 477 i, off, slen); 478 } 479 } 480 481 static void parse_rps_bitmap(const char *arg) 482 { 483 unsigned long bitmap; 484 int i; 485 486 bitmap = strtoul(arg, NULL, 0); 487 488 if (bitmap & ~(RPS_MAX_CPUS - 1)) 489 error(1, 0, "rps bitmap 0x%lx out of bounds 0..%lu", 490 bitmap, RPS_MAX_CPUS - 1); 491 492 for (i = 0; i < RPS_MAX_CPUS; i++) 493 if (bitmap & 1UL << i) 494 rps_silo_to_cpu[cfg_num_rps_cpus++] = i; 495 } 496 497 static void read_rss_dev_info_ynl(void) 498 { 499 struct ethtool_rss_get_req *req; 500 struct ethtool_rss_get_rsp *rsp; 501 struct ynl_sock *ys; 502 503 ys = ynl_sock_create(&ynl_ethtool_family, NULL); 504 if (!ys) 505 error(1, errno, "ynl_sock_create failed"); 506 507 req = ethtool_rss_get_req_alloc(); 508 if (!req) 509 error(1, errno, "ethtool_rss_get_req_alloc failed"); 510 511 ethtool_rss_get_req_set_header_dev_name(req, cfg_ifname); 512 513 rsp = ethtool_rss_get(ys, req); 514 if (!rsp) 515 error(1, ys->err.code, "YNL: %s", ys->err.msg); 516 517 if (!rsp->_len.hkey) 518 error(1, 0, "RSS key not available for %s", cfg_ifname); 519 520 if (rsp->_len.hkey < TOEPLITZ_KEY_MIN_LEN || 521 rsp->_len.hkey > TOEPLITZ_KEY_MAX_LEN) 522 error(1, 0, "RSS key length %u out of bounds [%u, %u]", 523 rsp->_len.hkey, TOEPLITZ_KEY_MIN_LEN, 524 TOEPLITZ_KEY_MAX_LEN); 525 526 memcpy(toeplitz_key, rsp->hkey, rsp->_len.hkey); 527 528 if (rsp->_count.indir > RSS_MAX_INDIR) 529 error(1, 0, "RSS indirection table too large (%u > %u)", 530 rsp->_count.indir, RSS_MAX_INDIR); 531 532 /* If indir table not available we'll fallback to simple modulo math */ 533 if (rsp->_count.indir) { 534 memcpy(rss_indir_tbl, rsp->indir, 535 rsp->_count.indir * sizeof(rss_indir_tbl[0])); 536 rss_indir_tbl_size = rsp->_count.indir; 537 538 log_verbose("RSS indirection table size: %u\n", 539 rss_indir_tbl_size); 540 } 541 542 ethtool_rss_get_rsp_free(rsp); 543 ethtool_rss_get_req_free(req); 544 ynl_sock_destroy(ys); 545 } 546 547 static void parse_opts(int argc, char **argv) 548 { 549 static struct option long_options[] = { 550 {"dport", required_argument, 0, 'd'}, 551 {"cpus", required_argument, 0, 'C'}, 552 {"key", required_argument, 0, 'k'}, 553 {"iface", required_argument, 0, 'i'}, 554 {"ipv4", no_argument, 0, '4'}, 555 {"ipv6", no_argument, 0, '6'}, 556 {"sink", no_argument, 0, 's'}, 557 {"tcp", no_argument, 0, 't'}, 558 {"timeout", required_argument, 0, 'T'}, 559 {"udp", no_argument, 0, 'u'}, 560 {"verbose", no_argument, 0, 'v'}, 561 {"rps", required_argument, 0, 'r'}, 562 {0, 0, 0, 0} 563 }; 564 bool have_toeplitz = false; 565 int index, c; 566 567 while ((c = getopt_long(argc, argv, "46C:d:i:k:r:stT:uv", long_options, &index)) != -1) { 568 switch (c) { 569 case '4': 570 cfg_family = AF_INET; 571 break; 572 case '6': 573 cfg_family = AF_INET6; 574 break; 575 case 'C': 576 parse_cpulist(optarg); 577 break; 578 case 'd': 579 cfg_dport = strtol(optarg, NULL, 0); 580 break; 581 case 'i': 582 cfg_ifname = optarg; 583 break; 584 case 'k': 585 parse_toeplitz_key(optarg, strlen(optarg), 586 toeplitz_key); 587 have_toeplitz = true; 588 break; 589 case 'r': 590 parse_rps_bitmap(optarg); 591 break; 592 case 's': 593 cfg_sink = true; 594 break; 595 case 't': 596 cfg_type = SOCK_STREAM; 597 break; 598 case 'T': 599 cfg_timeout_msec = strtol(optarg, NULL, 0); 600 break; 601 case 'u': 602 cfg_type = SOCK_DGRAM; 603 break; 604 case 'v': 605 cfg_verbose = true; 606 break; 607 608 default: 609 error(1, 0, "unknown option %c", optopt); 610 break; 611 } 612 } 613 614 if (!have_toeplitz) 615 read_rss_dev_info_ynl(); 616 617 num_cpus = get_nprocs(); 618 if (num_cpus > RSS_MAX_CPUS) 619 error(1, 0, "increase RSS_MAX_CPUS"); 620 621 if (cfg_num_queues && cfg_num_rps_cpus) 622 error(1, 0, 623 "Can't supply both RSS cpus ('-C') and RPS map ('-r')"); 624 if (cfg_verbose) { 625 show_cpulist(); 626 show_silos(); 627 } 628 } 629 630 int main(int argc, char **argv) 631 { 632 const int min_tests = 10; 633 int fd_sink = -1; 634 635 parse_opts(argc, argv); 636 637 if (cfg_sink) 638 fd_sink = setup_sink(); 639 640 setup_rings(); 641 642 /* Signal to test framework that we're ready to receive */ 643 ksft_ready(); 644 645 process_rings(); 646 cleanup_rings(); 647 648 if (cfg_sink && close(fd_sink)) 649 error(1, errno, "close sink"); 650 651 if (frames_received - frames_nohash < min_tests) 652 error(1, 0, "too few frames for verification"); 653 654 return frames_error; 655 } 656