1 // SPDX-License-Identifier: GPL-2.0 2 /* Toeplitz test 3 * 4 * 1. Read packets and their rx_hash using PF_PACKET/TPACKET_V3 5 * 2. Compute the rx_hash in software based on the packet contents 6 * 3. Compare the two 7 * 8 * Optionally, either '-C $rx_irq_cpu_list' or '-r $rps_bitmap' may be given. 9 * 10 * If '-C $rx_irq_cpu_list' is given, also 11 * 12 * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU 13 * 5. Compute the rxqueue that RSS would select based on this rx_hash 14 * 6. Using the $rx_irq_cpu_list map, identify the arriving cpu based on rxq irq 15 * 7. Compare the cpus from 4 and 6 16 * 17 * Else if '-r $rps_bitmap' is given, also 18 * 19 * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU 20 * 5. Compute the cpu that RPS should select based on rx_hash and $rps_bitmap 21 * 6. Compare the cpus from 4 and 5 22 */ 23 24 #define _GNU_SOURCE 25 26 #include <arpa/inet.h> 27 #include <errno.h> 28 #include <error.h> 29 #include <fcntl.h> 30 #include <getopt.h> 31 #include <linux/filter.h> 32 #include <linux/if_ether.h> 33 #include <linux/if_packet.h> 34 #include <net/if.h> 35 #include <netdb.h> 36 #include <netinet/ip.h> 37 #include <netinet/ip6.h> 38 #include <netinet/tcp.h> 39 #include <netinet/udp.h> 40 #include <poll.h> 41 #include <stdbool.h> 42 #include <stddef.h> 43 #include <stdint.h> 44 #include <stdio.h> 45 #include <stdlib.h> 46 #include <string.h> 47 #include <sys/mman.h> 48 #include <sys/socket.h> 49 #include <sys/stat.h> 50 #include <sys/sysinfo.h> 51 #include <sys/time.h> 52 #include <sys/types.h> 53 #include <unistd.h> 54 55 #include <ynl.h> 56 #include "ethtool-user.h" 57 58 #include "kselftest.h" 59 #include "../../../net/lib/ksft.h" 60 61 #define TOEPLITZ_KEY_MIN_LEN 40 62 #define TOEPLITZ_KEY_MAX_LEN 256 63 64 #define TOEPLITZ_STR_LEN(K) (((K) * 3) - 1) /* hex encoded: AA:BB:CC:...:ZZ */ 65 #define TOEPLITZ_STR_MIN_LEN TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MIN_LEN) 66 #define TOEPLITZ_STR_MAX_LEN TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MAX_LEN) 67 68 #define FOUR_TUPLE_MAX_LEN ((sizeof(struct in6_addr) * 2) + (sizeof(uint16_t) * 2)) 69 70 #define RSS_MAX_CPUS (1 << 16) /* real constraint is PACKET_FANOUT_MAX */ 71 #define RSS_MAX_INDIR (1 << 16) 72 73 #define RPS_MAX_CPUS 16UL /* must be a power of 2 */ 74 75 #define MIN_PKT_SAMPLES 40 /* minimum number of packets to receive */ 76 77 /* configuration options (cmdline arguments) */ 78 static uint16_t cfg_dport = 8000; 79 static int cfg_family = AF_INET6; 80 static char *cfg_ifname = "eth0"; 81 static int cfg_num_queues; 82 static int cfg_num_rps_cpus; 83 static bool cfg_sink; 84 static int cfg_type = SOCK_STREAM; 85 static int cfg_timeout_msec = 1000; 86 static bool cfg_verbose; 87 88 /* global vars */ 89 static int num_cpus; 90 static int ring_block_nr; 91 static int ring_block_sz; 92 93 /* stats */ 94 static int frames_received; 95 static int frames_nohash; 96 static int frames_error; 97 98 #define log_verbose(args...) do { if (cfg_verbose) fprintf(stderr, args); } while (0) 99 100 /* tpacket ring */ 101 struct ring_state { 102 int fd; 103 char *mmap; 104 int idx; 105 int cpu; 106 }; 107 108 static unsigned int rx_irq_cpus[RSS_MAX_CPUS]; /* map from rxq to cpu */ 109 static int rps_silo_to_cpu[RPS_MAX_CPUS]; 110 static unsigned char toeplitz_key[TOEPLITZ_KEY_MAX_LEN]; 111 static unsigned int rss_indir_tbl[RSS_MAX_INDIR]; 112 static unsigned int rss_indir_tbl_size; 113 static struct ring_state rings[RSS_MAX_CPUS]; 114 115 static inline uint32_t toeplitz(const unsigned char *four_tuple, 116 const unsigned char *key) 117 { 118 int i, bit, ret = 0; 119 uint32_t key32; 120 121 key32 = ntohl(*((uint32_t *)key)); 122 key += 4; 123 124 for (i = 0; i < FOUR_TUPLE_MAX_LEN; i++) { 125 for (bit = 7; bit >= 0; bit--) { 126 if (four_tuple[i] & (1 << bit)) 127 ret ^= key32; 128 129 key32 <<= 1; 130 key32 |= !!(key[0] & (1 << bit)); 131 } 132 key++; 133 } 134 135 return ret; 136 } 137 138 /* Compare computed cpu with arrival cpu from packet_fanout_cpu */ 139 static void verify_rss(uint32_t rx_hash, int cpu) 140 { 141 int queue; 142 143 if (rss_indir_tbl_size) 144 queue = rss_indir_tbl[rx_hash % rss_indir_tbl_size]; 145 else 146 queue = rx_hash % cfg_num_queues; 147 148 log_verbose(" rxq %d (cpu %d)", queue, rx_irq_cpus[queue]); 149 if (rx_irq_cpus[queue] != cpu) { 150 log_verbose(". error: rss cpu mismatch (%d)", cpu); 151 frames_error++; 152 } 153 } 154 155 static void verify_rps(uint64_t rx_hash, int cpu) 156 { 157 int silo = (rx_hash * cfg_num_rps_cpus) >> 32; 158 159 log_verbose(" silo %d (cpu %d)", silo, rps_silo_to_cpu[silo]); 160 if (rps_silo_to_cpu[silo] != cpu) { 161 log_verbose(". error: rps cpu mismatch (%d)", cpu); 162 frames_error++; 163 } 164 } 165 166 static void log_rxhash(int cpu, uint32_t rx_hash, 167 const char *addrs, int addr_len) 168 { 169 char saddr[INET6_ADDRSTRLEN], daddr[INET6_ADDRSTRLEN]; 170 uint16_t *ports; 171 172 if (!inet_ntop(cfg_family, addrs, saddr, sizeof(saddr)) || 173 !inet_ntop(cfg_family, addrs + addr_len, daddr, sizeof(daddr))) 174 error(1, 0, "address parse error"); 175 176 ports = (void *)addrs + (addr_len * 2); 177 log_verbose("cpu %d: rx_hash 0x%08x [saddr %s daddr %s sport %02hu dport %02hu]", 178 cpu, rx_hash, saddr, daddr, 179 ntohs(ports[0]), ntohs(ports[1])); 180 } 181 182 /* Compare computed rxhash with rxhash received from tpacket_v3 */ 183 static void verify_rxhash(const char *pkt, uint32_t rx_hash, int cpu) 184 { 185 unsigned char four_tuple[FOUR_TUPLE_MAX_LEN] = {0}; 186 uint32_t rx_hash_sw; 187 const char *addrs; 188 int addr_len; 189 190 if (cfg_family == AF_INET) { 191 addr_len = sizeof(struct in_addr); 192 addrs = pkt + offsetof(struct iphdr, saddr); 193 } else { 194 addr_len = sizeof(struct in6_addr); 195 addrs = pkt + offsetof(struct ip6_hdr, ip6_src); 196 } 197 198 memcpy(four_tuple, addrs, (addr_len * 2) + (sizeof(uint16_t) * 2)); 199 rx_hash_sw = toeplitz(four_tuple, toeplitz_key); 200 201 if (cfg_verbose) 202 log_rxhash(cpu, rx_hash, addrs, addr_len); 203 204 if (rx_hash != rx_hash_sw) { 205 log_verbose(" != expected 0x%x\n", rx_hash_sw); 206 frames_error++; 207 return; 208 } 209 210 log_verbose(" OK"); 211 if (cfg_num_queues) 212 verify_rss(rx_hash, cpu); 213 else if (cfg_num_rps_cpus) 214 verify_rps(rx_hash, cpu); 215 log_verbose("\n"); 216 } 217 218 static char *recv_frame(const struct ring_state *ring, char *frame) 219 { 220 struct tpacket3_hdr *hdr = (void *)frame; 221 222 if (hdr->hv1.tp_rxhash) 223 verify_rxhash(frame + hdr->tp_net, hdr->hv1.tp_rxhash, 224 ring->cpu); 225 else 226 frames_nohash++; 227 228 return frame + hdr->tp_next_offset; 229 } 230 231 /* A single TPACKET_V3 block can hold multiple frames */ 232 static bool recv_block(struct ring_state *ring) 233 { 234 struct tpacket_block_desc *block; 235 char *frame; 236 int i; 237 238 block = (void *)(ring->mmap + ring->idx * ring_block_sz); 239 if (!(block->hdr.bh1.block_status & TP_STATUS_USER)) 240 return false; 241 242 frame = (char *)block; 243 frame += block->hdr.bh1.offset_to_first_pkt; 244 245 for (i = 0; i < block->hdr.bh1.num_pkts; i++) { 246 frame = recv_frame(ring, frame); 247 frames_received++; 248 } 249 250 block->hdr.bh1.block_status = TP_STATUS_KERNEL; 251 ring->idx = (ring->idx + 1) % ring_block_nr; 252 253 return true; 254 } 255 256 /* simple test: process all rings until MIN_PKT_SAMPLES packets are received, 257 * or the test times out. 258 */ 259 static void process_rings(void) 260 { 261 struct timeval start, now; 262 bool pkts_found = true; 263 long elapsed_usec; 264 int i; 265 266 gettimeofday(&start, NULL); 267 268 do { 269 if (!pkts_found) 270 usleep(100); 271 272 pkts_found = false; 273 for (i = 0; i < num_cpus; i++) 274 pkts_found |= recv_block(&rings[i]); 275 276 gettimeofday(&now, NULL); 277 elapsed_usec = (now.tv_sec - start.tv_sec) * 1000000 + 278 (now.tv_usec - start.tv_usec); 279 } while (frames_received - frames_nohash < MIN_PKT_SAMPLES && 280 elapsed_usec < cfg_timeout_msec * 1000); 281 282 fprintf(stderr, "count: pass=%u nohash=%u fail=%u\n", 283 frames_received - frames_nohash - frames_error, 284 frames_nohash, frames_error); 285 } 286 287 static char *setup_ring(int fd) 288 { 289 struct tpacket_req3 req3 = {0}; 290 void *ring; 291 292 req3.tp_retire_blk_tov = cfg_timeout_msec / 8; 293 req3.tp_feature_req_word = TP_FT_REQ_FILL_RXHASH; 294 295 req3.tp_frame_size = 2048; 296 req3.tp_frame_nr = 1 << 10; 297 req3.tp_block_nr = 16; 298 299 req3.tp_block_size = req3.tp_frame_size * req3.tp_frame_nr; 300 req3.tp_block_size /= req3.tp_block_nr; 301 302 if (setsockopt(fd, SOL_PACKET, PACKET_RX_RING, &req3, sizeof(req3))) 303 error(1, errno, "setsockopt PACKET_RX_RING"); 304 305 ring_block_sz = req3.tp_block_size; 306 ring_block_nr = req3.tp_block_nr; 307 308 ring = mmap(0, req3.tp_block_size * req3.tp_block_nr, 309 PROT_READ | PROT_WRITE, 310 MAP_SHARED | MAP_LOCKED | MAP_POPULATE, fd, 0); 311 if (ring == MAP_FAILED) 312 error(1, 0, "mmap failed"); 313 314 return ring; 315 } 316 317 static void __set_filter(int fd, int off_proto, uint8_t proto, int off_dport) 318 { 319 struct sock_filter filter[] = { 320 BPF_STMT(BPF_LD + BPF_B + BPF_ABS, SKF_AD_OFF + SKF_AD_PKTTYPE), 321 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, PACKET_HOST, 0, 4), 322 BPF_STMT(BPF_LD + BPF_B + BPF_ABS, off_proto), 323 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, proto, 0, 2), 324 BPF_STMT(BPF_LD + BPF_H + BPF_ABS, off_dport), 325 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, cfg_dport, 1, 0), 326 BPF_STMT(BPF_RET + BPF_K, 0), 327 BPF_STMT(BPF_RET + BPF_K, 0xFFFF), 328 }; 329 struct sock_fprog prog = {}; 330 331 prog.filter = filter; 332 prog.len = ARRAY_SIZE(filter); 333 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog))) 334 error(1, errno, "setsockopt filter"); 335 } 336 337 /* filter on transport protocol and destination port */ 338 static void set_filter(int fd) 339 { 340 const int off_dport = offsetof(struct tcphdr, dest); /* same for udp */ 341 uint8_t proto; 342 343 proto = cfg_type == SOCK_STREAM ? IPPROTO_TCP : IPPROTO_UDP; 344 if (cfg_family == AF_INET) 345 __set_filter(fd, offsetof(struct iphdr, protocol), proto, 346 sizeof(struct iphdr) + off_dport); 347 else 348 __set_filter(fd, offsetof(struct ip6_hdr, ip6_nxt), proto, 349 sizeof(struct ip6_hdr) + off_dport); 350 } 351 352 /* drop everything: used temporarily during setup */ 353 static void set_filter_null(int fd) 354 { 355 struct sock_filter filter[] = { 356 BPF_STMT(BPF_RET + BPF_K, 0), 357 }; 358 struct sock_fprog prog = {}; 359 360 prog.filter = filter; 361 prog.len = ARRAY_SIZE(filter); 362 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog))) 363 error(1, errno, "setsockopt filter"); 364 } 365 366 static int create_ring(char **ring) 367 { 368 struct fanout_args args = { 369 .id = 1, 370 .type_flags = PACKET_FANOUT_CPU, 371 .max_num_members = RSS_MAX_CPUS 372 }; 373 struct sockaddr_ll ll = { 0 }; 374 int fd, val; 375 376 fd = socket(PF_PACKET, SOCK_DGRAM, 0); 377 if (fd == -1) 378 error(1, errno, "socket creation failed"); 379 380 val = TPACKET_V3; 381 if (setsockopt(fd, SOL_PACKET, PACKET_VERSION, &val, sizeof(val))) 382 error(1, errno, "setsockopt PACKET_VERSION"); 383 *ring = setup_ring(fd); 384 385 /* block packets until all rings are added to the fanout group: 386 * else packets can arrive during setup and get misclassified 387 */ 388 set_filter_null(fd); 389 390 ll.sll_family = AF_PACKET; 391 ll.sll_ifindex = if_nametoindex(cfg_ifname); 392 ll.sll_protocol = cfg_family == AF_INET ? htons(ETH_P_IP) : 393 htons(ETH_P_IPV6); 394 if (bind(fd, (void *)&ll, sizeof(ll))) 395 error(1, errno, "bind"); 396 397 /* must come after bind: verifies all programs in group match */ 398 if (setsockopt(fd, SOL_PACKET, PACKET_FANOUT, &args, sizeof(args))) { 399 /* on failure, retry using old API if that is sufficient: 400 * it has a hard limit of 256 sockets, so only try if 401 * (a) only testing rxhash, not RSS or (b) <= 256 cpus. 402 * in this API, the third argument is left implicit. 403 */ 404 if (cfg_num_queues || num_cpus > 256 || 405 setsockopt(fd, SOL_PACKET, PACKET_FANOUT, 406 &args, sizeof(uint32_t))) 407 error(1, errno, "setsockopt PACKET_FANOUT cpu"); 408 } 409 410 return fd; 411 } 412 413 /* setup inet(6) socket to blackhole the test traffic, if arg '-s' */ 414 static int setup_sink(void) 415 { 416 int fd, val; 417 418 fd = socket(cfg_family, cfg_type, 0); 419 if (fd == -1) 420 error(1, errno, "socket %d.%d", cfg_family, cfg_type); 421 422 val = 1 << 20; 423 if (setsockopt(fd, SOL_SOCKET, SO_RCVBUFFORCE, &val, sizeof(val))) 424 error(1, errno, "setsockopt rcvbuf"); 425 426 return fd; 427 } 428 429 static void setup_rings(void) 430 { 431 int i; 432 433 for (i = 0; i < num_cpus; i++) { 434 rings[i].cpu = i; 435 rings[i].fd = create_ring(&rings[i].mmap); 436 } 437 438 /* accept packets once all rings in the fanout group are up */ 439 for (i = 0; i < num_cpus; i++) 440 set_filter(rings[i].fd); 441 } 442 443 static void cleanup_rings(void) 444 { 445 int i; 446 447 for (i = 0; i < num_cpus; i++) { 448 if (munmap(rings[i].mmap, ring_block_nr * ring_block_sz)) 449 error(1, errno, "munmap"); 450 if (close(rings[i].fd)) 451 error(1, errno, "close"); 452 } 453 } 454 455 static void parse_cpulist(const char *arg) 456 { 457 do { 458 rx_irq_cpus[cfg_num_queues++] = strtol(arg, NULL, 10); 459 460 arg = strchr(arg, ','); 461 if (!arg) 462 break; 463 arg++; // skip ',' 464 } while (1); 465 } 466 467 static void show_cpulist(void) 468 { 469 int i; 470 471 for (i = 0; i < cfg_num_queues; i++) 472 fprintf(stderr, "rxq %d: cpu %d\n", i, rx_irq_cpus[i]); 473 } 474 475 static void show_silos(void) 476 { 477 int i; 478 479 for (i = 0; i < cfg_num_rps_cpus; i++) 480 fprintf(stderr, "silo %d: cpu %d\n", i, rps_silo_to_cpu[i]); 481 } 482 483 static void parse_toeplitz_key(const char *str, int slen, unsigned char *key) 484 { 485 int i, ret, off; 486 487 if (slen < TOEPLITZ_STR_MIN_LEN || 488 slen > TOEPLITZ_STR_MAX_LEN + 1) 489 error(1, 0, "invalid toeplitz key"); 490 491 for (i = 0, off = 0; off < slen; i++, off += 3) { 492 ret = sscanf(str + off, "%hhx", &key[i]); 493 if (ret != 1) 494 error(1, 0, "key parse error at %d off %d len %d", 495 i, off, slen); 496 } 497 } 498 499 static void parse_rps_bitmap(const char *arg) 500 { 501 unsigned long bitmap; 502 int i; 503 504 bitmap = strtoul(arg, NULL, 0); 505 506 if (bitmap & ~((1UL << RPS_MAX_CPUS) - 1)) 507 error(1, 0, "rps bitmap 0x%lx out of bounds, max cpu %lu", 508 bitmap, RPS_MAX_CPUS - 1); 509 510 for (i = 0; i < RPS_MAX_CPUS; i++) 511 if (bitmap & 1UL << i) 512 rps_silo_to_cpu[cfg_num_rps_cpus++] = i; 513 } 514 515 static void read_rss_dev_info_ynl(void) 516 { 517 struct ethtool_rss_get_req *req; 518 struct ethtool_rss_get_rsp *rsp; 519 struct ynl_sock *ys; 520 521 ys = ynl_sock_create(&ynl_ethtool_family, NULL); 522 if (!ys) 523 error(1, errno, "ynl_sock_create failed"); 524 525 req = ethtool_rss_get_req_alloc(); 526 if (!req) 527 error(1, errno, "ethtool_rss_get_req_alloc failed"); 528 529 ethtool_rss_get_req_set_header_dev_name(req, cfg_ifname); 530 531 rsp = ethtool_rss_get(ys, req); 532 if (!rsp) 533 error(1, ys->err.code, "YNL: %s", ys->err.msg); 534 535 if (!rsp->_len.hkey) 536 error(1, 0, "RSS key not available for %s", cfg_ifname); 537 538 if (rsp->_len.hkey < TOEPLITZ_KEY_MIN_LEN || 539 rsp->_len.hkey > TOEPLITZ_KEY_MAX_LEN) 540 error(1, 0, "RSS key length %u out of bounds [%u, %u]", 541 rsp->_len.hkey, TOEPLITZ_KEY_MIN_LEN, 542 TOEPLITZ_KEY_MAX_LEN); 543 544 memcpy(toeplitz_key, rsp->hkey, rsp->_len.hkey); 545 546 if (rsp->_count.indir > RSS_MAX_INDIR) 547 error(1, 0, "RSS indirection table too large (%u > %u)", 548 rsp->_count.indir, RSS_MAX_INDIR); 549 550 /* If indir table not available we'll fallback to simple modulo math */ 551 if (rsp->_count.indir) { 552 memcpy(rss_indir_tbl, rsp->indir, 553 rsp->_count.indir * sizeof(rss_indir_tbl[0])); 554 rss_indir_tbl_size = rsp->_count.indir; 555 556 log_verbose("RSS indirection table size: %u\n", 557 rss_indir_tbl_size); 558 } 559 560 ethtool_rss_get_rsp_free(rsp); 561 ethtool_rss_get_req_free(req); 562 ynl_sock_destroy(ys); 563 } 564 565 static void parse_opts(int argc, char **argv) 566 { 567 static struct option long_options[] = { 568 {"dport", required_argument, 0, 'd'}, 569 {"cpus", required_argument, 0, 'C'}, 570 {"key", required_argument, 0, 'k'}, 571 {"iface", required_argument, 0, 'i'}, 572 {"ipv4", no_argument, 0, '4'}, 573 {"ipv6", no_argument, 0, '6'}, 574 {"sink", no_argument, 0, 's'}, 575 {"tcp", no_argument, 0, 't'}, 576 {"timeout", required_argument, 0, 'T'}, 577 {"udp", no_argument, 0, 'u'}, 578 {"verbose", no_argument, 0, 'v'}, 579 {"rps", required_argument, 0, 'r'}, 580 {0, 0, 0, 0} 581 }; 582 bool have_toeplitz = false; 583 int index, c; 584 585 while ((c = getopt_long(argc, argv, "46C:d:i:k:r:stT:uv", long_options, &index)) != -1) { 586 switch (c) { 587 case '4': 588 cfg_family = AF_INET; 589 break; 590 case '6': 591 cfg_family = AF_INET6; 592 break; 593 case 'C': 594 parse_cpulist(optarg); 595 break; 596 case 'd': 597 cfg_dport = strtol(optarg, NULL, 0); 598 break; 599 case 'i': 600 cfg_ifname = optarg; 601 break; 602 case 'k': 603 parse_toeplitz_key(optarg, strlen(optarg), 604 toeplitz_key); 605 have_toeplitz = true; 606 break; 607 case 'r': 608 parse_rps_bitmap(optarg); 609 break; 610 case 's': 611 cfg_sink = true; 612 break; 613 case 't': 614 cfg_type = SOCK_STREAM; 615 break; 616 case 'T': 617 cfg_timeout_msec = strtol(optarg, NULL, 0); 618 break; 619 case 'u': 620 cfg_type = SOCK_DGRAM; 621 break; 622 case 'v': 623 cfg_verbose = true; 624 break; 625 626 default: 627 error(1, 0, "unknown option %c", optopt); 628 break; 629 } 630 } 631 632 if (!have_toeplitz) 633 read_rss_dev_info_ynl(); 634 635 num_cpus = get_nprocs(); 636 if (num_cpus > RSS_MAX_CPUS) 637 error(1, 0, "increase RSS_MAX_CPUS"); 638 639 if (cfg_num_queues && cfg_num_rps_cpus) 640 error(1, 0, 641 "Can't supply both RSS cpus ('-C') and RPS map ('-r')"); 642 if (cfg_verbose) { 643 show_cpulist(); 644 show_silos(); 645 } 646 } 647 648 int main(int argc, char **argv) 649 { 650 const int min_tests = 10; 651 int fd_sink = -1; 652 653 parse_opts(argc, argv); 654 655 if (cfg_sink) 656 fd_sink = setup_sink(); 657 658 setup_rings(); 659 660 /* Signal to test framework that we're ready to receive */ 661 ksft_ready(); 662 663 process_rings(); 664 cleanup_rings(); 665 666 if (cfg_sink && close(fd_sink)) 667 error(1, errno, "close sink"); 668 669 if (frames_received - frames_nohash < min_tests) 670 error(1, 0, "too few frames for verification"); 671 672 return frames_error; 673 } 674