1 // SPDX-License-Identifier: GPL-2.0
2 /* Toeplitz test
3 *
4 * 1. Read packets and their rx_hash using PF_PACKET/TPACKET_V3
5 * 2. Compute the rx_hash in software based on the packet contents
6 * 3. Compare the two
7 *
8 * Optionally, either '-C $rx_irq_cpu_list' or '-r $rps_bitmap' may be given.
9 *
10 * If '-C $rx_irq_cpu_list' is given, also
11 *
12 * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU
13 * 5. Compute the rxqueue that RSS would select based on this rx_hash
14 * 6. Using the $rx_irq_cpu_list map, identify the arriving cpu based on rxq irq
15 * 7. Compare the cpus from 4 and 6
16 *
17 * Else if '-r $rps_bitmap' is given, also
18 *
19 * 4. Identify the cpu on which the packet arrived with PACKET_FANOUT_CPU
20 * 5. Compute the cpu that RPS should select based on rx_hash and $rps_bitmap
21 * 6. Compare the cpus from 4 and 5
22 */
23
24 #define _GNU_SOURCE
25
26 #include <arpa/inet.h>
27 #include <errno.h>
28 #include <error.h>
29 #include <fcntl.h>
30 #include <getopt.h>
31 #include <linux/filter.h>
32 #include <linux/if_ether.h>
33 #include <linux/if_packet.h>
34 #include <net/if.h>
35 #include <netdb.h>
36 #include <netinet/ip.h>
37 #include <netinet/ip6.h>
38 #include <netinet/tcp.h>
39 #include <netinet/udp.h>
40 #include <poll.h>
41 #include <stdbool.h>
42 #include <stddef.h>
43 #include <stdint.h>
44 #include <stdio.h>
45 #include <stdlib.h>
46 #include <string.h>
47 #include <sys/mman.h>
48 #include <sys/socket.h>
49 #include <sys/stat.h>
50 #include <sys/sysinfo.h>
51 #include <sys/time.h>
52 #include <sys/types.h>
53 #include <unistd.h>
54
55 #include <ynl.h>
56 #include "ethtool-user.h"
57
58 #include "kselftest.h"
59 #include "../../../net/lib/ksft.h"
60
61 #define TOEPLITZ_KEY_MIN_LEN 40
62 #define TOEPLITZ_KEY_MAX_LEN 60
63
64 #define TOEPLITZ_STR_LEN(K) (((K) * 3) - 1) /* hex encoded: AA:BB:CC:...:ZZ */
65 #define TOEPLITZ_STR_MIN_LEN TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MIN_LEN)
66 #define TOEPLITZ_STR_MAX_LEN TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MAX_LEN)
67
68 #define FOUR_TUPLE_MAX_LEN ((sizeof(struct in6_addr) * 2) + (sizeof(uint16_t) * 2))
69
70 #define RSS_MAX_CPUS (1 << 16) /* real constraint is PACKET_FANOUT_MAX */
71 #define RSS_MAX_INDIR (1 << 16)
72
73 #define RPS_MAX_CPUS 16UL /* must be a power of 2 */
74
75 /* configuration options (cmdline arguments) */
76 static uint16_t cfg_dport = 8000;
77 static int cfg_family = AF_INET6;
78 static char *cfg_ifname = "eth0";
79 static int cfg_num_queues;
80 static int cfg_num_rps_cpus;
81 static bool cfg_sink;
82 static int cfg_type = SOCK_STREAM;
83 static int cfg_timeout_msec = 1000;
84 static bool cfg_verbose;
85
86 /* global vars */
87 static int num_cpus;
88 static int ring_block_nr;
89 static int ring_block_sz;
90
91 /* stats */
92 static int frames_received;
93 static int frames_nohash;
94 static int frames_error;
95
96 #define log_verbose(args...) do { if (cfg_verbose) fprintf(stderr, args); } while (0)
97
98 /* tpacket ring */
99 struct ring_state {
100 int fd;
101 char *mmap;
102 int idx;
103 int cpu;
104 };
105
106 static unsigned int rx_irq_cpus[RSS_MAX_CPUS]; /* map from rxq to cpu */
107 static int rps_silo_to_cpu[RPS_MAX_CPUS];
108 static unsigned char toeplitz_key[TOEPLITZ_KEY_MAX_LEN];
109 static unsigned int rss_indir_tbl[RSS_MAX_INDIR];
110 static unsigned int rss_indir_tbl_size;
111 static struct ring_state rings[RSS_MAX_CPUS];
112
toeplitz(const unsigned char * four_tuple,const unsigned char * key)113 static inline uint32_t toeplitz(const unsigned char *four_tuple,
114 const unsigned char *key)
115 {
116 int i, bit, ret = 0;
117 uint32_t key32;
118
119 key32 = ntohl(*((uint32_t *)key));
120 key += 4;
121
122 for (i = 0; i < FOUR_TUPLE_MAX_LEN; i++) {
123 for (bit = 7; bit >= 0; bit--) {
124 if (four_tuple[i] & (1 << bit))
125 ret ^= key32;
126
127 key32 <<= 1;
128 key32 |= !!(key[0] & (1 << bit));
129 }
130 key++;
131 }
132
133 return ret;
134 }
135
136 /* Compare computed cpu with arrival cpu from packet_fanout_cpu */
verify_rss(uint32_t rx_hash,int cpu)137 static void verify_rss(uint32_t rx_hash, int cpu)
138 {
139 int queue;
140
141 if (rss_indir_tbl_size)
142 queue = rss_indir_tbl[rx_hash % rss_indir_tbl_size];
143 else
144 queue = rx_hash % cfg_num_queues;
145
146 log_verbose(" rxq %d (cpu %d)", queue, rx_irq_cpus[queue]);
147 if (rx_irq_cpus[queue] != cpu) {
148 log_verbose(". error: rss cpu mismatch (%d)", cpu);
149 frames_error++;
150 }
151 }
152
verify_rps(uint64_t rx_hash,int cpu)153 static void verify_rps(uint64_t rx_hash, int cpu)
154 {
155 int silo = (rx_hash * cfg_num_rps_cpus) >> 32;
156
157 log_verbose(" silo %d (cpu %d)", silo, rps_silo_to_cpu[silo]);
158 if (rps_silo_to_cpu[silo] != cpu) {
159 log_verbose(". error: rps cpu mismatch (%d)", cpu);
160 frames_error++;
161 }
162 }
163
log_rxhash(int cpu,uint32_t rx_hash,const char * addrs,int addr_len)164 static void log_rxhash(int cpu, uint32_t rx_hash,
165 const char *addrs, int addr_len)
166 {
167 char saddr[INET6_ADDRSTRLEN], daddr[INET6_ADDRSTRLEN];
168 uint16_t *ports;
169
170 if (!inet_ntop(cfg_family, addrs, saddr, sizeof(saddr)) ||
171 !inet_ntop(cfg_family, addrs + addr_len, daddr, sizeof(daddr)))
172 error(1, 0, "address parse error");
173
174 ports = (void *)addrs + (addr_len * 2);
175 log_verbose("cpu %d: rx_hash 0x%08x [saddr %s daddr %s sport %02hu dport %02hu]",
176 cpu, rx_hash, saddr, daddr,
177 ntohs(ports[0]), ntohs(ports[1]));
178 }
179
180 /* Compare computed rxhash with rxhash received from tpacket_v3 */
verify_rxhash(const char * pkt,uint32_t rx_hash,int cpu)181 static void verify_rxhash(const char *pkt, uint32_t rx_hash, int cpu)
182 {
183 unsigned char four_tuple[FOUR_TUPLE_MAX_LEN] = {0};
184 uint32_t rx_hash_sw;
185 const char *addrs;
186 int addr_len;
187
188 if (cfg_family == AF_INET) {
189 addr_len = sizeof(struct in_addr);
190 addrs = pkt + offsetof(struct iphdr, saddr);
191 } else {
192 addr_len = sizeof(struct in6_addr);
193 addrs = pkt + offsetof(struct ip6_hdr, ip6_src);
194 }
195
196 memcpy(four_tuple, addrs, (addr_len * 2) + (sizeof(uint16_t) * 2));
197 rx_hash_sw = toeplitz(four_tuple, toeplitz_key);
198
199 if (cfg_verbose)
200 log_rxhash(cpu, rx_hash, addrs, addr_len);
201
202 if (rx_hash != rx_hash_sw) {
203 log_verbose(" != expected 0x%x\n", rx_hash_sw);
204 frames_error++;
205 return;
206 }
207
208 log_verbose(" OK");
209 if (cfg_num_queues)
210 verify_rss(rx_hash, cpu);
211 else if (cfg_num_rps_cpus)
212 verify_rps(rx_hash, cpu);
213 log_verbose("\n");
214 }
215
recv_frame(const struct ring_state * ring,char * frame)216 static char *recv_frame(const struct ring_state *ring, char *frame)
217 {
218 struct tpacket3_hdr *hdr = (void *)frame;
219
220 if (hdr->hv1.tp_rxhash)
221 verify_rxhash(frame + hdr->tp_net, hdr->hv1.tp_rxhash,
222 ring->cpu);
223 else
224 frames_nohash++;
225
226 return frame + hdr->tp_next_offset;
227 }
228
229 /* A single TPACKET_V3 block can hold multiple frames */
recv_block(struct ring_state * ring)230 static bool recv_block(struct ring_state *ring)
231 {
232 struct tpacket_block_desc *block;
233 char *frame;
234 int i;
235
236 block = (void *)(ring->mmap + ring->idx * ring_block_sz);
237 if (!(block->hdr.bh1.block_status & TP_STATUS_USER))
238 return false;
239
240 frame = (char *)block;
241 frame += block->hdr.bh1.offset_to_first_pkt;
242
243 for (i = 0; i < block->hdr.bh1.num_pkts; i++) {
244 frame = recv_frame(ring, frame);
245 frames_received++;
246 }
247
248 block->hdr.bh1.block_status = TP_STATUS_KERNEL;
249 ring->idx = (ring->idx + 1) % ring_block_nr;
250
251 return true;
252 }
253
254 /* simple test: sleep once unconditionally and then process all rings */
process_rings(void)255 static void process_rings(void)
256 {
257 int i;
258
259 usleep(1000 * cfg_timeout_msec);
260
261 for (i = 0; i < num_cpus; i++)
262 do {} while (recv_block(&rings[i]));
263
264 fprintf(stderr, "count: pass=%u nohash=%u fail=%u\n",
265 frames_received - frames_nohash - frames_error,
266 frames_nohash, frames_error);
267 }
268
setup_ring(int fd)269 static char *setup_ring(int fd)
270 {
271 struct tpacket_req3 req3 = {0};
272 void *ring;
273
274 req3.tp_retire_blk_tov = cfg_timeout_msec / 8;
275 req3.tp_feature_req_word = TP_FT_REQ_FILL_RXHASH;
276
277 req3.tp_frame_size = 2048;
278 req3.tp_frame_nr = 1 << 10;
279 req3.tp_block_nr = 16;
280
281 req3.tp_block_size = req3.tp_frame_size * req3.tp_frame_nr;
282 req3.tp_block_size /= req3.tp_block_nr;
283
284 if (setsockopt(fd, SOL_PACKET, PACKET_RX_RING, &req3, sizeof(req3)))
285 error(1, errno, "setsockopt PACKET_RX_RING");
286
287 ring_block_sz = req3.tp_block_size;
288 ring_block_nr = req3.tp_block_nr;
289
290 ring = mmap(0, req3.tp_block_size * req3.tp_block_nr,
291 PROT_READ | PROT_WRITE,
292 MAP_SHARED | MAP_LOCKED | MAP_POPULATE, fd, 0);
293 if (ring == MAP_FAILED)
294 error(1, 0, "mmap failed");
295
296 return ring;
297 }
298
__set_filter(int fd,int off_proto,uint8_t proto,int off_dport)299 static void __set_filter(int fd, int off_proto, uint8_t proto, int off_dport)
300 {
301 struct sock_filter filter[] = {
302 BPF_STMT(BPF_LD + BPF_B + BPF_ABS, SKF_AD_OFF + SKF_AD_PKTTYPE),
303 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, PACKET_HOST, 0, 4),
304 BPF_STMT(BPF_LD + BPF_B + BPF_ABS, off_proto),
305 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, proto, 0, 2),
306 BPF_STMT(BPF_LD + BPF_H + BPF_ABS, off_dport),
307 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, cfg_dport, 1, 0),
308 BPF_STMT(BPF_RET + BPF_K, 0),
309 BPF_STMT(BPF_RET + BPF_K, 0xFFFF),
310 };
311 struct sock_fprog prog = {};
312
313 prog.filter = filter;
314 prog.len = ARRAY_SIZE(filter);
315 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog)))
316 error(1, errno, "setsockopt filter");
317 }
318
319 /* filter on transport protocol and destination port */
set_filter(int fd)320 static void set_filter(int fd)
321 {
322 const int off_dport = offsetof(struct tcphdr, dest); /* same for udp */
323 uint8_t proto;
324
325 proto = cfg_type == SOCK_STREAM ? IPPROTO_TCP : IPPROTO_UDP;
326 if (cfg_family == AF_INET)
327 __set_filter(fd, offsetof(struct iphdr, protocol), proto,
328 sizeof(struct iphdr) + off_dport);
329 else
330 __set_filter(fd, offsetof(struct ip6_hdr, ip6_nxt), proto,
331 sizeof(struct ip6_hdr) + off_dport);
332 }
333
334 /* drop everything: used temporarily during setup */
set_filter_null(int fd)335 static void set_filter_null(int fd)
336 {
337 struct sock_filter filter[] = {
338 BPF_STMT(BPF_RET + BPF_K, 0),
339 };
340 struct sock_fprog prog = {};
341
342 prog.filter = filter;
343 prog.len = ARRAY_SIZE(filter);
344 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog)))
345 error(1, errno, "setsockopt filter");
346 }
347
create_ring(char ** ring)348 static int create_ring(char **ring)
349 {
350 struct fanout_args args = {
351 .id = 1,
352 .type_flags = PACKET_FANOUT_CPU,
353 .max_num_members = RSS_MAX_CPUS
354 };
355 struct sockaddr_ll ll = { 0 };
356 int fd, val;
357
358 fd = socket(PF_PACKET, SOCK_DGRAM, 0);
359 if (fd == -1)
360 error(1, errno, "socket creation failed");
361
362 val = TPACKET_V3;
363 if (setsockopt(fd, SOL_PACKET, PACKET_VERSION, &val, sizeof(val)))
364 error(1, errno, "setsockopt PACKET_VERSION");
365 *ring = setup_ring(fd);
366
367 /* block packets until all rings are added to the fanout group:
368 * else packets can arrive during setup and get misclassified
369 */
370 set_filter_null(fd);
371
372 ll.sll_family = AF_PACKET;
373 ll.sll_ifindex = if_nametoindex(cfg_ifname);
374 ll.sll_protocol = cfg_family == AF_INET ? htons(ETH_P_IP) :
375 htons(ETH_P_IPV6);
376 if (bind(fd, (void *)&ll, sizeof(ll)))
377 error(1, errno, "bind");
378
379 /* must come after bind: verifies all programs in group match */
380 if (setsockopt(fd, SOL_PACKET, PACKET_FANOUT, &args, sizeof(args))) {
381 /* on failure, retry using old API if that is sufficient:
382 * it has a hard limit of 256 sockets, so only try if
383 * (a) only testing rxhash, not RSS or (b) <= 256 cpus.
384 * in this API, the third argument is left implicit.
385 */
386 if (cfg_num_queues || num_cpus > 256 ||
387 setsockopt(fd, SOL_PACKET, PACKET_FANOUT,
388 &args, sizeof(uint32_t)))
389 error(1, errno, "setsockopt PACKET_FANOUT cpu");
390 }
391
392 return fd;
393 }
394
395 /* setup inet(6) socket to blackhole the test traffic, if arg '-s' */
setup_sink(void)396 static int setup_sink(void)
397 {
398 int fd, val;
399
400 fd = socket(cfg_family, cfg_type, 0);
401 if (fd == -1)
402 error(1, errno, "socket %d.%d", cfg_family, cfg_type);
403
404 val = 1 << 20;
405 if (setsockopt(fd, SOL_SOCKET, SO_RCVBUFFORCE, &val, sizeof(val)))
406 error(1, errno, "setsockopt rcvbuf");
407
408 return fd;
409 }
410
setup_rings(void)411 static void setup_rings(void)
412 {
413 int i;
414
415 for (i = 0; i < num_cpus; i++) {
416 rings[i].cpu = i;
417 rings[i].fd = create_ring(&rings[i].mmap);
418 }
419
420 /* accept packets once all rings in the fanout group are up */
421 for (i = 0; i < num_cpus; i++)
422 set_filter(rings[i].fd);
423 }
424
cleanup_rings(void)425 static void cleanup_rings(void)
426 {
427 int i;
428
429 for (i = 0; i < num_cpus; i++) {
430 if (munmap(rings[i].mmap, ring_block_nr * ring_block_sz))
431 error(1, errno, "munmap");
432 if (close(rings[i].fd))
433 error(1, errno, "close");
434 }
435 }
436
parse_cpulist(const char * arg)437 static void parse_cpulist(const char *arg)
438 {
439 do {
440 rx_irq_cpus[cfg_num_queues++] = strtol(arg, NULL, 10);
441
442 arg = strchr(arg, ',');
443 if (!arg)
444 break;
445 arg++; // skip ','
446 } while (1);
447 }
448
show_cpulist(void)449 static void show_cpulist(void)
450 {
451 int i;
452
453 for (i = 0; i < cfg_num_queues; i++)
454 fprintf(stderr, "rxq %d: cpu %d\n", i, rx_irq_cpus[i]);
455 }
456
show_silos(void)457 static void show_silos(void)
458 {
459 int i;
460
461 for (i = 0; i < cfg_num_rps_cpus; i++)
462 fprintf(stderr, "silo %d: cpu %d\n", i, rps_silo_to_cpu[i]);
463 }
464
parse_toeplitz_key(const char * str,int slen,unsigned char * key)465 static void parse_toeplitz_key(const char *str, int slen, unsigned char *key)
466 {
467 int i, ret, off;
468
469 if (slen < TOEPLITZ_STR_MIN_LEN ||
470 slen > TOEPLITZ_STR_MAX_LEN + 1)
471 error(1, 0, "invalid toeplitz key");
472
473 for (i = 0, off = 0; off < slen; i++, off += 3) {
474 ret = sscanf(str + off, "%hhx", &key[i]);
475 if (ret != 1)
476 error(1, 0, "key parse error at %d off %d len %d",
477 i, off, slen);
478 }
479 }
480
parse_rps_bitmap(const char * arg)481 static void parse_rps_bitmap(const char *arg)
482 {
483 unsigned long bitmap;
484 int i;
485
486 bitmap = strtoul(arg, NULL, 0);
487
488 if (bitmap & ~(RPS_MAX_CPUS - 1))
489 error(1, 0, "rps bitmap 0x%lx out of bounds 0..%lu",
490 bitmap, RPS_MAX_CPUS - 1);
491
492 for (i = 0; i < RPS_MAX_CPUS; i++)
493 if (bitmap & 1UL << i)
494 rps_silo_to_cpu[cfg_num_rps_cpus++] = i;
495 }
496
read_rss_dev_info_ynl(void)497 static void read_rss_dev_info_ynl(void)
498 {
499 struct ethtool_rss_get_req *req;
500 struct ethtool_rss_get_rsp *rsp;
501 struct ynl_sock *ys;
502
503 ys = ynl_sock_create(&ynl_ethtool_family, NULL);
504 if (!ys)
505 error(1, errno, "ynl_sock_create failed");
506
507 req = ethtool_rss_get_req_alloc();
508 if (!req)
509 error(1, errno, "ethtool_rss_get_req_alloc failed");
510
511 ethtool_rss_get_req_set_header_dev_name(req, cfg_ifname);
512
513 rsp = ethtool_rss_get(ys, req);
514 if (!rsp)
515 error(1, ys->err.code, "YNL: %s", ys->err.msg);
516
517 if (!rsp->_len.hkey)
518 error(1, 0, "RSS key not available for %s", cfg_ifname);
519
520 if (rsp->_len.hkey < TOEPLITZ_KEY_MIN_LEN ||
521 rsp->_len.hkey > TOEPLITZ_KEY_MAX_LEN)
522 error(1, 0, "RSS key length %u out of bounds [%u, %u]",
523 rsp->_len.hkey, TOEPLITZ_KEY_MIN_LEN,
524 TOEPLITZ_KEY_MAX_LEN);
525
526 memcpy(toeplitz_key, rsp->hkey, rsp->_len.hkey);
527
528 if (rsp->_count.indir > RSS_MAX_INDIR)
529 error(1, 0, "RSS indirection table too large (%u > %u)",
530 rsp->_count.indir, RSS_MAX_INDIR);
531
532 /* If indir table not available we'll fallback to simple modulo math */
533 if (rsp->_count.indir) {
534 memcpy(rss_indir_tbl, rsp->indir,
535 rsp->_count.indir * sizeof(rss_indir_tbl[0]));
536 rss_indir_tbl_size = rsp->_count.indir;
537
538 log_verbose("RSS indirection table size: %u\n",
539 rss_indir_tbl_size);
540 }
541
542 ethtool_rss_get_rsp_free(rsp);
543 ethtool_rss_get_req_free(req);
544 ynl_sock_destroy(ys);
545 }
546
parse_opts(int argc,char ** argv)547 static void parse_opts(int argc, char **argv)
548 {
549 static struct option long_options[] = {
550 {"dport", required_argument, 0, 'd'},
551 {"cpus", required_argument, 0, 'C'},
552 {"key", required_argument, 0, 'k'},
553 {"iface", required_argument, 0, 'i'},
554 {"ipv4", no_argument, 0, '4'},
555 {"ipv6", no_argument, 0, '6'},
556 {"sink", no_argument, 0, 's'},
557 {"tcp", no_argument, 0, 't'},
558 {"timeout", required_argument, 0, 'T'},
559 {"udp", no_argument, 0, 'u'},
560 {"verbose", no_argument, 0, 'v'},
561 {"rps", required_argument, 0, 'r'},
562 {0, 0, 0, 0}
563 };
564 bool have_toeplitz = false;
565 int index, c;
566
567 while ((c = getopt_long(argc, argv, "46C:d:i:k:r:stT:uv", long_options, &index)) != -1) {
568 switch (c) {
569 case '4':
570 cfg_family = AF_INET;
571 break;
572 case '6':
573 cfg_family = AF_INET6;
574 break;
575 case 'C':
576 parse_cpulist(optarg);
577 break;
578 case 'd':
579 cfg_dport = strtol(optarg, NULL, 0);
580 break;
581 case 'i':
582 cfg_ifname = optarg;
583 break;
584 case 'k':
585 parse_toeplitz_key(optarg, strlen(optarg),
586 toeplitz_key);
587 have_toeplitz = true;
588 break;
589 case 'r':
590 parse_rps_bitmap(optarg);
591 break;
592 case 's':
593 cfg_sink = true;
594 break;
595 case 't':
596 cfg_type = SOCK_STREAM;
597 break;
598 case 'T':
599 cfg_timeout_msec = strtol(optarg, NULL, 0);
600 break;
601 case 'u':
602 cfg_type = SOCK_DGRAM;
603 break;
604 case 'v':
605 cfg_verbose = true;
606 break;
607
608 default:
609 error(1, 0, "unknown option %c", optopt);
610 break;
611 }
612 }
613
614 if (!have_toeplitz)
615 read_rss_dev_info_ynl();
616
617 num_cpus = get_nprocs();
618 if (num_cpus > RSS_MAX_CPUS)
619 error(1, 0, "increase RSS_MAX_CPUS");
620
621 if (cfg_num_queues && cfg_num_rps_cpus)
622 error(1, 0,
623 "Can't supply both RSS cpus ('-C') and RPS map ('-r')");
624 if (cfg_verbose) {
625 show_cpulist();
626 show_silos();
627 }
628 }
629
main(int argc,char ** argv)630 int main(int argc, char **argv)
631 {
632 const int min_tests = 10;
633 int fd_sink = -1;
634
635 parse_opts(argc, argv);
636
637 if (cfg_sink)
638 fd_sink = setup_sink();
639
640 setup_rings();
641
642 /* Signal to test framework that we're ready to receive */
643 ksft_ready();
644
645 process_rings();
646 cleanup_rings();
647
648 if (cfg_sink && close(fd_sink))
649 error(1, errno, "close sink");
650
651 if (frames_received - frames_nohash < min_tests)
652 error(1, 0, "too few frames for verification");
653
654 return frames_error;
655 }
656