1 // SPDX-License-Identifier: GPL-2.0 2 3 #define _GNU_SOURCE 4 5 #include <arpa/inet.h> 6 #include <errno.h> 7 #include <error.h> 8 #include <linux/errqueue.h> 9 #include <linux/net_tstamp.h> 10 #include <netinet/if_ether.h> 11 #include <netinet/in.h> 12 #include <netinet/ip.h> 13 #include <netinet/ip6.h> 14 #include <netinet/udp.h> 15 #include <poll.h> 16 #include <sched.h> 17 #include <signal.h> 18 #include <stdbool.h> 19 #include <stdio.h> 20 #include <stdlib.h> 21 #include <string.h> 22 #include <sys/socket.h> 23 #include <sys/time.h> 24 #include <sys/poll.h> 25 #include <sys/types.h> 26 #include <unistd.h> 27 28 #include "../kselftest.h" 29 30 #ifndef ETH_MAX_MTU 31 #define ETH_MAX_MTU 0xFFFFU 32 #endif 33 34 #ifndef UDP_SEGMENT 35 #define UDP_SEGMENT 103 36 #endif 37 38 #ifndef SO_ZEROCOPY 39 #define SO_ZEROCOPY 60 40 #endif 41 42 #ifndef SO_EE_ORIGIN_ZEROCOPY 43 #define SO_EE_ORIGIN_ZEROCOPY 5 44 #endif 45 46 #ifndef MSG_ZEROCOPY 47 #define MSG_ZEROCOPY 0x4000000 48 #endif 49 50 #ifndef ENOTSUPP 51 #define ENOTSUPP 524 52 #endif 53 54 #define NUM_PKT 100 55 56 static bool cfg_cache_trash; 57 static int cfg_cpu = -1; 58 static int cfg_connected = true; 59 static int cfg_family = PF_UNSPEC; 60 static uint16_t cfg_mss; 61 static int cfg_payload_len = (1472 * 42); 62 static int cfg_port = 8000; 63 static int cfg_runtime_ms = -1; 64 static bool cfg_poll; 65 static bool cfg_segment; 66 static bool cfg_sendmmsg; 67 static bool cfg_tcp; 68 static uint32_t cfg_tx_ts = SOF_TIMESTAMPING_TX_SOFTWARE; 69 static bool cfg_tx_tstamp; 70 static bool cfg_audit; 71 static bool cfg_verbose; 72 static bool cfg_zerocopy; 73 static int cfg_msg_nr; 74 static uint16_t cfg_gso_size; 75 static unsigned long total_num_msgs; 76 static unsigned long total_num_sends; 77 static unsigned long stat_tx_ts; 78 static unsigned long stat_tx_ts_errors; 79 static unsigned long tstart; 80 static unsigned long tend; 81 static unsigned long stat_zcopies; 82 83 static socklen_t cfg_alen; 84 static struct sockaddr_storage cfg_dst_addr; 85 86 static bool interrupted; 87 static char buf[NUM_PKT][ETH_MAX_MTU]; 88 89 static void sigint_handler(int signum) 90 { 91 if (signum == SIGINT) 92 interrupted = true; 93 } 94 95 static unsigned long gettimeofday_ms(void) 96 { 97 struct timeval tv; 98 99 gettimeofday(&tv, NULL); 100 return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); 101 } 102 103 static int set_cpu(int cpu) 104 { 105 cpu_set_t mask; 106 107 CPU_ZERO(&mask); 108 CPU_SET(cpu, &mask); 109 if (sched_setaffinity(0, sizeof(mask), &mask)) 110 error(1, 0, "setaffinity %d", cpu); 111 112 return 0; 113 } 114 115 static void setup_sockaddr(int domain, const char *str_addr, void *sockaddr) 116 { 117 struct sockaddr_in6 *addr6 = (void *) sockaddr; 118 struct sockaddr_in *addr4 = (void *) sockaddr; 119 120 switch (domain) { 121 case PF_INET: 122 addr4->sin_family = AF_INET; 123 addr4->sin_port = htons(cfg_port); 124 if (inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1) 125 error(1, 0, "ipv4 parse error: %s", str_addr); 126 break; 127 case PF_INET6: 128 addr6->sin6_family = AF_INET6; 129 addr6->sin6_port = htons(cfg_port); 130 if (inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1) 131 error(1, 0, "ipv6 parse error: %s", str_addr); 132 break; 133 default: 134 error(1, 0, "illegal domain"); 135 } 136 } 137 138 static void flush_cmsg(struct cmsghdr *cmsg) 139 { 140 struct sock_extended_err *err; 141 struct scm_timestamping *tss; 142 __u32 lo; 143 __u32 hi; 144 int i; 145 146 switch (cmsg->cmsg_level) { 147 case SOL_SOCKET: 148 if (cmsg->cmsg_type == SO_TIMESTAMPING) { 149 i = (cfg_tx_ts == SOF_TIMESTAMPING_TX_HARDWARE) ? 2 : 0; 150 tss = (struct scm_timestamping *)CMSG_DATA(cmsg); 151 if (tss->ts[i].tv_sec == 0) 152 stat_tx_ts_errors++; 153 } else { 154 error(1, 0, "unknown SOL_SOCKET cmsg type=%u\n", 155 cmsg->cmsg_type); 156 } 157 break; 158 case SOL_IP: 159 case SOL_IPV6: 160 switch (cmsg->cmsg_type) { 161 case IP_RECVERR: 162 case IPV6_RECVERR: 163 { 164 err = (struct sock_extended_err *)CMSG_DATA(cmsg); 165 switch (err->ee_origin) { 166 case SO_EE_ORIGIN_TIMESTAMPING: 167 /* Got a TX timestamp from error queue */ 168 stat_tx_ts++; 169 break; 170 case SO_EE_ORIGIN_ICMP: 171 case SO_EE_ORIGIN_ICMP6: 172 if (cfg_verbose) 173 fprintf(stderr, 174 "received ICMP error: type=%u, code=%u\n", 175 err->ee_type, err->ee_code); 176 break; 177 case SO_EE_ORIGIN_ZEROCOPY: 178 { 179 lo = err->ee_info; 180 hi = err->ee_data; 181 /* range of IDs acknowledged */ 182 stat_zcopies += hi - lo + 1; 183 break; 184 } 185 case SO_EE_ORIGIN_LOCAL: 186 if (cfg_verbose) 187 fprintf(stderr, 188 "received packet with local origin: %u\n", 189 err->ee_origin); 190 break; 191 default: 192 error(0, 1, "received packet with origin: %u", 193 err->ee_origin); 194 } 195 break; 196 } 197 default: 198 error(0, 1, "unknown IP msg type=%u\n", 199 cmsg->cmsg_type); 200 break; 201 } 202 break; 203 default: 204 error(0, 1, "unknown cmsg level=%u\n", 205 cmsg->cmsg_level); 206 } 207 } 208 209 static void flush_errqueue_recv(int fd) 210 { 211 char control[CMSG_SPACE(sizeof(struct scm_timestamping)) + 212 CMSG_SPACE(sizeof(struct sock_extended_err)) + 213 CMSG_SPACE(sizeof(struct sockaddr_in6))] = {0}; 214 struct msghdr msg = {0}; 215 struct cmsghdr *cmsg; 216 int ret; 217 218 while (1) { 219 msg.msg_control = control; 220 msg.msg_controllen = sizeof(control); 221 ret = recvmsg(fd, &msg, MSG_ERRQUEUE); 222 if (ret == -1 && errno == EAGAIN) 223 break; 224 if (ret == -1) 225 error(1, errno, "errqueue"); 226 if (msg.msg_flags != MSG_ERRQUEUE) 227 error(1, 0, "errqueue: flags 0x%x\n", msg.msg_flags); 228 if (cfg_audit) { 229 for (cmsg = CMSG_FIRSTHDR(&msg); 230 cmsg; 231 cmsg = CMSG_NXTHDR(&msg, cmsg)) 232 flush_cmsg(cmsg); 233 } 234 msg.msg_flags = 0; 235 } 236 } 237 238 static void flush_errqueue(int fd, const bool do_poll) 239 { 240 if (do_poll) { 241 struct pollfd fds = {0}; 242 int ret; 243 244 fds.fd = fd; 245 ret = poll(&fds, 1, 500); 246 if (ret == 0) { 247 if (cfg_verbose) 248 fprintf(stderr, "poll timeout\n"); 249 } else if (ret < 0) { 250 error(1, errno, "poll"); 251 } 252 } 253 254 flush_errqueue_recv(fd); 255 } 256 257 static int send_tcp(int fd, char *data) 258 { 259 int ret, done = 0, count = 0; 260 261 while (done < cfg_payload_len) { 262 ret = send(fd, data + done, cfg_payload_len - done, 263 cfg_zerocopy ? MSG_ZEROCOPY : 0); 264 if (ret == -1) 265 error(1, errno, "write"); 266 267 done += ret; 268 count++; 269 } 270 271 return count; 272 } 273 274 static int send_udp(int fd, char *data) 275 { 276 int ret, total_len, len, count = 0; 277 278 total_len = cfg_payload_len; 279 280 while (total_len) { 281 len = total_len < cfg_mss ? total_len : cfg_mss; 282 283 ret = sendto(fd, data, len, cfg_zerocopy ? MSG_ZEROCOPY : 0, 284 cfg_connected ? NULL : (void *)&cfg_dst_addr, 285 cfg_connected ? 0 : cfg_alen); 286 if (ret == -1) 287 error(1, errno, "write"); 288 if (ret != len) 289 error(1, errno, "write: %uB != %uB\n", ret, len); 290 291 total_len -= len; 292 count++; 293 } 294 295 return count; 296 } 297 298 static void send_ts_cmsg(struct cmsghdr *cm) 299 { 300 uint32_t *valp; 301 302 cm->cmsg_level = SOL_SOCKET; 303 cm->cmsg_type = SO_TIMESTAMPING; 304 cm->cmsg_len = CMSG_LEN(sizeof(cfg_tx_ts)); 305 valp = (void *)CMSG_DATA(cm); 306 *valp = cfg_tx_ts; 307 } 308 309 static int send_udp_sendmmsg(int fd, char *data) 310 { 311 char control[CMSG_SPACE(sizeof(cfg_tx_ts))] = {0}; 312 const int max_nr_msg = ETH_MAX_MTU / ETH_DATA_LEN; 313 struct mmsghdr mmsgs[max_nr_msg]; 314 struct iovec iov[max_nr_msg]; 315 unsigned int off = 0, left; 316 size_t msg_controllen = 0; 317 int i = 0, ret; 318 319 memset(mmsgs, 0, sizeof(mmsgs)); 320 321 if (cfg_tx_tstamp) { 322 struct msghdr msg = {0}; 323 struct cmsghdr *cmsg; 324 325 msg.msg_control = control; 326 msg.msg_controllen = sizeof(control); 327 cmsg = CMSG_FIRSTHDR(&msg); 328 send_ts_cmsg(cmsg); 329 msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts)); 330 } 331 332 left = cfg_payload_len; 333 while (left) { 334 if (i == max_nr_msg) 335 error(1, 0, "sendmmsg: exceeds max_nr_msg"); 336 337 iov[i].iov_base = data + off; 338 iov[i].iov_len = cfg_mss < left ? cfg_mss : left; 339 340 mmsgs[i].msg_hdr.msg_iov = iov + i; 341 mmsgs[i].msg_hdr.msg_iovlen = 1; 342 343 mmsgs[i].msg_hdr.msg_name = (void *)&cfg_dst_addr; 344 mmsgs[i].msg_hdr.msg_namelen = cfg_alen; 345 if (msg_controllen) { 346 mmsgs[i].msg_hdr.msg_control = control; 347 mmsgs[i].msg_hdr.msg_controllen = msg_controllen; 348 } 349 350 off += iov[i].iov_len; 351 left -= iov[i].iov_len; 352 i++; 353 } 354 355 ret = sendmmsg(fd, mmsgs, i, cfg_zerocopy ? MSG_ZEROCOPY : 0); 356 if (ret == -1) 357 error(1, errno, "sendmmsg"); 358 359 return ret; 360 } 361 362 static void send_udp_segment_cmsg(struct cmsghdr *cm) 363 { 364 uint16_t *valp; 365 366 cm->cmsg_level = SOL_UDP; 367 cm->cmsg_type = UDP_SEGMENT; 368 cm->cmsg_len = CMSG_LEN(sizeof(cfg_gso_size)); 369 valp = (void *)CMSG_DATA(cm); 370 *valp = cfg_gso_size; 371 } 372 373 static int send_udp_segment(int fd, char *data) 374 { 375 char control[CMSG_SPACE(sizeof(cfg_gso_size)) + 376 CMSG_SPACE(sizeof(cfg_tx_ts))] = {0}; 377 struct msghdr msg = {0}; 378 struct iovec iov = {0}; 379 size_t msg_controllen; 380 struct cmsghdr *cmsg; 381 int ret; 382 383 iov.iov_base = data; 384 iov.iov_len = cfg_payload_len; 385 386 msg.msg_iov = &iov; 387 msg.msg_iovlen = 1; 388 389 msg.msg_control = control; 390 msg.msg_controllen = sizeof(control); 391 cmsg = CMSG_FIRSTHDR(&msg); 392 send_udp_segment_cmsg(cmsg); 393 msg_controllen = CMSG_SPACE(sizeof(cfg_mss)); 394 if (cfg_tx_tstamp) { 395 cmsg = CMSG_NXTHDR(&msg, cmsg); 396 send_ts_cmsg(cmsg); 397 msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts)); 398 } 399 400 msg.msg_controllen = msg_controllen; 401 msg.msg_name = (void *)&cfg_dst_addr; 402 msg.msg_namelen = cfg_alen; 403 404 ret = sendmsg(fd, &msg, cfg_zerocopy ? MSG_ZEROCOPY : 0); 405 if (ret == -1) 406 error(1, errno, "sendmsg"); 407 if (ret != iov.iov_len) 408 error(1, 0, "sendmsg: %u != %llu\n", ret, 409 (unsigned long long)iov.iov_len); 410 411 return 1; 412 } 413 414 static void usage(const char *filepath) 415 { 416 error(1, 0, "Usage: %s [-46acmHPtTuvz] [-C cpu] [-D dst ip] [-l secs] [-M messagenr] [-p port] [-s sendsize] [-S gsosize]", 417 filepath); 418 } 419 420 static void parse_opts(int argc, char **argv) 421 { 422 const char *bind_addr = NULL; 423 int max_len, hdrlen; 424 int c; 425 426 while ((c = getopt(argc, argv, "46acC:D:Hl:mM:p:s:PS:tTuvz")) != -1) { 427 switch (c) { 428 case '4': 429 if (cfg_family != PF_UNSPEC) 430 error(1, 0, "Pass one of -4 or -6"); 431 cfg_family = PF_INET; 432 cfg_alen = sizeof(struct sockaddr_in); 433 break; 434 case '6': 435 if (cfg_family != PF_UNSPEC) 436 error(1, 0, "Pass one of -4 or -6"); 437 cfg_family = PF_INET6; 438 cfg_alen = sizeof(struct sockaddr_in6); 439 break; 440 case 'a': 441 cfg_audit = true; 442 break; 443 case 'c': 444 cfg_cache_trash = true; 445 break; 446 case 'C': 447 cfg_cpu = strtol(optarg, NULL, 0); 448 break; 449 case 'D': 450 bind_addr = optarg; 451 break; 452 case 'l': 453 cfg_runtime_ms = strtoul(optarg, NULL, 10) * 1000; 454 break; 455 case 'm': 456 cfg_sendmmsg = true; 457 break; 458 case 'M': 459 cfg_msg_nr = strtoul(optarg, NULL, 10); 460 break; 461 case 'p': 462 cfg_port = strtoul(optarg, NULL, 0); 463 break; 464 case 'P': 465 cfg_poll = true; 466 break; 467 case 's': 468 cfg_payload_len = strtoul(optarg, NULL, 0); 469 break; 470 case 'S': 471 cfg_gso_size = strtoul(optarg, NULL, 0); 472 cfg_segment = true; 473 break; 474 case 'H': 475 cfg_tx_ts = SOF_TIMESTAMPING_TX_HARDWARE; 476 cfg_tx_tstamp = true; 477 break; 478 case 't': 479 cfg_tcp = true; 480 break; 481 case 'T': 482 cfg_tx_tstamp = true; 483 break; 484 case 'u': 485 cfg_connected = false; 486 break; 487 case 'v': 488 cfg_verbose = true; 489 break; 490 case 'z': 491 cfg_zerocopy = true; 492 break; 493 } 494 } 495 496 if (!bind_addr) 497 bind_addr = cfg_family == PF_INET6 ? "::" : "0.0.0.0"; 498 499 setup_sockaddr(cfg_family, bind_addr, &cfg_dst_addr); 500 501 if (optind != argc) 502 usage(argv[0]); 503 504 if (cfg_family == PF_UNSPEC) 505 error(1, 0, "must pass one of -4 or -6"); 506 if (cfg_tcp && !cfg_connected) 507 error(1, 0, "connectionless tcp makes no sense"); 508 if (cfg_segment && cfg_sendmmsg) 509 error(1, 0, "cannot combine segment offload and sendmmsg"); 510 if (cfg_tx_tstamp && !(cfg_segment || cfg_sendmmsg)) 511 error(1, 0, "Options -T and -H require either -S or -m option"); 512 513 if (cfg_family == PF_INET) 514 hdrlen = sizeof(struct iphdr) + sizeof(struct udphdr); 515 else 516 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct udphdr); 517 518 cfg_mss = ETH_DATA_LEN - hdrlen; 519 max_len = ETH_MAX_MTU - hdrlen; 520 if (!cfg_gso_size) 521 cfg_gso_size = cfg_mss; 522 523 if (cfg_payload_len > max_len) 524 error(1, 0, "payload length %u exceeds max %u", 525 cfg_payload_len, max_len); 526 } 527 528 static void set_pmtu_discover(int fd, bool is_ipv4) 529 { 530 int level, name, val; 531 532 if (is_ipv4) { 533 level = SOL_IP; 534 name = IP_MTU_DISCOVER; 535 val = IP_PMTUDISC_DO; 536 } else { 537 level = SOL_IPV6; 538 name = IPV6_MTU_DISCOVER; 539 val = IPV6_PMTUDISC_DO; 540 } 541 542 if (setsockopt(fd, level, name, &val, sizeof(val))) 543 error(1, errno, "setsockopt path mtu"); 544 } 545 546 static void set_tx_timestamping(int fd) 547 { 548 int val = SOF_TIMESTAMPING_OPT_CMSG | SOF_TIMESTAMPING_OPT_ID | 549 SOF_TIMESTAMPING_OPT_TSONLY; 550 551 if (cfg_tx_ts == SOF_TIMESTAMPING_TX_SOFTWARE) 552 val |= SOF_TIMESTAMPING_SOFTWARE; 553 else 554 val |= SOF_TIMESTAMPING_RAW_HARDWARE; 555 556 if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, &val, sizeof(val))) 557 error(1, errno, "setsockopt tx timestamping"); 558 } 559 560 static void print_audit_report(unsigned long num_msgs, unsigned long num_sends) 561 { 562 unsigned long tdelta; 563 564 tdelta = tend - tstart; 565 if (!tdelta) 566 return; 567 568 fprintf(stderr, "Summary over %lu.%03lu seconds...\n", 569 tdelta / 1000, tdelta % 1000); 570 fprintf(stderr, 571 "sum %s tx: %6lu MB/s %10lu calls (%lu/s) %10lu msgs (%lu/s)\n", 572 cfg_tcp ? "tcp" : "udp", 573 ((num_msgs * cfg_payload_len) >> 10) / tdelta, 574 num_sends, num_sends * 1000 / tdelta, 575 num_msgs, num_msgs * 1000 / tdelta); 576 577 if (cfg_tx_tstamp) { 578 if (stat_tx_ts_errors) 579 error(1, 0, 580 "Expected clean TX Timestamps: %9lu msgs received %6lu errors", 581 stat_tx_ts, stat_tx_ts_errors); 582 if (stat_tx_ts != num_sends) 583 error(1, 0, 584 "Unexpected number of TX Timestamps: %9lu expected %9lu received", 585 num_sends, stat_tx_ts); 586 fprintf(stderr, 587 "Tx Timestamps: %19lu received %17lu errors\n", 588 stat_tx_ts, stat_tx_ts_errors); 589 } 590 591 if (cfg_zerocopy) { 592 if (stat_zcopies != num_sends) 593 error(1, 0, "Unexpected number of Zerocopy completions: %9lu expected %9lu received", 594 num_sends, stat_zcopies); 595 fprintf(stderr, 596 "Zerocopy acks: %19lu\n", 597 stat_zcopies); 598 } 599 } 600 601 static void print_report(unsigned long num_msgs, unsigned long num_sends) 602 { 603 fprintf(stderr, 604 "%s tx: %6lu MB/s %8lu calls/s %6lu msg/s\n", 605 cfg_tcp ? "tcp" : "udp", 606 (num_msgs * cfg_payload_len) >> 20, 607 num_sends, num_msgs); 608 609 if (cfg_audit) { 610 total_num_msgs += num_msgs; 611 total_num_sends += num_sends; 612 } 613 } 614 615 int main(int argc, char **argv) 616 { 617 unsigned long num_msgs, num_sends; 618 unsigned long tnow, treport, tstop; 619 int fd, i, val, ret; 620 621 parse_opts(argc, argv); 622 623 if (cfg_cpu > 0) 624 set_cpu(cfg_cpu); 625 626 for (i = 0; i < sizeof(buf[0]); i++) 627 buf[0][i] = 'a' + (i % 26); 628 for (i = 1; i < NUM_PKT; i++) 629 memcpy(buf[i], buf[0], sizeof(buf[0])); 630 631 signal(SIGINT, sigint_handler); 632 633 fd = socket(cfg_family, cfg_tcp ? SOCK_STREAM : SOCK_DGRAM, 0); 634 if (fd == -1) 635 error(1, errno, "socket"); 636 637 if (cfg_zerocopy) { 638 val = 1; 639 640 ret = setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY, 641 &val, sizeof(val)); 642 if (ret) { 643 if (errno == ENOPROTOOPT || errno == ENOTSUPP) { 644 fprintf(stderr, "SO_ZEROCOPY not supported"); 645 exit(KSFT_SKIP); 646 } 647 error(1, errno, "setsockopt zerocopy"); 648 } 649 } 650 651 if (cfg_connected && 652 connect(fd, (void *)&cfg_dst_addr, cfg_alen)) 653 error(1, errno, "connect"); 654 655 if (cfg_segment) 656 set_pmtu_discover(fd, cfg_family == PF_INET); 657 658 if (cfg_tx_tstamp) 659 set_tx_timestamping(fd); 660 661 num_msgs = num_sends = 0; 662 tnow = gettimeofday_ms(); 663 tstart = tnow; 664 tend = tnow; 665 tstop = tnow + cfg_runtime_ms; 666 treport = tnow + 1000; 667 668 i = 0; 669 do { 670 if (cfg_tcp) 671 num_sends += send_tcp(fd, buf[i]); 672 else if (cfg_segment) 673 num_sends += send_udp_segment(fd, buf[i]); 674 else if (cfg_sendmmsg) 675 num_sends += send_udp_sendmmsg(fd, buf[i]); 676 else 677 num_sends += send_udp(fd, buf[i]); 678 num_msgs++; 679 if ((cfg_zerocopy && ((num_msgs & 0xF) == 0)) || cfg_tx_tstamp) 680 flush_errqueue(fd, cfg_poll); 681 682 if (cfg_msg_nr && num_msgs >= cfg_msg_nr) 683 break; 684 685 tnow = gettimeofday_ms(); 686 if (tnow >= treport) { 687 print_report(num_msgs, num_sends); 688 num_msgs = num_sends = 0; 689 treport = tnow + 1000; 690 } 691 692 /* cold cache when writing buffer */ 693 if (cfg_cache_trash) 694 i = ++i < NUM_PKT ? i : 0; 695 696 } while (!interrupted && (cfg_runtime_ms == -1 || tnow < tstop)); 697 698 if (cfg_zerocopy || cfg_tx_tstamp) 699 flush_errqueue(fd, true); 700 701 if (close(fd)) 702 error(1, errno, "close"); 703 704 if (cfg_audit) { 705 tend = tnow; 706 total_num_msgs += num_msgs; 707 total_num_sends += num_sends; 708 print_audit_report(total_num_msgs, total_num_sends); 709 } 710 711 return 0; 712 } 713