1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2018 Google Inc. 4 * Author: Eric Dumazet (edumazet@google.com) 5 * 6 * Reference program demonstrating tcp mmap() usage, 7 * and SO_RCVLOWAT hints for receiver. 8 * 9 * Note : NIC with header split is needed to use mmap() on TCP : 10 * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload. 11 * 12 * How to use on loopback interface : 13 * 14 * ifconfig lo mtu 61512 # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header) 15 * tcp_mmap -s -z & 16 * tcp_mmap -H ::1 -z 17 * 18 * Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12) 19 * (4096 : page size on x86, 12: TCP TS option length) 20 * tcp_mmap -s -z -M $((4096+12)) & 21 * tcp_mmap -H ::1 -z -M $((4096+12)) 22 * 23 * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface. 24 * We might use sendfile() instead, but really this test program is about mmap(), for receivers ;) 25 * 26 * $ ./tcp_mmap -s & # Without mmap() 27 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done 28 * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit 29 * cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches 30 * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit 31 * cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches 32 * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit 33 * cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches 34 * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit 35 * cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches 36 * $ kill %1 # kill tcp_mmap server 37 * 38 * $ ./tcp_mmap -s -z & # With mmap() 39 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done 40 * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit 41 * cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches 42 * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit 43 * cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches 44 * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit 45 * cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches 46 * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit 47 * cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches 48 */ 49 #define _GNU_SOURCE 50 #include <pthread.h> 51 #include <sys/types.h> 52 #include <fcntl.h> 53 #include <error.h> 54 #include <sys/socket.h> 55 #include <sys/mman.h> 56 #include <sys/resource.h> 57 #include <unistd.h> 58 #include <string.h> 59 #include <stdlib.h> 60 #include <stdio.h> 61 #include <errno.h> 62 #include <time.h> 63 #include <sys/time.h> 64 #include <netinet/in.h> 65 #include <arpa/inet.h> 66 #include <poll.h> 67 #include <linux/tcp.h> 68 #include <assert.h> 69 #include <openssl/pem.h> 70 71 #ifndef MSG_ZEROCOPY 72 #define MSG_ZEROCOPY 0x4000000 73 #endif 74 75 #ifndef min 76 #define min(a, b) ((a) < (b) ? (a) : (b)) 77 #endif 78 79 #define FILE_SZ (1ULL << 35) 80 static int cfg_family = AF_INET6; 81 static socklen_t cfg_alen = sizeof(struct sockaddr_in6); 82 static int cfg_port = 8787; 83 84 static int rcvbuf; /* Default: autotuning. Can be set with -r <integer> option */ 85 static int sndbuf; /* Default: autotuning. Can be set with -w <integer> option */ 86 static int zflg; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */ 87 static int xflg; /* hash received data (simple xor) (-h option) */ 88 static int integrity; /* -i option: sender and receiver compute sha256 over the data.*/ 89 90 static size_t chunk_size = 512*1024; 91 92 static size_t map_align; 93 94 unsigned long htotal; 95 unsigned int digest_len; 96 97 static inline void prefetch(const void *x) 98 { 99 #if defined(__x86_64__) 100 asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x)); 101 #endif 102 } 103 104 void hash_zone(void *zone, unsigned int length) 105 { 106 unsigned long temp = htotal; 107 108 while (length >= 8*sizeof(long)) { 109 prefetch(zone + 384); 110 temp ^= *(unsigned long *)zone; 111 temp ^= *(unsigned long *)(zone + sizeof(long)); 112 temp ^= *(unsigned long *)(zone + 2*sizeof(long)); 113 temp ^= *(unsigned long *)(zone + 3*sizeof(long)); 114 temp ^= *(unsigned long *)(zone + 4*sizeof(long)); 115 temp ^= *(unsigned long *)(zone + 5*sizeof(long)); 116 temp ^= *(unsigned long *)(zone + 6*sizeof(long)); 117 temp ^= *(unsigned long *)(zone + 7*sizeof(long)); 118 zone += 8*sizeof(long); 119 length -= 8*sizeof(long); 120 } 121 while (length >= 1) { 122 temp ^= *(unsigned char *)zone; 123 zone += 1; 124 length--; 125 } 126 htotal = temp; 127 } 128 129 #define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1)) 130 #define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to)) 131 132 133 static void *mmap_large_buffer(size_t need, size_t *allocated) 134 { 135 void *buffer; 136 size_t sz; 137 138 /* Attempt to use huge pages if possible. */ 139 sz = ALIGN_UP(need, map_align); 140 buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE, 141 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); 142 143 if (buffer == MAP_FAILED) { 144 sz = need; 145 buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE, 146 MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE, 147 -1, 0); 148 if (buffer != MAP_FAILED) 149 fprintf(stderr, "MAP_HUGETLB attempt failed, look at /sys/kernel/mm/hugepages for optimal performance\n"); 150 } 151 *allocated = sz; 152 return buffer; 153 } 154 155 static uint32_t tcp_info_get_rcv_mss(int fd) 156 { 157 socklen_t sz = sizeof(struct tcp_info); 158 struct tcp_info info; 159 160 if (getsockopt(fd, IPPROTO_TCP, TCP_INFO, &info, &sz)) { 161 fprintf(stderr, "Error fetching TCP_INFO\n"); 162 return 0; 163 } 164 165 return info.tcpi_rcv_mss; 166 } 167 168 void *child_thread(void *arg) 169 { 170 unsigned char digest[SHA256_DIGEST_LENGTH]; 171 unsigned long total_mmap = 0, total = 0; 172 struct tcp_zerocopy_receive zc; 173 unsigned char *buffer = NULL; 174 unsigned long delta_usec; 175 EVP_MD_CTX *ctx = NULL; 176 int flags = MAP_SHARED; 177 struct timeval t0, t1; 178 void *raddr = NULL; 179 void *addr = NULL; 180 double throughput; 181 struct rusage ru; 182 size_t buffer_sz; 183 int lu, fd; 184 185 fd = (int)(unsigned long)arg; 186 187 gettimeofday(&t0, NULL); 188 189 fcntl(fd, F_SETFL, O_NDELAY); 190 buffer = mmap_large_buffer(chunk_size, &buffer_sz); 191 if (buffer == MAP_FAILED) { 192 perror("mmap"); 193 goto error; 194 } 195 if (zflg) { 196 raddr = mmap(NULL, chunk_size + map_align, PROT_READ, flags, fd, 0); 197 if (raddr == MAP_FAILED) { 198 perror("mmap"); 199 zflg = 0; 200 } else { 201 addr = ALIGN_PTR_UP(raddr, map_align); 202 } 203 } 204 if (integrity) { 205 ctx = EVP_MD_CTX_new(); 206 if (!ctx) { 207 perror("cannot enable SHA computing"); 208 goto error; 209 } 210 EVP_DigestInit_ex(ctx, EVP_sha256(), NULL); 211 } 212 while (1) { 213 struct pollfd pfd = { .fd = fd, .events = POLLIN, }; 214 int sub; 215 216 poll(&pfd, 1, 10000); 217 if (zflg) { 218 socklen_t zc_len = sizeof(zc); 219 int res; 220 221 memset(&zc, 0, sizeof(zc)); 222 zc.address = (__u64)((unsigned long)addr); 223 zc.length = min(chunk_size, FILE_SZ - total); 224 225 res = getsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE, 226 &zc, &zc_len); 227 if (res == -1) 228 break; 229 230 if (zc.length) { 231 assert(zc.length <= chunk_size); 232 if (integrity) 233 EVP_DigestUpdate(ctx, addr, zc.length); 234 total_mmap += zc.length; 235 if (xflg) 236 hash_zone(addr, zc.length); 237 /* It is more efficient to unmap the pages right now, 238 * instead of doing this in next TCP_ZEROCOPY_RECEIVE. 239 */ 240 madvise(addr, zc.length, MADV_DONTNEED); 241 total += zc.length; 242 } 243 if (zc.recv_skip_hint) { 244 assert(zc.recv_skip_hint <= chunk_size); 245 lu = read(fd, buffer, min(zc.recv_skip_hint, 246 FILE_SZ - total)); 247 if (lu > 0) { 248 if (integrity) 249 EVP_DigestUpdate(ctx, buffer, lu); 250 if (xflg) 251 hash_zone(buffer, lu); 252 total += lu; 253 } 254 if (lu == 0) 255 goto end; 256 } 257 continue; 258 } 259 sub = 0; 260 while (sub < chunk_size) { 261 lu = read(fd, buffer + sub, min(chunk_size - sub, 262 FILE_SZ - total)); 263 if (lu == 0) 264 goto end; 265 if (lu < 0) 266 break; 267 if (integrity) 268 EVP_DigestUpdate(ctx, buffer + sub, lu); 269 if (xflg) 270 hash_zone(buffer + sub, lu); 271 total += lu; 272 sub += lu; 273 } 274 } 275 end: 276 gettimeofday(&t1, NULL); 277 delta_usec = (t1.tv_sec - t0.tv_sec) * 1000000 + t1.tv_usec - t0.tv_usec; 278 279 if (integrity) { 280 fcntl(fd, F_SETFL, 0); 281 EVP_DigestFinal_ex(ctx, digest, &digest_len); 282 lu = read(fd, buffer, SHA256_DIGEST_LENGTH); 283 if (lu != SHA256_DIGEST_LENGTH) 284 perror("Error: Cannot read SHA256\n"); 285 286 if (memcmp(digest, buffer, 287 SHA256_DIGEST_LENGTH)) 288 fprintf(stderr, "Error: SHA256 of the data is not right\n"); 289 else 290 printf("\nSHA256 is correct\n"); 291 } 292 293 throughput = 0; 294 if (delta_usec) 295 throughput = total * 8.0 / (double)delta_usec / 1000.0; 296 getrusage(RUSAGE_THREAD, &ru); 297 if (total > 1024*1024) { 298 unsigned long total_usec; 299 unsigned long mb = total >> 20; 300 total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec + 301 1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec; 302 printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n" 303 " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches, rcv_mss %u\n", 304 total / (1024.0 * 1024.0), 305 100.0*total_mmap/total, 306 (double)delta_usec / 1000000.0, 307 throughput, 308 (double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0, 309 (double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0, 310 (double)total_usec/mb, 311 ru.ru_nvcsw, 312 tcp_info_get_rcv_mss(fd)); 313 } 314 error: 315 if (ctx) 316 EVP_MD_CTX_free(ctx); 317 munmap(buffer, buffer_sz); 318 close(fd); 319 if (zflg) 320 munmap(raddr, chunk_size + map_align); 321 pthread_exit(0); 322 } 323 324 static void apply_rcvsnd_buf(int fd) 325 { 326 if (rcvbuf && setsockopt(fd, SOL_SOCKET, 327 SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) == -1) { 328 perror("setsockopt SO_RCVBUF"); 329 } 330 331 if (sndbuf && setsockopt(fd, SOL_SOCKET, 332 SO_SNDBUF, &sndbuf, sizeof(sndbuf)) == -1) { 333 perror("setsockopt SO_SNDBUF"); 334 } 335 } 336 337 338 static void setup_sockaddr(int domain, const char *str_addr, 339 struct sockaddr_storage *sockaddr) 340 { 341 struct sockaddr_in6 *addr6 = (void *) sockaddr; 342 struct sockaddr_in *addr4 = (void *) sockaddr; 343 344 switch (domain) { 345 case PF_INET: 346 memset(addr4, 0, sizeof(*addr4)); 347 addr4->sin_family = AF_INET; 348 addr4->sin_port = htons(cfg_port); 349 if (str_addr && 350 inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1) 351 error(1, 0, "ipv4 parse error: %s", str_addr); 352 break; 353 case PF_INET6: 354 memset(addr6, 0, sizeof(*addr6)); 355 addr6->sin6_family = AF_INET6; 356 addr6->sin6_port = htons(cfg_port); 357 if (str_addr && 358 inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1) 359 error(1, 0, "ipv6 parse error: %s", str_addr); 360 break; 361 default: 362 error(1, 0, "illegal domain"); 363 } 364 } 365 366 static void do_accept(int fdlisten) 367 { 368 pthread_attr_t attr; 369 int rcvlowat; 370 371 pthread_attr_init(&attr); 372 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); 373 374 rcvlowat = chunk_size; 375 if (setsockopt(fdlisten, SOL_SOCKET, SO_RCVLOWAT, 376 &rcvlowat, sizeof(rcvlowat)) == -1) { 377 perror("setsockopt SO_RCVLOWAT"); 378 } 379 380 apply_rcvsnd_buf(fdlisten); 381 382 while (1) { 383 struct sockaddr_in addr; 384 socklen_t addrlen = sizeof(addr); 385 pthread_t th; 386 int fd, res; 387 388 fd = accept(fdlisten, (struct sockaddr *)&addr, &addrlen); 389 if (fd == -1) { 390 perror("accept"); 391 continue; 392 } 393 res = pthread_create(&th, &attr, child_thread, 394 (void *)(unsigned long)fd); 395 if (res) { 396 errno = res; 397 perror("pthread_create"); 398 close(fd); 399 } 400 } 401 } 402 403 /* Each thread should reserve a big enough vma to avoid 404 * spinlock collisions in ptl locks. 405 * This size is 2MB on x86_64, and is exported in /proc/meminfo. 406 */ 407 static unsigned long default_huge_page_size(void) 408 { 409 FILE *f = fopen("/proc/meminfo", "r"); 410 unsigned long hps = 0; 411 size_t linelen = 0; 412 char *line = NULL; 413 414 if (!f) 415 return 0; 416 while (getline(&line, &linelen, f) > 0) { 417 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { 418 hps <<= 10; 419 break; 420 } 421 } 422 free(line); 423 fclose(f); 424 return hps; 425 } 426 427 static void randomize(void *target, size_t count) 428 { 429 static int urandom = -1; 430 ssize_t got; 431 432 urandom = open("/dev/urandom", O_RDONLY); 433 if (urandom < 0) { 434 perror("open /dev/urandom"); 435 exit(1); 436 } 437 got = read(urandom, target, count); 438 if (got != count) { 439 perror("read /dev/urandom"); 440 exit(1); 441 } 442 } 443 444 int main(int argc, char *argv[]) 445 { 446 unsigned char digest[SHA256_DIGEST_LENGTH]; 447 struct sockaddr_storage listenaddr, addr; 448 unsigned int max_pacing_rate = 0; 449 EVP_MD_CTX *ctx = NULL; 450 unsigned char *buffer; 451 uint64_t total = 0; 452 char *host = NULL; 453 int fd, c, on = 1; 454 size_t buffer_sz; 455 int sflg = 0; 456 int mss = 0; 457 458 while ((c = getopt(argc, argv, "46p:svr:w:H:zxP:M:C:a:i")) != -1) { 459 switch (c) { 460 case '4': 461 cfg_family = PF_INET; 462 cfg_alen = sizeof(struct sockaddr_in); 463 break; 464 case '6': 465 cfg_family = PF_INET6; 466 cfg_alen = sizeof(struct sockaddr_in6); 467 break; 468 case 'p': 469 cfg_port = atoi(optarg); 470 break; 471 case 'H': 472 host = optarg; 473 break; 474 case 's': /* server : listen for incoming connections */ 475 sflg++; 476 break; 477 case 'r': 478 rcvbuf = atoi(optarg); 479 break; 480 case 'w': 481 sndbuf = atoi(optarg); 482 break; 483 case 'z': 484 zflg = 1; 485 break; 486 case 'M': 487 mss = atoi(optarg); 488 break; 489 case 'x': 490 xflg = 1; 491 break; 492 case 'P': 493 max_pacing_rate = atoi(optarg) ; 494 break; 495 case 'C': 496 chunk_size = atol(optarg); 497 break; 498 case 'a': 499 map_align = atol(optarg); 500 break; 501 case 'i': 502 integrity = 1; 503 break; 504 default: 505 exit(1); 506 } 507 } 508 if (!map_align) { 509 map_align = default_huge_page_size(); 510 /* if really /proc/meminfo is not helping, 511 * we use the default x86_64 hugepagesize. 512 */ 513 if (!map_align) 514 map_align = 2*1024*1024; 515 } 516 if (sflg) { 517 int fdlisten = socket(cfg_family, SOCK_STREAM, 0); 518 519 if (fdlisten == -1) { 520 perror("socket"); 521 exit(1); 522 } 523 apply_rcvsnd_buf(fdlisten); 524 setsockopt(fdlisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); 525 526 setup_sockaddr(cfg_family, host, &listenaddr); 527 528 if (mss && 529 setsockopt(fdlisten, IPPROTO_TCP, TCP_MAXSEG, 530 &mss, sizeof(mss)) == -1) { 531 perror("setsockopt TCP_MAXSEG"); 532 exit(1); 533 } 534 if (bind(fdlisten, (const struct sockaddr *)&listenaddr, cfg_alen) == -1) { 535 perror("bind"); 536 exit(1); 537 } 538 if (listen(fdlisten, 128) == -1) { 539 perror("listen"); 540 exit(1); 541 } 542 do_accept(fdlisten); 543 } 544 545 buffer = mmap_large_buffer(chunk_size, &buffer_sz); 546 if (buffer == MAP_FAILED) { 547 perror("mmap"); 548 exit(1); 549 } 550 551 fd = socket(cfg_family, SOCK_STREAM, 0); 552 if (fd == -1) { 553 perror("socket"); 554 exit(1); 555 } 556 apply_rcvsnd_buf(fd); 557 558 setup_sockaddr(cfg_family, host, &addr); 559 560 if (mss && 561 setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) == -1) { 562 perror("setsockopt TCP_MAXSEG"); 563 exit(1); 564 } 565 if (connect(fd, (const struct sockaddr *)&addr, cfg_alen) == -1) { 566 perror("connect"); 567 exit(1); 568 } 569 if (max_pacing_rate && 570 setsockopt(fd, SOL_SOCKET, SO_MAX_PACING_RATE, 571 &max_pacing_rate, sizeof(max_pacing_rate)) == -1) 572 perror("setsockopt SO_MAX_PACING_RATE"); 573 574 if (zflg && setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY, 575 &on, sizeof(on)) == -1) { 576 perror("setsockopt SO_ZEROCOPY, (-z option disabled)"); 577 zflg = 0; 578 } 579 if (integrity) { 580 randomize(buffer, buffer_sz); 581 ctx = EVP_MD_CTX_new(); 582 if (!ctx) { 583 perror("cannot enable SHA computing"); 584 exit(1); 585 } 586 EVP_DigestInit_ex(ctx, EVP_sha256(), NULL); 587 } 588 while (total < FILE_SZ) { 589 size_t offset = total % chunk_size; 590 int64_t wr = FILE_SZ - total; 591 592 if (wr > chunk_size - offset) 593 wr = chunk_size - offset; 594 /* Note : we just want to fill the pipe with random bytes */ 595 wr = send(fd, buffer + offset, 596 (size_t)wr, zflg ? MSG_ZEROCOPY : 0); 597 if (wr <= 0) 598 break; 599 if (integrity) 600 EVP_DigestUpdate(ctx, buffer + offset, wr); 601 total += wr; 602 } 603 if (integrity && total == FILE_SZ) { 604 EVP_DigestFinal_ex(ctx, digest, &digest_len); 605 send(fd, digest, (size_t)SHA256_DIGEST_LENGTH, 0); 606 } 607 if (ctx) 608 EVP_MD_CTX_free(ctx); 609 close(fd); 610 munmap(buffer, buffer_sz); 611 return 0; 612 } 613