1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <sys/socket.h>
6 #include <sys/ioctl.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
9 #include <arpa/inet.h>
10 #include <unistd.h>
11 #include <string.h>
12 #include <errno.h>
13 #include <stdbool.h>
14 #include <signal.h>
15 #include <fcntl.h>
16 #include <sys/wait.h>
17 #include <time.h>
18 #include <sched.h>
19
20 #include <sys/time.h>
21 #include <sys/types.h>
22 #include <sys/sendfile.h>
23
24 #include <linux/netlink.h>
25 #include <linux/socket.h>
26 #include <linux/sock_diag.h>
27 #include <linux/bpf.h>
28 #include <linux/if_link.h>
29 #include <linux/tls.h>
30 #include <assert.h>
31 #include <libgen.h>
32
33 #include <getopt.h>
34
35 #include <bpf/bpf.h>
36 #include <bpf/libbpf.h>
37
38 #include "bpf_util.h"
39 #include "cgroup_helpers.h"
40
41 int running;
42 static void running_handler(int a);
43
44 #ifndef TCP_ULP
45 # define TCP_ULP 31
46 #endif
47 #ifndef SOL_TLS
48 # define SOL_TLS 282
49 #endif
50
51 /* randomly selected ports for testing on lo */
52 #define S1_PORT 10000
53 #define S2_PORT 10001
54
55 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o"
56 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
57 #define CG_PATH "/sockmap"
58
59 /* global sockets */
60 int s1, s2, c1, c2, p1, p2;
61 int test_cnt;
62 int passed;
63 int failed;
64 int map_fd[9];
65 struct bpf_map *maps[9];
66 struct bpf_program *progs[9];
67 struct bpf_link *links[9];
68
69 int txmsg_pass;
70 int txmsg_redir;
71 int txmsg_drop;
72 int txmsg_apply;
73 int txmsg_cork;
74 int txmsg_start;
75 int txmsg_end;
76 int txmsg_start_push;
77 int txmsg_end_push;
78 int txmsg_start_pop;
79 int txmsg_pop;
80 int txmsg_ingress;
81 int txmsg_redir_skb;
82 int txmsg_ktls_skb;
83 int txmsg_ktls_skb_drop;
84 int txmsg_ktls_skb_redir;
85 int ktls;
86 int peek_flag;
87 int skb_use_parser;
88 int txmsg_omit_skb_parser;
89
90 static const struct option long_options[] = {
91 {"help", no_argument, NULL, 'h' },
92 {"cgroup", required_argument, NULL, 'c' },
93 {"rate", required_argument, NULL, 'r' },
94 {"verbose", optional_argument, NULL, 'v' },
95 {"iov_count", required_argument, NULL, 'i' },
96 {"length", required_argument, NULL, 'l' },
97 {"test", required_argument, NULL, 't' },
98 {"data_test", no_argument, NULL, 'd' },
99 {"txmsg", no_argument, &txmsg_pass, 1 },
100 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
101 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
102 {"txmsg_apply", required_argument, NULL, 'a'},
103 {"txmsg_cork", required_argument, NULL, 'k'},
104 {"txmsg_start", required_argument, NULL, 's'},
105 {"txmsg_end", required_argument, NULL, 'e'},
106 {"txmsg_start_push", required_argument, NULL, 'p'},
107 {"txmsg_end_push", required_argument, NULL, 'q'},
108 {"txmsg_start_pop", required_argument, NULL, 'w'},
109 {"txmsg_pop", required_argument, NULL, 'x'},
110 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
111 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
112 {"ktls", no_argument, &ktls, 1 },
113 {"peek", no_argument, &peek_flag, 1 },
114 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
115 {"whitelist", required_argument, NULL, 'n' },
116 {"blacklist", required_argument, NULL, 'b' },
117 {0, 0, NULL, 0 }
118 };
119
120 struct test_env {
121 const char *type;
122 const char *subtest;
123 const char *prepend;
124
125 int test_num;
126 int subtest_num;
127
128 int succ_cnt;
129 int fail_cnt;
130 int fail_last;
131 };
132
133 struct test_env env;
134
135 struct sockmap_options {
136 int verbose;
137 bool base;
138 bool sendpage;
139 bool data_test;
140 bool drop_expected;
141 bool check_recved_len;
142 bool tx_wait_mem;
143 int iov_count;
144 int iov_length;
145 int rate;
146 char *map;
147 char *whitelist;
148 char *blacklist;
149 char *prepend;
150 };
151
152 struct _test {
153 char *title;
154 void (*tester)(int cg_fd, struct sockmap_options *opt);
155 };
156
test_start(void)157 static void test_start(void)
158 {
159 env.subtest_num++;
160 }
161
test_fail(void)162 static void test_fail(void)
163 {
164 env.fail_cnt++;
165 }
166
test_pass(void)167 static void test_pass(void)
168 {
169 env.succ_cnt++;
170 }
171
test_reset(void)172 static void test_reset(void)
173 {
174 txmsg_start = txmsg_end = 0;
175 txmsg_start_pop = txmsg_pop = 0;
176 txmsg_start_push = txmsg_end_push = 0;
177 txmsg_pass = txmsg_drop = txmsg_redir = 0;
178 txmsg_apply = txmsg_cork = 0;
179 txmsg_ingress = txmsg_redir_skb = 0;
180 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
181 txmsg_omit_skb_parser = 0;
182 skb_use_parser = 0;
183 }
184
test_start_subtest(const struct _test * t,struct sockmap_options * o)185 static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
186 {
187 env.type = o->map;
188 env.subtest = t->title;
189 env.prepend = o->prepend;
190 env.test_num++;
191 env.subtest_num = 0;
192 env.fail_last = env.fail_cnt;
193 test_reset();
194 return 0;
195 }
196
test_end_subtest(void)197 static void test_end_subtest(void)
198 {
199 int error = env.fail_cnt - env.fail_last;
200 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
201
202 if (!error)
203 test_pass();
204
205 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
206 env.test_num, env.subtest_num,
207 !type ? "sockmap" : "sockhash",
208 env.prepend ? : "",
209 env.subtest, error ? "FAIL" : "OK");
210 }
211
test_print_results(void)212 static void test_print_results(void)
213 {
214 fprintf(stdout, "Pass: %d Fail: %d\n",
215 env.succ_cnt, env.fail_cnt);
216 }
217
usage(char * argv[])218 static void usage(char *argv[])
219 {
220 int i;
221
222 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
223 printf(" options:\n");
224 for (i = 0; long_options[i].name != 0; i++) {
225 printf(" --%-12s", long_options[i].name);
226 if (long_options[i].flag != NULL)
227 printf(" flag (internal value:%d)\n",
228 *long_options[i].flag);
229 else
230 printf(" -%c\n", long_options[i].val);
231 }
232 printf("\n");
233 }
234
sock_to_string(int s)235 char *sock_to_string(int s)
236 {
237 if (s == c1)
238 return "client1";
239 else if (s == c2)
240 return "client2";
241 else if (s == s1)
242 return "server1";
243 else if (s == s2)
244 return "server2";
245 else if (s == p1)
246 return "peer1";
247 else if (s == p2)
248 return "peer2";
249 else
250 return "unknown";
251 }
252
sockmap_init_ktls(int verbose,int s)253 static int sockmap_init_ktls(int verbose, int s)
254 {
255 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
256 .info = {
257 .version = TLS_1_2_VERSION,
258 .cipher_type = TLS_CIPHER_AES_GCM_128,
259 },
260 };
261 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
262 .info = {
263 .version = TLS_1_2_VERSION,
264 .cipher_type = TLS_CIPHER_AES_GCM_128,
265 },
266 };
267 int so_buf = 6553500;
268 int err;
269
270 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
271 if (err) {
272 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
273 return -EINVAL;
274 }
275 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
276 if (err) {
277 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
278 return -EINVAL;
279 }
280 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
281 if (err) {
282 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
283 return -EINVAL;
284 }
285 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
286 if (err) {
287 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
288 return -EINVAL;
289 }
290 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
291 if (err) {
292 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
293 return -EINVAL;
294 }
295
296 if (verbose)
297 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
298 return 0;
299 }
sockmap_init_sockets(int verbose)300 static int sockmap_init_sockets(int verbose)
301 {
302 int i, err, one = 1;
303 struct sockaddr_in addr;
304 int *fds[4] = {&s1, &s2, &c1, &c2};
305
306 s1 = s2 = p1 = p2 = c1 = c2 = 0;
307
308 /* Init sockets */
309 for (i = 0; i < 4; i++) {
310 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
311 if (*fds[i] < 0) {
312 perror("socket s1 failed()");
313 return errno;
314 }
315 }
316
317 /* Allow reuse */
318 for (i = 0; i < 2; i++) {
319 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
320 (char *)&one, sizeof(one));
321 if (err) {
322 perror("setsockopt failed()");
323 return errno;
324 }
325 }
326
327 /* Non-blocking sockets */
328 for (i = 0; i < 2; i++) {
329 err = ioctl(*fds[i], FIONBIO, (char *)&one);
330 if (err < 0) {
331 perror("ioctl s1 failed()");
332 return errno;
333 }
334 }
335
336 /* Bind server sockets */
337 memset(&addr, 0, sizeof(struct sockaddr_in));
338 addr.sin_family = AF_INET;
339 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
340
341 addr.sin_port = htons(S1_PORT);
342 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
343 if (err < 0) {
344 perror("bind s1 failed()");
345 return errno;
346 }
347
348 addr.sin_port = htons(S2_PORT);
349 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
350 if (err < 0) {
351 perror("bind s2 failed()");
352 return errno;
353 }
354
355 /* Listen server sockets */
356 addr.sin_port = htons(S1_PORT);
357 err = listen(s1, 32);
358 if (err < 0) {
359 perror("listen s1 failed()");
360 return errno;
361 }
362
363 addr.sin_port = htons(S2_PORT);
364 err = listen(s2, 32);
365 if (err < 0) {
366 perror("listen s1 failed()");
367 return errno;
368 }
369
370 /* Initiate Connect */
371 addr.sin_port = htons(S1_PORT);
372 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
373 if (err < 0 && errno != EINPROGRESS) {
374 perror("connect c1 failed()");
375 return errno;
376 }
377
378 addr.sin_port = htons(S2_PORT);
379 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
380 if (err < 0 && errno != EINPROGRESS) {
381 perror("connect c2 failed()");
382 return errno;
383 } else if (err < 0) {
384 err = 0;
385 }
386
387 /* Accept Connecrtions */
388 p1 = accept(s1, NULL, NULL);
389 if (p1 < 0) {
390 perror("accept s1 failed()");
391 return errno;
392 }
393
394 p2 = accept(s2, NULL, NULL);
395 if (p2 < 0) {
396 perror("accept s1 failed()");
397 return errno;
398 }
399
400 if (verbose > 1) {
401 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
402 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
403 c1, s1, c2, s2);
404 }
405 return 0;
406 }
407
408 struct msg_stats {
409 size_t bytes_sent;
410 size_t bytes_recvd;
411 struct timespec start;
412 struct timespec end;
413 };
414
msg_loop_sendpage(int fd,int iov_length,int cnt,struct msg_stats * s,struct sockmap_options * opt)415 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
416 struct msg_stats *s,
417 struct sockmap_options *opt)
418 {
419 bool drop = opt->drop_expected;
420 unsigned char k = 0;
421 FILE *file;
422 int i, fp;
423
424 file = tmpfile();
425 if (!file) {
426 perror("create file for sendpage");
427 return 1;
428 }
429 for (i = 0; i < iov_length * cnt; i++, k++)
430 fwrite(&k, sizeof(char), 1, file);
431 fflush(file);
432 fseek(file, 0, SEEK_SET);
433
434 fp = fileno(file);
435
436 clock_gettime(CLOCK_MONOTONIC, &s->start);
437 for (i = 0; i < cnt; i++) {
438 int sent;
439
440 errno = 0;
441 sent = sendfile(fd, fp, NULL, iov_length);
442
443 if (!drop && sent < 0) {
444 perror("sendpage loop error");
445 fclose(file);
446 return sent;
447 } else if (drop && sent >= 0) {
448 printf("sendpage loop error expected: %i errno %i\n",
449 sent, errno);
450 fclose(file);
451 return -EIO;
452 }
453
454 if (sent > 0)
455 s->bytes_sent += sent;
456 }
457 clock_gettime(CLOCK_MONOTONIC, &s->end);
458 fclose(file);
459 return 0;
460 }
461
msg_free_iov(struct msghdr * msg)462 static void msg_free_iov(struct msghdr *msg)
463 {
464 int i;
465
466 for (i = 0; i < msg->msg_iovlen; i++)
467 free(msg->msg_iov[i].iov_base);
468 free(msg->msg_iov);
469 msg->msg_iov = NULL;
470 msg->msg_iovlen = 0;
471 }
472
msg_alloc_iov(struct msghdr * msg,int iov_count,int iov_length,bool data,bool xmit)473 static int msg_alloc_iov(struct msghdr *msg,
474 int iov_count, int iov_length,
475 bool data, bool xmit)
476 {
477 unsigned char k = 0;
478 struct iovec *iov;
479 int i;
480
481 iov = calloc(iov_count, sizeof(struct iovec));
482 if (!iov)
483 return errno;
484
485 for (i = 0; i < iov_count; i++) {
486 unsigned char *d = calloc(iov_length, sizeof(char));
487
488 if (!d) {
489 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
490 goto unwind_iov;
491 }
492 iov[i].iov_base = d;
493 iov[i].iov_len = iov_length;
494
495 if (data && xmit) {
496 int j;
497
498 for (j = 0; j < iov_length; j++)
499 d[j] = k++;
500 }
501 }
502
503 msg->msg_iov = iov;
504 msg->msg_iovlen = iov_count;
505
506 return 0;
507 unwind_iov:
508 for (i--; i >= 0 ; i--)
509 free(msg->msg_iov[i].iov_base);
510 return -ENOMEM;
511 }
512
msg_verify_data(struct msghdr * msg,int size,int chunk_sz)513 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
514 {
515 int i, j = 0, bytes_cnt = 0;
516 unsigned char k = 0;
517
518 for (i = 0; i < msg->msg_iovlen; i++) {
519 unsigned char *d = msg->msg_iov[i].iov_base;
520
521 /* Special case test for skb ingress + ktls */
522 if (i == 0 && txmsg_ktls_skb) {
523 if (msg->msg_iov[i].iov_len < 4)
524 return -EIO;
525 if (memcmp(d, "PASS", 4) != 0) {
526 fprintf(stderr,
527 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
528 i, 0, d[0], d[1], d[2], d[3]);
529 return -EIO;
530 }
531 j = 4; /* advance index past PASS header */
532 }
533
534 for (; j < msg->msg_iov[i].iov_len && size; j++) {
535 if (d[j] != k++) {
536 fprintf(stderr,
537 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
538 i, j, d[j], k - 1, d[j+1], k);
539 return -EIO;
540 }
541 bytes_cnt++;
542 if (bytes_cnt == chunk_sz) {
543 k = 0;
544 bytes_cnt = 0;
545 }
546 size--;
547 }
548 }
549 return 0;
550 }
551
msg_loop(int fd,int iov_count,int iov_length,int cnt,struct msg_stats * s,bool tx,struct sockmap_options * opt)552 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
553 struct msg_stats *s, bool tx,
554 struct sockmap_options *opt)
555 {
556 struct msghdr msg = {0}, msg_peek = {0};
557 int err, i, flags = MSG_NOSIGNAL;
558 bool drop = opt->drop_expected;
559 bool data = opt->data_test;
560 int iov_alloc_length = iov_length;
561
562 if (!tx && opt->check_recved_len)
563 iov_alloc_length *= 2;
564
565 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
566 if (err)
567 goto out_errno;
568 if (peek_flag) {
569 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
570 if (err)
571 goto out_errno;
572 }
573
574 if (tx) {
575 clock_gettime(CLOCK_MONOTONIC, &s->start);
576 for (i = 0; i < cnt; i++) {
577 int sent;
578
579 errno = 0;
580 sent = sendmsg(fd, &msg, flags);
581
582 if (!drop && sent < 0) {
583 if (opt->tx_wait_mem && errno == EACCES) {
584 errno = 0;
585 goto out_errno;
586 }
587 perror("sendmsg loop error");
588 goto out_errno;
589 } else if (drop && sent >= 0) {
590 fprintf(stderr,
591 "sendmsg loop error expected: %i errno %i\n",
592 sent, errno);
593 errno = -EIO;
594 goto out_errno;
595 }
596 if (sent > 0)
597 s->bytes_sent += sent;
598 }
599 clock_gettime(CLOCK_MONOTONIC, &s->end);
600 } else {
601 int slct, recvp = 0, recv, max_fd = fd;
602 float total_bytes, txmsg_pop_total;
603 int fd_flags = O_NONBLOCK;
604 struct timeval timeout;
605 fd_set w;
606
607 fcntl(fd, fd_flags);
608 /* Account for pop bytes noting each iteration of apply will
609 * call msg_pop_data helper so we need to account for this
610 * by calculating the number of apply iterations. Note user
611 * of the tool can create cases where no data is sent by
612 * manipulating pop/push/pull/etc. For example txmsg_apply 1
613 * with txmsg_pop 1 will try to apply 1B at a time but each
614 * iteration will then pop 1B so no data will ever be sent.
615 * This is really only useful for testing edge cases in code
616 * paths.
617 */
618 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
619 if (txmsg_apply)
620 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
621 else
622 txmsg_pop_total = txmsg_pop * cnt;
623 total_bytes -= txmsg_pop_total;
624 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
625 if (err < 0)
626 perror("recv start time");
627 while (s->bytes_recvd < total_bytes) {
628 if (txmsg_cork) {
629 timeout.tv_sec = 0;
630 timeout.tv_usec = 300000;
631 } else {
632 timeout.tv_sec = 3;
633 timeout.tv_usec = 0;
634 }
635
636 /* FD sets */
637 FD_ZERO(&w);
638 FD_SET(fd, &w);
639
640 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
641 if (slct == -1) {
642 perror("select()");
643 clock_gettime(CLOCK_MONOTONIC, &s->end);
644 goto out_errno;
645 } else if (!slct) {
646 if (opt->verbose)
647 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
648 errno = -EIO;
649 clock_gettime(CLOCK_MONOTONIC, &s->end);
650 goto out_errno;
651 }
652
653 if (opt->tx_wait_mem) {
654 FD_ZERO(&w);
655 FD_SET(fd, &w);
656 slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
657 errno = 0;
658 close(fd);
659 goto out_errno;
660 }
661
662 errno = 0;
663 if (peek_flag) {
664 flags |= MSG_PEEK;
665 recvp = recvmsg(fd, &msg_peek, flags);
666 if (recvp < 0) {
667 if (errno != EWOULDBLOCK) {
668 clock_gettime(CLOCK_MONOTONIC, &s->end);
669 goto out_errno;
670 }
671 }
672 flags = 0;
673 }
674
675 recv = recvmsg(fd, &msg, flags);
676 if (recv < 0) {
677 if (errno != EWOULDBLOCK) {
678 clock_gettime(CLOCK_MONOTONIC, &s->end);
679 perror("recv failed()");
680 goto out_errno;
681 }
682 }
683
684 if (recv > 0)
685 s->bytes_recvd += recv;
686
687 if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
688 errno = EMSGSIZE;
689 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
690 s->bytes_recvd, total_bytes);
691 goto out_errno;
692 }
693
694 if (data) {
695 int chunk_sz = opt->sendpage ?
696 iov_length * cnt :
697 iov_length * iov_count;
698
699 errno = msg_verify_data(&msg, recv, chunk_sz);
700 if (errno) {
701 perror("data verify msg failed");
702 goto out_errno;
703 }
704 if (recvp) {
705 errno = msg_verify_data(&msg_peek,
706 recvp,
707 chunk_sz);
708 if (errno) {
709 perror("data verify msg_peek failed");
710 goto out_errno;
711 }
712 }
713 }
714 }
715 clock_gettime(CLOCK_MONOTONIC, &s->end);
716 }
717
718 msg_free_iov(&msg);
719 msg_free_iov(&msg_peek);
720 return err;
721 out_errno:
722 msg_free_iov(&msg);
723 msg_free_iov(&msg_peek);
724 return errno;
725 }
726
727 static float giga = 1000000000;
728
sentBps(struct msg_stats s)729 static inline float sentBps(struct msg_stats s)
730 {
731 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
732 }
733
recvdBps(struct msg_stats s)734 static inline float recvdBps(struct msg_stats s)
735 {
736 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
737 }
738
sendmsg_test(struct sockmap_options * opt)739 static int sendmsg_test(struct sockmap_options *opt)
740 {
741 float sent_Bps = 0, recvd_Bps = 0;
742 int rx_fd, txpid, rxpid, err = 0;
743 struct msg_stats s = {0};
744 int iov_count = opt->iov_count;
745 int iov_buf = opt->iov_length;
746 int rx_status, tx_status;
747 int cnt = opt->rate;
748
749 errno = 0;
750
751 if (opt->base)
752 rx_fd = p1;
753 else
754 rx_fd = p2;
755
756 if (ktls) {
757 /* Redirecting into non-TLS socket which sends into a TLS
758 * socket is not a valid test. So in this case lets not
759 * enable kTLS but still run the test.
760 */
761 if (!txmsg_redir || txmsg_ingress) {
762 err = sockmap_init_ktls(opt->verbose, rx_fd);
763 if (err)
764 return err;
765 }
766 err = sockmap_init_ktls(opt->verbose, c1);
767 if (err)
768 return err;
769 }
770
771 if (opt->tx_wait_mem) {
772 struct timeval timeout;
773 int rxtx_buf_len = 1024;
774
775 timeout.tv_sec = 3;
776 timeout.tv_usec = 0;
777
778 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
779 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
780 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
781 if (err) {
782 perror("setsockopt failed()");
783 return errno;
784 }
785 }
786
787 rxpid = fork();
788 if (rxpid == 0) {
789 if (txmsg_pop || txmsg_start_pop)
790 iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
791 if (opt->drop_expected || txmsg_ktls_skb_drop)
792 _exit(0);
793
794 if (!iov_buf) /* zero bytes sent case */
795 _exit(0);
796
797 if (opt->sendpage)
798 iov_count = 1;
799 err = msg_loop(rx_fd, iov_count, iov_buf,
800 cnt, &s, false, opt);
801 if (opt->verbose > 1)
802 fprintf(stderr,
803 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
804 iov_count, iov_buf, cnt, err);
805 if (s.end.tv_sec - s.start.tv_sec) {
806 sent_Bps = sentBps(s);
807 recvd_Bps = recvdBps(s);
808 }
809 if (opt->verbose > 1)
810 fprintf(stdout,
811 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
812 s.bytes_sent, sent_Bps, sent_Bps/giga,
813 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
814 peek_flag ? "(peek_msg)" : "");
815 if (err && txmsg_cork)
816 err = 0;
817 exit(err ? 1 : 0);
818 } else if (rxpid == -1) {
819 perror("msg_loop_rx");
820 return errno;
821 }
822
823 if (opt->tx_wait_mem)
824 close(c2);
825
826 txpid = fork();
827 if (txpid == 0) {
828 if (opt->sendpage)
829 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
830 else
831 err = msg_loop(c1, iov_count, iov_buf,
832 cnt, &s, true, opt);
833
834 if (err)
835 fprintf(stderr,
836 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
837 iov_count, iov_buf, cnt, err);
838 if (s.end.tv_sec - s.start.tv_sec) {
839 sent_Bps = sentBps(s);
840 recvd_Bps = recvdBps(s);
841 }
842 if (opt->verbose > 1)
843 fprintf(stdout,
844 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
845 s.bytes_sent, sent_Bps, sent_Bps/giga,
846 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
847 exit(err ? 1 : 0);
848 } else if (txpid == -1) {
849 perror("msg_loop_tx");
850 return errno;
851 }
852
853 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
854 assert(waitpid(txpid, &tx_status, 0) == txpid);
855 if (WIFEXITED(rx_status)) {
856 err = WEXITSTATUS(rx_status);
857 if (err) {
858 fprintf(stderr, "rx thread exited with err %d.\n", err);
859 goto out;
860 }
861 }
862 if (WIFEXITED(tx_status)) {
863 err = WEXITSTATUS(tx_status);
864 if (err)
865 fprintf(stderr, "tx thread exited with err %d.\n", err);
866 }
867 out:
868 return err;
869 }
870
forever_ping_pong(int rate,struct sockmap_options * opt)871 static int forever_ping_pong(int rate, struct sockmap_options *opt)
872 {
873 struct timeval timeout;
874 char buf[1024] = {0};
875 int sc;
876
877 timeout.tv_sec = 10;
878 timeout.tv_usec = 0;
879
880 /* Ping/Pong data from client to server */
881 sc = send(c1, buf, sizeof(buf), 0);
882 if (sc < 0) {
883 perror("send failed()");
884 return sc;
885 }
886
887 do {
888 int s, rc, i, max_fd = p2;
889 fd_set w;
890
891 /* FD sets */
892 FD_ZERO(&w);
893 FD_SET(c1, &w);
894 FD_SET(c2, &w);
895 FD_SET(p1, &w);
896 FD_SET(p2, &w);
897
898 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
899 if (s == -1) {
900 perror("select()");
901 break;
902 } else if (!s) {
903 fprintf(stderr, "unexpected timeout\n");
904 break;
905 }
906
907 for (i = 0; i <= max_fd && s > 0; ++i) {
908 if (!FD_ISSET(i, &w))
909 continue;
910
911 s--;
912
913 rc = recv(i, buf, sizeof(buf), 0);
914 if (rc < 0) {
915 if (errno != EWOULDBLOCK) {
916 perror("recv failed()");
917 return rc;
918 }
919 }
920
921 if (rc == 0) {
922 close(i);
923 break;
924 }
925
926 sc = send(i, buf, rc, 0);
927 if (sc < 0) {
928 perror("send failed()");
929 return sc;
930 }
931 }
932
933 if (rate)
934 sleep(rate);
935
936 if (opt->verbose) {
937 printf(".");
938 fflush(stdout);
939
940 }
941 } while (running);
942
943 return 0;
944 }
945
946 enum {
947 SELFTESTS,
948 PING_PONG,
949 SENDMSG,
950 BASE,
951 BASE_SENDPAGE,
952 SENDPAGE,
953 };
954
run_options(struct sockmap_options * options,int cg_fd,int test)955 static int run_options(struct sockmap_options *options, int cg_fd, int test)
956 {
957 int i, key, next_key, err, zero = 0;
958 struct bpf_program *tx_prog;
959
960 /* If base test skip BPF setup */
961 if (test == BASE || test == BASE_SENDPAGE)
962 goto run;
963
964 /* Attach programs to sockmap */
965 if (!txmsg_omit_skb_parser) {
966 links[0] = bpf_program__attach_sockmap(progs[0], map_fd[0]);
967 if (!links[0]) {
968 fprintf(stderr,
969 "ERROR: bpf_program__attach_sockmap (sockmap %i->%i): (%s)\n",
970 bpf_program__fd(progs[0]), map_fd[0], strerror(errno));
971 return -1;
972 }
973 }
974
975 links[1] = bpf_program__attach_sockmap(progs[1], map_fd[0]);
976 if (!links[1]) {
977 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (sockmap): (%s)\n",
978 strerror(errno));
979 return -1;
980 }
981
982 /* Attach programs to TLS sockmap */
983 if (txmsg_ktls_skb) {
984 if (!txmsg_omit_skb_parser) {
985 links[2] = bpf_program__attach_sockmap(progs[0], map_fd[8]);
986 if (!links[2]) {
987 fprintf(stderr,
988 "ERROR: bpf_program__attach_sockmap (TLS sockmap %i->%i): (%s)\n",
989 bpf_program__fd(progs[0]), map_fd[8], strerror(errno));
990 return -1;
991 }
992 }
993
994 links[3] = bpf_program__attach_sockmap(progs[2], map_fd[8]);
995 if (!links[3]) {
996 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (TLS sockmap): (%s)\n",
997 strerror(errno));
998 return -1;
999 }
1000 }
1001
1002 /* Attach to cgroups */
1003 err = bpf_prog_attach(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS, 0);
1004 if (err) {
1005 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
1006 err, strerror(errno));
1007 return err;
1008 }
1009
1010 run:
1011 err = sockmap_init_sockets(options->verbose);
1012 if (err) {
1013 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
1014 goto out;
1015 }
1016
1017 /* Attach txmsg program to sockmap */
1018 if (txmsg_pass)
1019 tx_prog = progs[4];
1020 else if (txmsg_redir)
1021 tx_prog = progs[5];
1022 else if (txmsg_apply)
1023 tx_prog = progs[6];
1024 else if (txmsg_cork)
1025 tx_prog = progs[7];
1026 else if (txmsg_drop)
1027 tx_prog = progs[8];
1028 else
1029 tx_prog = NULL;
1030
1031 if (tx_prog) {
1032 int redir_fd;
1033
1034 links[4] = bpf_program__attach_sockmap(tx_prog, map_fd[1]);
1035 if (!links[4]) {
1036 fprintf(stderr,
1037 "ERROR: bpf_program__attach_sockmap (txmsg): (%s)\n",
1038 strerror(errno));
1039 err = -1;
1040 goto out;
1041 }
1042
1043 i = 0;
1044 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1045 if (err) {
1046 fprintf(stderr,
1047 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1048 err, strerror(errno));
1049 goto out;
1050 }
1051
1052 if (txmsg_redir)
1053 redir_fd = c2;
1054 else
1055 redir_fd = c1;
1056
1057 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1058 if (err) {
1059 fprintf(stderr,
1060 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1061 err, strerror(errno));
1062 goto out;
1063 }
1064
1065 if (txmsg_apply) {
1066 err = bpf_map_update_elem(map_fd[3],
1067 &i, &txmsg_apply, BPF_ANY);
1068 if (err) {
1069 fprintf(stderr,
1070 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
1071 err, strerror(errno));
1072 goto out;
1073 }
1074 }
1075
1076 if (txmsg_cork) {
1077 err = bpf_map_update_elem(map_fd[4],
1078 &i, &txmsg_cork, BPF_ANY);
1079 if (err) {
1080 fprintf(stderr,
1081 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
1082 err, strerror(errno));
1083 goto out;
1084 }
1085 }
1086
1087 if (txmsg_start) {
1088 err = bpf_map_update_elem(map_fd[5],
1089 &i, &txmsg_start, BPF_ANY);
1090 if (err) {
1091 fprintf(stderr,
1092 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
1093 err, strerror(errno));
1094 goto out;
1095 }
1096 }
1097
1098 if (txmsg_end) {
1099 i = 1;
1100 err = bpf_map_update_elem(map_fd[5],
1101 &i, &txmsg_end, BPF_ANY);
1102 if (err) {
1103 fprintf(stderr,
1104 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
1105 err, strerror(errno));
1106 goto out;
1107 }
1108 }
1109
1110 if (txmsg_start_push) {
1111 i = 2;
1112 err = bpf_map_update_elem(map_fd[5],
1113 &i, &txmsg_start_push, BPF_ANY);
1114 if (err) {
1115 fprintf(stderr,
1116 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
1117 err, strerror(errno));
1118 goto out;
1119 }
1120 }
1121
1122 if (txmsg_end_push) {
1123 i = 3;
1124 err = bpf_map_update_elem(map_fd[5],
1125 &i, &txmsg_end_push, BPF_ANY);
1126 if (err) {
1127 fprintf(stderr,
1128 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
1129 txmsg_end_push, i, err, strerror(errno));
1130 goto out;
1131 }
1132 }
1133
1134 if (txmsg_start_pop) {
1135 i = 4;
1136 err = bpf_map_update_elem(map_fd[5],
1137 &i, &txmsg_start_pop, BPF_ANY);
1138 if (err) {
1139 fprintf(stderr,
1140 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
1141 txmsg_start_pop, i, err, strerror(errno));
1142 goto out;
1143 }
1144 } else {
1145 i = 4;
1146 bpf_map_update_elem(map_fd[5],
1147 &i, &txmsg_start_pop, BPF_ANY);
1148 }
1149
1150 if (txmsg_pop) {
1151 i = 5;
1152 err = bpf_map_update_elem(map_fd[5],
1153 &i, &txmsg_pop, BPF_ANY);
1154 if (err) {
1155 fprintf(stderr,
1156 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
1157 txmsg_pop, i, err, strerror(errno));
1158 goto out;
1159 }
1160 } else {
1161 i = 5;
1162 bpf_map_update_elem(map_fd[5],
1163 &i, &txmsg_pop, BPF_ANY);
1164
1165 }
1166
1167 if (txmsg_ingress) {
1168 int in = BPF_F_INGRESS;
1169
1170 i = 0;
1171 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1172 if (err) {
1173 fprintf(stderr,
1174 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1175 err, strerror(errno));
1176 }
1177 i = 1;
1178 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1179 if (err) {
1180 fprintf(stderr,
1181 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1182 err, strerror(errno));
1183 }
1184 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1185 if (err) {
1186 fprintf(stderr,
1187 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1188 err, strerror(errno));
1189 }
1190
1191 i = 2;
1192 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1193 if (err) {
1194 fprintf(stderr,
1195 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1196 err, strerror(errno));
1197 }
1198 }
1199
1200 if (txmsg_ktls_skb) {
1201 int ingress = BPF_F_INGRESS;
1202
1203 i = 0;
1204 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1205 if (err) {
1206 fprintf(stderr,
1207 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1208 err, strerror(errno));
1209 }
1210
1211 if (txmsg_ktls_skb_redir) {
1212 i = 1;
1213 err = bpf_map_update_elem(map_fd[7],
1214 &i, &ingress, BPF_ANY);
1215 if (err) {
1216 fprintf(stderr,
1217 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1218 err, strerror(errno));
1219 }
1220 }
1221
1222 if (txmsg_ktls_skb_drop) {
1223 i = 1;
1224 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1225 }
1226 }
1227
1228 if (txmsg_redir_skb) {
1229 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1230 p2 : p1;
1231 int ingress = BPF_F_INGRESS;
1232
1233 i = 0;
1234 err = bpf_map_update_elem(map_fd[7],
1235 &i, &ingress, BPF_ANY);
1236 if (err) {
1237 fprintf(stderr,
1238 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1239 err, strerror(errno));
1240 }
1241
1242 i = 3;
1243 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1244 if (err) {
1245 fprintf(stderr,
1246 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1247 err, strerror(errno));
1248 }
1249 }
1250 }
1251
1252 if (skb_use_parser) {
1253 i = 2;
1254 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1255 }
1256
1257 if (txmsg_drop)
1258 options->drop_expected = true;
1259
1260 if (test == PING_PONG)
1261 err = forever_ping_pong(options->rate, options);
1262 else if (test == SENDMSG) {
1263 options->base = false;
1264 options->sendpage = false;
1265 err = sendmsg_test(options);
1266 } else if (test == SENDPAGE) {
1267 options->base = false;
1268 options->sendpage = true;
1269 err = sendmsg_test(options);
1270 } else if (test == BASE) {
1271 options->base = true;
1272 options->sendpage = false;
1273 err = sendmsg_test(options);
1274 } else if (test == BASE_SENDPAGE) {
1275 options->base = true;
1276 options->sendpage = true;
1277 err = sendmsg_test(options);
1278 } else
1279 fprintf(stderr, "unknown test\n");
1280 out:
1281 /* Detatch and zero all the maps */
1282 bpf_prog_detach2(bpf_program__fd(progs[3]), cg_fd, BPF_CGROUP_SOCK_OPS);
1283
1284 for (i = 0; i < ARRAY_SIZE(links); i++) {
1285 if (links[i])
1286 bpf_link__detach(links[i]);
1287 }
1288
1289 for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1290 key = next_key = 0;
1291 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1292 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1293 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1294 key = next_key;
1295 }
1296 }
1297
1298 close(s1);
1299 close(s2);
1300 close(p1);
1301 close(p2);
1302 close(c1);
1303 close(c2);
1304 return err;
1305 }
1306
test_to_str(int test)1307 static char *test_to_str(int test)
1308 {
1309 switch (test) {
1310 case SENDMSG:
1311 return "sendmsg";
1312 case SENDPAGE:
1313 return "sendpage";
1314 }
1315 return "unknown";
1316 }
1317
append_str(char * dst,const char * src,size_t dst_cap)1318 static void append_str(char *dst, const char *src, size_t dst_cap)
1319 {
1320 size_t avail = dst_cap - strlen(dst);
1321
1322 if (avail <= 1) /* just zero byte could be written */
1323 return;
1324
1325 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1326 }
1327
1328 #define OPTSTRING 60
test_options(char * options)1329 static void test_options(char *options)
1330 {
1331 char tstr[OPTSTRING];
1332
1333 memset(options, 0, OPTSTRING);
1334
1335 if (txmsg_pass)
1336 append_str(options, "pass,", OPTSTRING);
1337 if (txmsg_redir)
1338 append_str(options, "redir,", OPTSTRING);
1339 if (txmsg_drop)
1340 append_str(options, "drop,", OPTSTRING);
1341 if (txmsg_apply) {
1342 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1343 append_str(options, tstr, OPTSTRING);
1344 }
1345 if (txmsg_cork) {
1346 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1347 append_str(options, tstr, OPTSTRING);
1348 }
1349 if (txmsg_start) {
1350 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1351 append_str(options, tstr, OPTSTRING);
1352 }
1353 if (txmsg_end) {
1354 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1355 append_str(options, tstr, OPTSTRING);
1356 }
1357 if (txmsg_start_pop) {
1358 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1359 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1360 append_str(options, tstr, OPTSTRING);
1361 }
1362 if (txmsg_ingress)
1363 append_str(options, "ingress,", OPTSTRING);
1364 if (txmsg_redir_skb)
1365 append_str(options, "redir_skb,", OPTSTRING);
1366 if (txmsg_ktls_skb)
1367 append_str(options, "ktls_skb,", OPTSTRING);
1368 if (ktls)
1369 append_str(options, "ktls,", OPTSTRING);
1370 if (peek_flag)
1371 append_str(options, "peek,", OPTSTRING);
1372 }
1373
__test_exec(int cgrp,int test,struct sockmap_options * opt)1374 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1375 {
1376 char *options = calloc(OPTSTRING, sizeof(char));
1377 int err;
1378
1379 if (test == SENDPAGE)
1380 opt->sendpage = true;
1381 else
1382 opt->sendpage = false;
1383
1384 if (txmsg_drop)
1385 opt->drop_expected = true;
1386 else
1387 opt->drop_expected = false;
1388
1389 test_options(options);
1390
1391 if (opt->verbose) {
1392 fprintf(stdout,
1393 " [TEST %i]: (%i, %i, %i, %s, %s): ",
1394 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1395 test_to_str(test), options);
1396 fflush(stdout);
1397 }
1398 err = run_options(opt, cgrp, test);
1399 if (opt->verbose)
1400 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1401 test_cnt++;
1402 !err ? passed++ : failed++;
1403 free(options);
1404 return err;
1405 }
1406
test_exec(int cgrp,struct sockmap_options * opt)1407 static void test_exec(int cgrp, struct sockmap_options *opt)
1408 {
1409 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1410 int err;
1411
1412 if (type == 0) {
1413 test_start();
1414 err = __test_exec(cgrp, SENDMSG, opt);
1415 if (err)
1416 test_fail();
1417 } else {
1418 test_start();
1419 err = __test_exec(cgrp, SENDPAGE, opt);
1420 if (err)
1421 test_fail();
1422 }
1423 }
1424
test_send_one(struct sockmap_options * opt,int cgrp)1425 static void test_send_one(struct sockmap_options *opt, int cgrp)
1426 {
1427 opt->iov_length = 1;
1428 opt->iov_count = 1;
1429 opt->rate = 1;
1430 test_exec(cgrp, opt);
1431
1432 opt->iov_length = 1;
1433 opt->iov_count = 1024;
1434 opt->rate = 1;
1435 test_exec(cgrp, opt);
1436
1437 opt->iov_length = 1024;
1438 opt->iov_count = 1;
1439 opt->rate = 1;
1440 test_exec(cgrp, opt);
1441
1442 }
1443
test_send_many(struct sockmap_options * opt,int cgrp)1444 static void test_send_many(struct sockmap_options *opt, int cgrp)
1445 {
1446 opt->iov_length = 3;
1447 opt->iov_count = 1;
1448 opt->rate = 512;
1449 test_exec(cgrp, opt);
1450
1451 opt->rate = 100;
1452 opt->iov_count = 1;
1453 opt->iov_length = 5;
1454 test_exec(cgrp, opt);
1455 }
1456
test_send_large(struct sockmap_options * opt,int cgrp)1457 static void test_send_large(struct sockmap_options *opt, int cgrp)
1458 {
1459 opt->iov_length = 256;
1460 opt->iov_count = 1024;
1461 opt->rate = 2;
1462 test_exec(cgrp, opt);
1463 }
1464
test_send(struct sockmap_options * opt,int cgrp)1465 static void test_send(struct sockmap_options *opt, int cgrp)
1466 {
1467 test_send_one(opt, cgrp);
1468 test_send_many(opt, cgrp);
1469 test_send_large(opt, cgrp);
1470 sched_yield();
1471 }
1472
test_txmsg_pass(int cgrp,struct sockmap_options * opt)1473 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1474 {
1475 /* Test small and large iov_count values with pass/redir/apply/cork */
1476 txmsg_pass = 1;
1477 test_send(opt, cgrp);
1478 }
1479
test_txmsg_redir(int cgrp,struct sockmap_options * opt)1480 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1481 {
1482 txmsg_redir = 1;
1483 test_send(opt, cgrp);
1484 }
1485
test_txmsg_redir_wait_sndmem(int cgrp,struct sockmap_options * opt)1486 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
1487 {
1488 txmsg_redir = 1;
1489 opt->tx_wait_mem = true;
1490 test_send_large(opt, cgrp);
1491 opt->tx_wait_mem = false;
1492 }
1493
test_txmsg_drop(int cgrp,struct sockmap_options * opt)1494 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1495 {
1496 txmsg_drop = 1;
1497 test_send(opt, cgrp);
1498 }
1499
test_txmsg_ingress_redir(int cgrp,struct sockmap_options * opt)1500 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1501 {
1502 txmsg_pass = txmsg_drop = 0;
1503 txmsg_ingress = txmsg_redir = 1;
1504 test_send(opt, cgrp);
1505 }
1506
test_txmsg_skb(int cgrp,struct sockmap_options * opt)1507 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1508 {
1509 bool data = opt->data_test;
1510 int k = ktls;
1511
1512 opt->data_test = true;
1513 ktls = 1;
1514
1515 txmsg_pass = txmsg_drop = 0;
1516 txmsg_ingress = txmsg_redir = 0;
1517 txmsg_ktls_skb = 1;
1518 txmsg_pass = 1;
1519
1520 /* Using data verification so ensure iov layout is
1521 * expected from test receiver side. e.g. has enough
1522 * bytes to write test code.
1523 */
1524 opt->iov_length = 100;
1525 opt->iov_count = 1;
1526 opt->rate = 1;
1527 test_exec(cgrp, opt);
1528
1529 txmsg_ktls_skb_drop = 1;
1530 test_exec(cgrp, opt);
1531
1532 txmsg_ktls_skb_drop = 0;
1533 txmsg_ktls_skb_redir = 1;
1534 test_exec(cgrp, opt);
1535 txmsg_ktls_skb_redir = 0;
1536
1537 /* Tests that omit skb_parser */
1538 txmsg_omit_skb_parser = 1;
1539 ktls = 0;
1540 txmsg_ktls_skb = 0;
1541 test_exec(cgrp, opt);
1542
1543 txmsg_ktls_skb_drop = 1;
1544 test_exec(cgrp, opt);
1545 txmsg_ktls_skb_drop = 0;
1546
1547 txmsg_ktls_skb_redir = 1;
1548 test_exec(cgrp, opt);
1549
1550 ktls = 1;
1551 test_exec(cgrp, opt);
1552 txmsg_omit_skb_parser = 0;
1553
1554 opt->data_test = data;
1555 ktls = k;
1556 }
1557
1558 /* Test cork with hung data. This tests poor usage patterns where
1559 * cork can leave data on the ring if user program is buggy and
1560 * doesn't flush them somehow. They do take some time however
1561 * because they wait for a timeout. Test pass, redir and cork with
1562 * apply logic. Use cork size of 4097 with send_large to avoid
1563 * aligning cork size with send size.
1564 */
test_txmsg_cork_hangs(int cgrp,struct sockmap_options * opt)1565 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1566 {
1567 txmsg_pass = 1;
1568 txmsg_redir = 0;
1569 txmsg_cork = 4097;
1570 txmsg_apply = 4097;
1571 test_send_large(opt, cgrp);
1572
1573 txmsg_pass = 0;
1574 txmsg_redir = 1;
1575 txmsg_apply = 0;
1576 txmsg_cork = 4097;
1577 test_send_large(opt, cgrp);
1578
1579 txmsg_pass = 0;
1580 txmsg_redir = 1;
1581 txmsg_apply = 4097;
1582 txmsg_cork = 4097;
1583 test_send_large(opt, cgrp);
1584 }
1585
test_txmsg_pull(int cgrp,struct sockmap_options * opt)1586 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1587 {
1588 /* Test basic start/end */
1589 txmsg_start = 1;
1590 txmsg_end = 2;
1591 test_send(opt, cgrp);
1592
1593 /* Test >4k pull */
1594 txmsg_start = 4096;
1595 txmsg_end = 9182;
1596 test_send_large(opt, cgrp);
1597
1598 /* Test pull + redirect */
1599 txmsg_redir = 0;
1600 txmsg_start = 1;
1601 txmsg_end = 2;
1602 test_send(opt, cgrp);
1603
1604 /* Test pull + cork */
1605 txmsg_redir = 0;
1606 txmsg_cork = 512;
1607 txmsg_start = 1;
1608 txmsg_end = 2;
1609 test_send_many(opt, cgrp);
1610
1611 /* Test pull + cork + redirect */
1612 txmsg_redir = 1;
1613 txmsg_cork = 512;
1614 txmsg_start = 1;
1615 txmsg_end = 2;
1616 test_send_many(opt, cgrp);
1617 }
1618
test_txmsg_pop(int cgrp,struct sockmap_options * opt)1619 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1620 {
1621 /* Test basic pop */
1622 txmsg_start_pop = 1;
1623 txmsg_pop = 2;
1624 test_send_many(opt, cgrp);
1625
1626 /* Test pop with >4k */
1627 txmsg_start_pop = 4096;
1628 txmsg_pop = 4096;
1629 test_send_large(opt, cgrp);
1630
1631 /* Test pop + redirect */
1632 txmsg_redir = 1;
1633 txmsg_start_pop = 1;
1634 txmsg_pop = 2;
1635 test_send_many(opt, cgrp);
1636
1637 /* Test pop + cork */
1638 txmsg_redir = 0;
1639 txmsg_cork = 512;
1640 txmsg_start_pop = 1;
1641 txmsg_pop = 2;
1642 test_send_many(opt, cgrp);
1643
1644 /* Test pop + redirect + cork */
1645 txmsg_redir = 1;
1646 txmsg_cork = 4;
1647 txmsg_start_pop = 1;
1648 txmsg_pop = 2;
1649 test_send_many(opt, cgrp);
1650 }
1651
test_txmsg_push(int cgrp,struct sockmap_options * opt)1652 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1653 {
1654 /* Test basic push */
1655 txmsg_start_push = 1;
1656 txmsg_end_push = 1;
1657 test_send(opt, cgrp);
1658
1659 /* Test push 4kB >4k */
1660 txmsg_start_push = 4096;
1661 txmsg_end_push = 4096;
1662 test_send_large(opt, cgrp);
1663
1664 /* Test push + redirect */
1665 txmsg_redir = 1;
1666 txmsg_start_push = 1;
1667 txmsg_end_push = 2;
1668 test_send_many(opt, cgrp);
1669
1670 /* Test push + cork */
1671 txmsg_redir = 0;
1672 txmsg_cork = 512;
1673 txmsg_start_push = 1;
1674 txmsg_end_push = 2;
1675 test_send_many(opt, cgrp);
1676 }
1677
test_txmsg_push_pop(int cgrp,struct sockmap_options * opt)1678 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1679 {
1680 txmsg_start_push = 1;
1681 txmsg_end_push = 10;
1682 txmsg_start_pop = 5;
1683 txmsg_pop = 4;
1684 test_send_large(opt, cgrp);
1685 }
1686
test_txmsg_apply(int cgrp,struct sockmap_options * opt)1687 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1688 {
1689 txmsg_pass = 1;
1690 txmsg_redir = 0;
1691 txmsg_ingress = 0;
1692 txmsg_apply = 1;
1693 txmsg_cork = 0;
1694 test_send_one(opt, cgrp);
1695
1696 txmsg_pass = 0;
1697 txmsg_redir = 1;
1698 txmsg_ingress = 0;
1699 txmsg_apply = 1;
1700 txmsg_cork = 0;
1701 test_send_one(opt, cgrp);
1702
1703 txmsg_pass = 0;
1704 txmsg_redir = 1;
1705 txmsg_ingress = 1;
1706 txmsg_apply = 1;
1707 txmsg_cork = 0;
1708 test_send_one(opt, cgrp);
1709
1710 txmsg_pass = 1;
1711 txmsg_redir = 0;
1712 txmsg_ingress = 0;
1713 txmsg_apply = 1024;
1714 txmsg_cork = 0;
1715 test_send_large(opt, cgrp);
1716
1717 txmsg_pass = 0;
1718 txmsg_redir = 1;
1719 txmsg_ingress = 0;
1720 txmsg_apply = 1024;
1721 txmsg_cork = 0;
1722 test_send_large(opt, cgrp);
1723
1724 txmsg_pass = 0;
1725 txmsg_redir = 1;
1726 txmsg_ingress = 1;
1727 txmsg_apply = 1024;
1728 txmsg_cork = 0;
1729 test_send_large(opt, cgrp);
1730 }
1731
test_txmsg_cork(int cgrp,struct sockmap_options * opt)1732 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1733 {
1734 txmsg_pass = 1;
1735 txmsg_redir = 0;
1736 txmsg_apply = 0;
1737 txmsg_cork = 1;
1738 test_send(opt, cgrp);
1739
1740 txmsg_pass = 1;
1741 txmsg_redir = 0;
1742 txmsg_apply = 1;
1743 txmsg_cork = 1;
1744 test_send(opt, cgrp);
1745 }
1746
test_txmsg_ingress_parser(int cgrp,struct sockmap_options * opt)1747 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1748 {
1749 txmsg_pass = 1;
1750 skb_use_parser = 512;
1751 if (ktls == 1)
1752 skb_use_parser = 570;
1753 opt->iov_length = 256;
1754 opt->iov_count = 1;
1755 opt->rate = 2;
1756 test_exec(cgrp, opt);
1757 }
1758
test_txmsg_ingress_parser2(int cgrp,struct sockmap_options * opt)1759 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1760 {
1761 if (ktls == 1)
1762 return;
1763 skb_use_parser = 10;
1764 opt->iov_length = 20;
1765 opt->iov_count = 1;
1766 opt->rate = 1;
1767 opt->check_recved_len = true;
1768 test_exec(cgrp, opt);
1769 opt->check_recved_len = false;
1770 }
1771
1772 char *map_names[] = {
1773 "sock_map",
1774 "sock_map_txmsg",
1775 "sock_map_redir",
1776 "sock_apply_bytes",
1777 "sock_cork_bytes",
1778 "sock_bytes",
1779 "sock_redir_flags",
1780 "sock_skb_opts",
1781 "tls_sock_map",
1782 };
1783
populate_progs(char * bpf_file)1784 static int populate_progs(char *bpf_file)
1785 {
1786 struct bpf_program *prog;
1787 struct bpf_object *obj;
1788 int i = 0;
1789 long err;
1790
1791 obj = bpf_object__open(bpf_file);
1792 err = libbpf_get_error(obj);
1793 if (err) {
1794 char err_buf[256];
1795
1796 libbpf_strerror(err, err_buf, sizeof(err_buf));
1797 printf("Unable to load eBPF objects in file '%s' : %s\n",
1798 bpf_file, err_buf);
1799 return -1;
1800 }
1801
1802 i = bpf_object__load(obj);
1803 i = 0;
1804 bpf_object__for_each_program(prog, obj) {
1805 progs[i] = prog;
1806 i++;
1807 }
1808
1809 for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1810 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1811 map_fd[i] = bpf_map__fd(maps[i]);
1812 if (map_fd[i] < 0) {
1813 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1814 map_fd[i], strerror(errno));
1815 return -1;
1816 }
1817 }
1818
1819 for (i = 0; i < ARRAY_SIZE(links); i++)
1820 links[i] = NULL;
1821
1822 return 0;
1823 }
1824
1825 struct _test test[] = {
1826 {"txmsg test passthrough", test_txmsg_pass},
1827 {"txmsg test redirect", test_txmsg_redir},
1828 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
1829 {"txmsg test drop", test_txmsg_drop},
1830 {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1831 {"txmsg test skb", test_txmsg_skb},
1832 {"txmsg test apply", test_txmsg_apply},
1833 {"txmsg test cork", test_txmsg_cork},
1834 {"txmsg test hanging corks", test_txmsg_cork_hangs},
1835 {"txmsg test push_data", test_txmsg_push},
1836 {"txmsg test pull-data", test_txmsg_pull},
1837 {"txmsg test pop-data", test_txmsg_pop},
1838 {"txmsg test push/pop data", test_txmsg_push_pop},
1839 {"txmsg test ingress parser", test_txmsg_ingress_parser},
1840 {"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1841 };
1842
check_whitelist(struct _test * t,struct sockmap_options * opt)1843 static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1844 {
1845 char *entry, *ptr;
1846
1847 if (!opt->whitelist)
1848 return 0;
1849 ptr = strdup(opt->whitelist);
1850 if (!ptr)
1851 return -ENOMEM;
1852 entry = strtok(ptr, ",");
1853 while (entry) {
1854 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1855 strstr(opt->map, entry) != 0 ||
1856 strstr(t->title, entry) != 0) {
1857 free(ptr);
1858 return 0;
1859 }
1860 entry = strtok(NULL, ",");
1861 }
1862 free(ptr);
1863 return -EINVAL;
1864 }
1865
check_blacklist(struct _test * t,struct sockmap_options * opt)1866 static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1867 {
1868 char *entry, *ptr;
1869
1870 if (!opt->blacklist)
1871 return -EINVAL;
1872 ptr = strdup(opt->blacklist);
1873 if (!ptr)
1874 return -ENOMEM;
1875 entry = strtok(ptr, ",");
1876 while (entry) {
1877 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1878 strstr(opt->map, entry) != 0 ||
1879 strstr(t->title, entry) != 0) {
1880 free(ptr);
1881 return 0;
1882 }
1883 entry = strtok(NULL, ",");
1884 }
1885 free(ptr);
1886 return -EINVAL;
1887 }
1888
__test_selftests(int cg_fd,struct sockmap_options * opt)1889 static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1890 {
1891 int i, err;
1892
1893 err = populate_progs(opt->map);
1894 if (err < 0) {
1895 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1896 return err;
1897 }
1898
1899 /* Tests basic commands and APIs */
1900 for (i = 0; i < ARRAY_SIZE(test); i++) {
1901 struct _test t = test[i];
1902
1903 if (check_whitelist(&t, opt) != 0)
1904 continue;
1905 if (check_blacklist(&t, opt) == 0)
1906 continue;
1907
1908 test_start_subtest(&t, opt);
1909 t.tester(cg_fd, opt);
1910 test_end_subtest();
1911 }
1912
1913 return err;
1914 }
1915
test_selftests_sockmap(int cg_fd,struct sockmap_options * opt)1916 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1917 {
1918 opt->map = BPF_SOCKMAP_FILENAME;
1919 __test_selftests(cg_fd, opt);
1920 }
1921
test_selftests_sockhash(int cg_fd,struct sockmap_options * opt)1922 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1923 {
1924 opt->map = BPF_SOCKHASH_FILENAME;
1925 __test_selftests(cg_fd, opt);
1926 }
1927
test_selftests_ktls(int cg_fd,struct sockmap_options * opt)1928 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1929 {
1930 opt->map = BPF_SOCKHASH_FILENAME;
1931 opt->prepend = "ktls";
1932 ktls = 1;
1933 __test_selftests(cg_fd, opt);
1934 ktls = 0;
1935 }
1936
test_selftest(int cg_fd,struct sockmap_options * opt)1937 static int test_selftest(int cg_fd, struct sockmap_options *opt)
1938 {
1939 test_selftests_sockmap(cg_fd, opt);
1940 test_selftests_sockhash(cg_fd, opt);
1941 test_selftests_ktls(cg_fd, opt);
1942 test_print_results();
1943 return 0;
1944 }
1945
main(int argc,char ** argv)1946 int main(int argc, char **argv)
1947 {
1948 int iov_count = 1, length = 1024, rate = 1;
1949 struct sockmap_options options = {0};
1950 int opt, longindex, err, cg_fd = 0;
1951 char *bpf_file = BPF_SOCKMAP_FILENAME;
1952 int test = SELFTESTS;
1953 bool cg_created = 0;
1954
1955 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1956 long_options, &longindex)) != -1) {
1957 switch (opt) {
1958 case 's':
1959 txmsg_start = atoi(optarg);
1960 break;
1961 case 'e':
1962 txmsg_end = atoi(optarg);
1963 break;
1964 case 'p':
1965 txmsg_start_push = atoi(optarg);
1966 break;
1967 case 'q':
1968 txmsg_end_push = atoi(optarg);
1969 break;
1970 case 'w':
1971 txmsg_start_pop = atoi(optarg);
1972 break;
1973 case 'x':
1974 txmsg_pop = atoi(optarg);
1975 break;
1976 case 'a':
1977 txmsg_apply = atoi(optarg);
1978 break;
1979 case 'k':
1980 txmsg_cork = atoi(optarg);
1981 break;
1982 case 'c':
1983 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1984 if (cg_fd < 0) {
1985 fprintf(stderr,
1986 "ERROR: (%i) open cg path failed: %s\n",
1987 cg_fd, optarg);
1988 return cg_fd;
1989 }
1990 break;
1991 case 'r':
1992 rate = atoi(optarg);
1993 break;
1994 case 'v':
1995 options.verbose = 1;
1996 if (optarg)
1997 options.verbose = atoi(optarg);
1998 break;
1999 case 'i':
2000 iov_count = atoi(optarg);
2001 break;
2002 case 'l':
2003 length = atoi(optarg);
2004 break;
2005 case 'd':
2006 options.data_test = true;
2007 break;
2008 case 't':
2009 if (strcmp(optarg, "ping") == 0) {
2010 test = PING_PONG;
2011 } else if (strcmp(optarg, "sendmsg") == 0) {
2012 test = SENDMSG;
2013 } else if (strcmp(optarg, "base") == 0) {
2014 test = BASE;
2015 } else if (strcmp(optarg, "base_sendpage") == 0) {
2016 test = BASE_SENDPAGE;
2017 } else if (strcmp(optarg, "sendpage") == 0) {
2018 test = SENDPAGE;
2019 } else {
2020 usage(argv);
2021 return -1;
2022 }
2023 break;
2024 case 'n':
2025 options.whitelist = strdup(optarg);
2026 if (!options.whitelist)
2027 return -ENOMEM;
2028 break;
2029 case 'b':
2030 options.blacklist = strdup(optarg);
2031 if (!options.blacklist)
2032 return -ENOMEM;
2033 case 0:
2034 break;
2035 case 'h':
2036 default:
2037 usage(argv);
2038 return -1;
2039 }
2040 }
2041
2042 if (!cg_fd) {
2043 cg_fd = cgroup_setup_and_join(CG_PATH);
2044 if (cg_fd < 0)
2045 return cg_fd;
2046 cg_created = 1;
2047 }
2048
2049 /* Use libbpf 1.0 API mode */
2050 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2051
2052 if (test == SELFTESTS) {
2053 err = test_selftest(cg_fd, &options);
2054 goto out;
2055 }
2056
2057 err = populate_progs(bpf_file);
2058 if (err) {
2059 fprintf(stderr, "populate program: (%s) %s\n",
2060 bpf_file, strerror(errno));
2061 return 1;
2062 }
2063 running = 1;
2064
2065 /* catch SIGINT */
2066 signal(SIGINT, running_handler);
2067
2068 options.iov_count = iov_count;
2069 options.iov_length = length;
2070 options.rate = rate;
2071
2072 err = run_options(&options, cg_fd, test);
2073 out:
2074 if (options.whitelist)
2075 free(options.whitelist);
2076 if (options.blacklist)
2077 free(options.blacklist);
2078 close(cg_fd);
2079 if (cg_created)
2080 cleanup_cgroup_environment();
2081 return err;
2082 }
2083
running_handler(int a)2084 void running_handler(int a)
2085 {
2086 running = 0;
2087 }
2088