1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <sys/socket.h>
6 #include <sys/ioctl.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
9 #include <arpa/inet.h>
10 #include <unistd.h>
11 #include <string.h>
12 #include <errno.h>
13 #include <stdbool.h>
14 #include <signal.h>
15 #include <fcntl.h>
16 #include <sys/wait.h>
17 #include <time.h>
18 #include <sched.h>
19
20 #include <sys/time.h>
21 #include <sys/types.h>
22 #include <sys/sendfile.h>
23
24 #include <linux/netlink.h>
25 #include <linux/socket.h>
26 #include <linux/sock_diag.h>
27 #include <linux/bpf.h>
28 #include <linux/if_link.h>
29 #include <assert.h>
30 #include <libgen.h>
31
32 #include <getopt.h>
33
34 #include <bpf/bpf.h>
35 #include <bpf/libbpf.h>
36
37 #include "bpf_util.h"
38 #include "cgroup_helpers.h"
39
40 int running;
41 static void running_handler(int a);
42
43 /* randomly selected ports for testing on lo */
44 #define S1_PORT 10000
45 #define S2_PORT 10001
46
47 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o"
48 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
49 #define CG_PATH "/sockmap"
50
51 #define EDATAINTEGRITY 2001
52
53 /* global sockets */
54 int s1, s2, c1, c2, p1, p2;
55 int test_cnt;
56 int passed;
57 int failed;
58 int map_fd[8];
59 struct bpf_map *maps[8];
60 struct bpf_program *progs[8];
61 struct bpf_link *links[8];
62
63 int txmsg_pass;
64 int txmsg_redir;
65 int txmsg_drop;
66 int txmsg_apply;
67 int txmsg_cork;
68 int txmsg_start;
69 int txmsg_end;
70 int txmsg_start_push;
71 int txmsg_end_push;
72 int txmsg_start_pop;
73 int txmsg_pop;
74 int txmsg_ingress;
75 int txmsg_redir_skb;
76 int peek_flag;
77 int skb_use_parser;
78 int txmsg_omit_skb_parser;
79 int verify_push_start;
80 int verify_push_len;
81 int verify_pop_start;
82 int verify_pop_len;
83
84 static const struct option long_options[] = {
85 {"help", no_argument, NULL, 'h' },
86 {"cgroup", required_argument, NULL, 'c' },
87 {"rate", required_argument, NULL, 'r' },
88 {"verbose", optional_argument, NULL, 'v' },
89 {"iov_count", required_argument, NULL, 'i' },
90 {"length", required_argument, NULL, 'l' },
91 {"test", required_argument, NULL, 't' },
92 {"data_test", no_argument, NULL, 'd' },
93 {"txmsg", no_argument, &txmsg_pass, 1 },
94 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
95 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
96 {"txmsg_apply", required_argument, NULL, 'a'},
97 {"txmsg_cork", required_argument, NULL, 'k'},
98 {"txmsg_start", required_argument, NULL, 's'},
99 {"txmsg_end", required_argument, NULL, 'e'},
100 {"txmsg_start_push", required_argument, NULL, 'p'},
101 {"txmsg_end_push", required_argument, NULL, 'q'},
102 {"txmsg_start_pop", required_argument, NULL, 'w'},
103 {"txmsg_pop", required_argument, NULL, 'x'},
104 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
105 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
106 {"peek", no_argument, &peek_flag, 1 },
107 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
108 {"whitelist", required_argument, NULL, 'n' },
109 {"blacklist", required_argument, NULL, 'b' },
110 {0, 0, NULL, 0 }
111 };
112
113 struct test_env {
114 const char *type;
115 const char *subtest;
116 const char *prepend;
117
118 int test_num;
119 int subtest_num;
120
121 int succ_cnt;
122 int fail_cnt;
123 int fail_last;
124 };
125
126 struct test_env env;
127
128 struct sockmap_options {
129 int verbose;
130 bool base;
131 bool sendpage;
132 bool data_test;
133 bool drop_expected;
134 bool check_recved_len;
135 bool tx_wait_mem;
136 int iov_count;
137 int iov_length;
138 int rate;
139 char *map;
140 char *whitelist;
141 char *blacklist;
142 char *prepend;
143 };
144
145 struct _test {
146 char *title;
147 void (*tester)(int cg_fd, struct sockmap_options *opt);
148 };
149
test_start(void)150 static void test_start(void)
151 {
152 env.subtest_num++;
153 }
154
test_fail(void)155 static void test_fail(void)
156 {
157 env.fail_cnt++;
158 }
159
test_pass(void)160 static void test_pass(void)
161 {
162 env.succ_cnt++;
163 }
164
test_reset(void)165 static void test_reset(void)
166 {
167 txmsg_start = txmsg_end = 0;
168 txmsg_start_pop = txmsg_pop = 0;
169 txmsg_start_push = txmsg_end_push = 0;
170 txmsg_pass = txmsg_drop = txmsg_redir = 0;
171 txmsg_apply = txmsg_cork = 0;
172 txmsg_ingress = txmsg_redir_skb = 0;
173 txmsg_omit_skb_parser = 0;
174 skb_use_parser = 0;
175 }
176
test_start_subtest(const struct _test * t,struct sockmap_options * o)177 static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
178 {
179 env.type = o->map;
180 env.subtest = t->title;
181 env.prepend = o->prepend;
182 env.test_num++;
183 env.subtest_num = 0;
184 env.fail_last = env.fail_cnt;
185 test_reset();
186 return 0;
187 }
188
test_end_subtest(void)189 static void test_end_subtest(void)
190 {
191 int error = env.fail_cnt - env.fail_last;
192 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
193
194 if (!error)
195 test_pass();
196
197 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
198 env.test_num, env.subtest_num,
199 !type ? "sockmap" : "sockhash",
200 env.prepend ? : "",
201 env.subtest, error ? "FAIL" : "OK");
202 }
203
test_print_results(void)204 static void test_print_results(void)
205 {
206 fprintf(stdout, "Pass: %d Fail: %d\n",
207 env.succ_cnt, env.fail_cnt);
208 }
209
usage(char * argv[])210 static void usage(char *argv[])
211 {
212 int i;
213
214 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
215 printf(" options:\n");
216 for (i = 0; long_options[i].name != 0; i++) {
217 printf(" --%-12s", long_options[i].name);
218 if (long_options[i].flag != NULL)
219 printf(" flag (internal value:%d)\n",
220 *long_options[i].flag);
221 else
222 printf(" -%c\n", long_options[i].val);
223 }
224 printf("\n");
225 }
226
sockmap_init_sockets(int verbose)227 static int sockmap_init_sockets(int verbose)
228 {
229 int i, err, one = 1;
230 struct sockaddr_in addr;
231 int *fds[4] = {&s1, &s2, &c1, &c2};
232
233 s1 = s2 = p1 = p2 = c1 = c2 = 0;
234
235 /* Init sockets */
236 for (i = 0; i < 4; i++) {
237 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
238 if (*fds[i] < 0) {
239 perror("socket s1 failed()");
240 return errno;
241 }
242 }
243
244 /* Allow reuse */
245 for (i = 0; i < 2; i++) {
246 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
247 (char *)&one, sizeof(one));
248 if (err) {
249 perror("setsockopt failed()");
250 return errno;
251 }
252 }
253
254 /* Non-blocking sockets */
255 for (i = 0; i < 2; i++) {
256 err = ioctl(*fds[i], FIONBIO, (char *)&one);
257 if (err < 0) {
258 perror("ioctl s1 failed()");
259 return errno;
260 }
261 }
262
263 /* Bind server sockets */
264 memset(&addr, 0, sizeof(struct sockaddr_in));
265 addr.sin_family = AF_INET;
266 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
267
268 addr.sin_port = htons(S1_PORT);
269 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
270 if (err < 0) {
271 perror("bind s1 failed()");
272 return errno;
273 }
274
275 addr.sin_port = htons(S2_PORT);
276 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
277 if (err < 0) {
278 perror("bind s2 failed()");
279 return errno;
280 }
281
282 /* Listen server sockets */
283 addr.sin_port = htons(S1_PORT);
284 err = listen(s1, 32);
285 if (err < 0) {
286 perror("listen s1 failed()");
287 return errno;
288 }
289
290 addr.sin_port = htons(S2_PORT);
291 err = listen(s2, 32);
292 if (err < 0) {
293 perror("listen s1 failed()");
294 return errno;
295 }
296
297 /* Initiate Connect */
298 addr.sin_port = htons(S1_PORT);
299 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
300 if (err < 0 && errno != EINPROGRESS) {
301 perror("connect c1 failed()");
302 return errno;
303 }
304
305 addr.sin_port = htons(S2_PORT);
306 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
307 if (err < 0 && errno != EINPROGRESS) {
308 perror("connect c2 failed()");
309 return errno;
310 } else if (err < 0) {
311 err = 0;
312 }
313
314 /* Accept Connecrtions */
315 p1 = accept(s1, NULL, NULL);
316 if (p1 < 0) {
317 perror("accept s1 failed()");
318 return errno;
319 }
320
321 p2 = accept(s2, NULL, NULL);
322 if (p2 < 0) {
323 perror("accept s1 failed()");
324 return errno;
325 }
326
327 if (verbose > 1) {
328 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
329 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
330 c1, s1, c2, s2);
331 }
332 return 0;
333 }
334
335 struct msg_stats {
336 size_t bytes_sent;
337 size_t bytes_recvd;
338 struct timespec start;
339 struct timespec end;
340 };
341
msg_loop_sendpage(int fd,int iov_length,int cnt,struct msg_stats * s,struct sockmap_options * opt)342 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
343 struct msg_stats *s,
344 struct sockmap_options *opt)
345 {
346 bool drop = opt->drop_expected;
347 unsigned char k = 0;
348 int i, j, fp;
349 FILE *file;
350
351 file = tmpfile();
352 if (!file) {
353 perror("create file for sendpage");
354 return 1;
355 }
356 for (i = 0; i < cnt; i++, k = 0) {
357 for (j = 0; j < iov_length; j++, k++)
358 fwrite(&k, sizeof(char), 1, file);
359 }
360 fflush(file);
361 fseek(file, 0, SEEK_SET);
362
363 fp = fileno(file);
364
365 clock_gettime(CLOCK_MONOTONIC, &s->start);
366 for (i = 0; i < cnt; i++) {
367 int sent;
368
369 errno = 0;
370 sent = sendfile(fd, fp, NULL, iov_length);
371
372 if (!drop && sent < 0) {
373 perror("sendpage loop error");
374 fclose(file);
375 return sent;
376 } else if (drop && sent >= 0) {
377 printf("sendpage loop error expected: %i errno %i\n",
378 sent, errno);
379 fclose(file);
380 return -EIO;
381 }
382
383 if (sent > 0)
384 s->bytes_sent += sent;
385 }
386 clock_gettime(CLOCK_MONOTONIC, &s->end);
387 fclose(file);
388 return 0;
389 }
390
msg_free_iov(struct msghdr * msg)391 static void msg_free_iov(struct msghdr *msg)
392 {
393 int i;
394
395 for (i = 0; i < msg->msg_iovlen; i++)
396 free(msg->msg_iov[i].iov_base);
397 free(msg->msg_iov);
398 msg->msg_iov = NULL;
399 msg->msg_iovlen = 0;
400 }
401
msg_alloc_iov(struct msghdr * msg,int iov_count,int iov_length,bool data,bool xmit)402 static int msg_alloc_iov(struct msghdr *msg,
403 int iov_count, int iov_length,
404 bool data, bool xmit)
405 {
406 unsigned char k = 0;
407 struct iovec *iov;
408 int i;
409
410 iov = calloc(iov_count, sizeof(struct iovec));
411 if (!iov)
412 return errno;
413
414 for (i = 0; i < iov_count; i++) {
415 unsigned char *d = calloc(iov_length, sizeof(char));
416
417 if (!d) {
418 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
419 goto unwind_iov;
420 }
421 iov[i].iov_base = d;
422 iov[i].iov_len = iov_length;
423
424 if (data && xmit) {
425 int j;
426
427 for (j = 0; j < iov_length; j++)
428 d[j] = k++;
429 }
430 }
431
432 msg->msg_iov = iov;
433 msg->msg_iovlen = iov_count;
434
435 return 0;
436 unwind_iov:
437 for (i--; i >= 0 ; i--)
438 free(iov[i].iov_base);
439 free(iov);
440 return -ENOMEM;
441 }
442
443 /* In push or pop test, we need to do some calculations for msg_verify_data */
msg_verify_date_prep(void)444 static void msg_verify_date_prep(void)
445 {
446 int push_range_end = txmsg_start_push + txmsg_end_push - 1;
447 int pop_range_end = txmsg_start_pop + txmsg_pop - 1;
448
449 if (txmsg_end_push && txmsg_pop &&
450 txmsg_start_push <= pop_range_end && txmsg_start_pop <= push_range_end) {
451 /* The push range and the pop range overlap */
452 int overlap_len;
453
454 verify_push_start = txmsg_start_push;
455 verify_pop_start = txmsg_start_pop;
456 if (txmsg_start_push < txmsg_start_pop)
457 overlap_len = min(push_range_end - txmsg_start_pop + 1, txmsg_pop);
458 else
459 overlap_len = min(pop_range_end - txmsg_start_push + 1, txmsg_end_push);
460 verify_push_len = max(txmsg_end_push - overlap_len, 0);
461 verify_pop_len = max(txmsg_pop - overlap_len, 0);
462 } else {
463 /* Otherwise */
464 verify_push_start = txmsg_start_push;
465 verify_pop_start = txmsg_start_pop;
466 verify_push_len = txmsg_end_push;
467 verify_pop_len = txmsg_pop;
468 }
469 }
470
msg_verify_data(struct msghdr * msg,int size,int chunk_sz,unsigned char * k_p,int * bytes_cnt_p,int * check_cnt_p,int * push_p)471 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz,
472 unsigned char *k_p, int *bytes_cnt_p,
473 int *check_cnt_p, int *push_p)
474 {
475 int bytes_cnt = *bytes_cnt_p, check_cnt = *check_cnt_p, push = *push_p;
476 unsigned char k = *k_p;
477 int i, j;
478
479 for (i = 0, j = 0; i < msg->msg_iovlen && size; i++, j = 0) {
480 unsigned char *d = msg->msg_iov[i].iov_base;
481
482 for (; j < msg->msg_iov[i].iov_len && size; j++) {
483 if (push > 0 &&
484 check_cnt == verify_push_start + verify_push_len - push) {
485 int skipped;
486 revisit_push:
487 skipped = push;
488 if (j + push >= msg->msg_iov[i].iov_len)
489 skipped = msg->msg_iov[i].iov_len - j;
490 push -= skipped;
491 size -= skipped;
492 j += skipped - 1;
493 check_cnt += skipped;
494 continue;
495 }
496
497 if (verify_pop_len > 0 && check_cnt == verify_pop_start) {
498 bytes_cnt += verify_pop_len;
499 check_cnt += verify_pop_len;
500 k += verify_pop_len;
501
502 if (bytes_cnt == chunk_sz) {
503 k = 0;
504 bytes_cnt = 0;
505 check_cnt = 0;
506 push = verify_push_len;
507 }
508
509 if (push > 0 &&
510 check_cnt == verify_push_start + verify_push_len - push)
511 goto revisit_push;
512 }
513
514 if (d[j] != k++) {
515 fprintf(stderr,
516 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
517 i, j, d[j], k - 1, d[j+1], k);
518 return -EDATAINTEGRITY;
519 }
520 bytes_cnt++;
521 check_cnt++;
522 if (bytes_cnt == chunk_sz) {
523 k = 0;
524 bytes_cnt = 0;
525 check_cnt = 0;
526 push = verify_push_len;
527 }
528 size--;
529 }
530 }
531 *k_p = k;
532 *bytes_cnt_p = bytes_cnt;
533 *check_cnt_p = check_cnt;
534 *push_p = push;
535 return 0;
536 }
537
msg_loop(int fd,int iov_count,int iov_length,int cnt,struct msg_stats * s,bool tx,struct sockmap_options * opt)538 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
539 struct msg_stats *s, bool tx,
540 struct sockmap_options *opt)
541 {
542 struct msghdr msg = {0}, msg_peek = {0};
543 int err, i, flags = MSG_NOSIGNAL;
544 bool drop = opt->drop_expected;
545 bool data = opt->data_test;
546 int iov_alloc_length = iov_length;
547
548 if (!tx && opt->check_recved_len)
549 iov_alloc_length *= 2;
550
551 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
552 if (err)
553 goto out_errno;
554 if (peek_flag) {
555 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
556 if (err)
557 goto out_errno;
558 }
559
560 if (tx) {
561 clock_gettime(CLOCK_MONOTONIC, &s->start);
562 for (i = 0; i < cnt; i++) {
563 int sent;
564
565 errno = 0;
566 sent = sendmsg(fd, &msg, flags);
567
568 if (!drop && sent < 0) {
569 if (opt->tx_wait_mem && errno == EACCES) {
570 errno = 0;
571 goto out_errno;
572 }
573 perror("sendmsg loop error");
574 goto out_errno;
575 } else if (drop && sent >= 0) {
576 fprintf(stderr,
577 "sendmsg loop error expected: %i errno %i\n",
578 sent, errno);
579 errno = -EIO;
580 goto out_errno;
581 }
582 if (sent > 0)
583 s->bytes_sent += sent;
584 }
585 clock_gettime(CLOCK_MONOTONIC, &s->end);
586 } else {
587 float total_bytes, txmsg_pop_total, txmsg_push_total;
588 int slct, recvp = 0, recv, max_fd = fd;
589 int fd_flags = O_NONBLOCK;
590 struct timeval timeout;
591 unsigned char k = 0;
592 int bytes_cnt = 0;
593 int check_cnt = 0;
594 int push = 0;
595 fd_set w;
596
597 fcntl(fd, fd_flags);
598 /* Account for pop bytes noting each iteration of apply will
599 * call msg_pop_data helper so we need to account for this
600 * by calculating the number of apply iterations. Note user
601 * of the tool can create cases where no data is sent by
602 * manipulating pop/push/pull/etc. For example txmsg_apply 1
603 * with txmsg_pop 1 will try to apply 1B at a time but each
604 * iteration will then pop 1B so no data will ever be sent.
605 * This is really only useful for testing edge cases in code
606 * paths.
607 */
608 total_bytes = (float)iov_length * (float)cnt;
609 if (!opt->sendpage)
610 total_bytes *= (float)iov_count;
611 if (txmsg_apply) {
612 txmsg_push_total = txmsg_end_push * (total_bytes / txmsg_apply);
613 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
614 } else {
615 txmsg_push_total = txmsg_end_push * cnt;
616 txmsg_pop_total = txmsg_pop * cnt;
617 }
618 total_bytes += txmsg_push_total;
619 total_bytes -= txmsg_pop_total;
620 if (data) {
621 msg_verify_date_prep();
622 push = verify_push_len;
623 }
624 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
625 if (err < 0)
626 perror("recv start time");
627 while (s->bytes_recvd < total_bytes) {
628 if (txmsg_cork) {
629 timeout.tv_sec = 0;
630 timeout.tv_usec = 300000;
631 } else {
632 timeout.tv_sec = 3;
633 timeout.tv_usec = 0;
634 }
635
636 /* FD sets */
637 FD_ZERO(&w);
638 FD_SET(fd, &w);
639
640 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
641 if (slct == -1) {
642 perror("select()");
643 clock_gettime(CLOCK_MONOTONIC, &s->end);
644 goto out_errno;
645 } else if (!slct) {
646 if (opt->verbose)
647 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
648 errno = -EIO;
649 clock_gettime(CLOCK_MONOTONIC, &s->end);
650 goto out_errno;
651 }
652
653 if (opt->tx_wait_mem) {
654 FD_ZERO(&w);
655 FD_SET(fd, &w);
656 slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
657 errno = 0;
658 close(fd);
659 goto out_errno;
660 }
661
662 errno = 0;
663 if (peek_flag) {
664 flags |= MSG_PEEK;
665 recvp = recvmsg(fd, &msg_peek, flags);
666 if (recvp < 0) {
667 if (errno != EWOULDBLOCK) {
668 clock_gettime(CLOCK_MONOTONIC, &s->end);
669 goto out_errno;
670 }
671 }
672 flags = 0;
673 }
674
675 recv = recvmsg(fd, &msg, flags);
676 if (recv < 0) {
677 if (errno != EWOULDBLOCK) {
678 clock_gettime(CLOCK_MONOTONIC, &s->end);
679 perror("recv failed()");
680 goto out_errno;
681 }
682 }
683
684 if (recv > 0)
685 s->bytes_recvd += recv;
686
687 if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
688 errno = EMSGSIZE;
689 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
690 s->bytes_recvd, total_bytes);
691 goto out_errno;
692 }
693
694 if (data) {
695 int chunk_sz = opt->sendpage ?
696 iov_length :
697 iov_length * iov_count;
698
699 errno = msg_verify_data(&msg, recv, chunk_sz, &k, &bytes_cnt,
700 &check_cnt, &push);
701 if (errno) {
702 perror("data verify msg failed");
703 goto out_errno;
704 }
705 if (recvp) {
706 errno = msg_verify_data(&msg_peek,
707 recvp,
708 chunk_sz,
709 &k,
710 &bytes_cnt,
711 &check_cnt,
712 &push);
713 if (errno) {
714 perror("data verify msg_peek failed");
715 goto out_errno;
716 }
717 }
718 }
719 }
720 clock_gettime(CLOCK_MONOTONIC, &s->end);
721 }
722
723 msg_free_iov(&msg);
724 msg_free_iov(&msg_peek);
725 return err;
726 out_errno:
727 msg_free_iov(&msg);
728 msg_free_iov(&msg_peek);
729 return errno;
730 }
731
732 static float giga = 1000000000;
733
sentBps(struct msg_stats s)734 static inline float sentBps(struct msg_stats s)
735 {
736 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
737 }
738
recvdBps(struct msg_stats s)739 static inline float recvdBps(struct msg_stats s)
740 {
741 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
742 }
743
sendmsg_test(struct sockmap_options * opt)744 static int sendmsg_test(struct sockmap_options *opt)
745 {
746 float sent_Bps = 0, recvd_Bps = 0;
747 int rx_fd, txpid, rxpid, err = 0;
748 struct msg_stats s = {0};
749 int iov_count = opt->iov_count;
750 int iov_buf = opt->iov_length;
751 int rx_status, tx_status;
752 int cnt = opt->rate;
753
754 errno = 0;
755
756 if (opt->base)
757 rx_fd = p1;
758 else
759 rx_fd = p2;
760
761 if (opt->tx_wait_mem) {
762 struct timeval timeout;
763 int rxtx_buf_len = 1024;
764
765 timeout.tv_sec = 3;
766 timeout.tv_usec = 0;
767
768 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
769 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
770 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
771 if (err) {
772 perror("setsockopt failed()");
773 return errno;
774 }
775 }
776
777 rxpid = fork();
778 if (rxpid == 0) {
779 if (opt->drop_expected)
780 _exit(0);
781
782 if (!iov_buf) /* zero bytes sent case */
783 _exit(0);
784
785 if (opt->sendpage)
786 iov_count = 1;
787 err = msg_loop(rx_fd, iov_count, iov_buf,
788 cnt, &s, false, opt);
789 if (opt->verbose > 1)
790 fprintf(stderr,
791 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
792 iov_count, iov_buf, cnt, err);
793 if (s.end.tv_sec - s.start.tv_sec) {
794 sent_Bps = sentBps(s);
795 recvd_Bps = recvdBps(s);
796 }
797 if (opt->verbose > 1)
798 fprintf(stdout,
799 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
800 s.bytes_sent, sent_Bps, sent_Bps/giga,
801 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
802 peek_flag ? "(peek_msg)" : "");
803 if (err && err != -EDATAINTEGRITY && txmsg_cork)
804 err = 0;
805 exit(err ? 1 : 0);
806 } else if (rxpid == -1) {
807 perror("msg_loop_rx");
808 return errno;
809 }
810
811 if (opt->tx_wait_mem)
812 close(c2);
813
814 txpid = fork();
815 if (txpid == 0) {
816 if (opt->sendpage)
817 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
818 else
819 err = msg_loop(c1, iov_count, iov_buf,
820 cnt, &s, true, opt);
821
822 if (err)
823 fprintf(stderr,
824 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
825 iov_count, iov_buf, cnt, err);
826 if (s.end.tv_sec - s.start.tv_sec) {
827 sent_Bps = sentBps(s);
828 recvd_Bps = recvdBps(s);
829 }
830 if (opt->verbose > 1)
831 fprintf(stdout,
832 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
833 s.bytes_sent, sent_Bps, sent_Bps/giga,
834 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
835 exit(err ? 1 : 0);
836 } else if (txpid == -1) {
837 perror("msg_loop_tx");
838 return errno;
839 }
840
841 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
842 assert(waitpid(txpid, &tx_status, 0) == txpid);
843 if (WIFEXITED(rx_status)) {
844 err = WEXITSTATUS(rx_status);
845 if (err) {
846 fprintf(stderr, "rx thread exited with err %d.\n", err);
847 goto out;
848 }
849 }
850 if (WIFEXITED(tx_status)) {
851 err = WEXITSTATUS(tx_status);
852 if (err)
853 fprintf(stderr, "tx thread exited with err %d.\n", err);
854 }
855 out:
856 return err;
857 }
858
forever_ping_pong(int rate,struct sockmap_options * opt)859 static int forever_ping_pong(int rate, struct sockmap_options *opt)
860 {
861 struct timeval timeout;
862 char buf[1024] = {0};
863 int sc;
864
865 timeout.tv_sec = 10;
866 timeout.tv_usec = 0;
867
868 /* Ping/Pong data from client to server */
869 sc = send(c1, buf, sizeof(buf), 0);
870 if (sc < 0) {
871 perror("send failed()");
872 return sc;
873 }
874
875 do {
876 int s, rc, i, max_fd = p2;
877 fd_set w;
878
879 /* FD sets */
880 FD_ZERO(&w);
881 FD_SET(c1, &w);
882 FD_SET(c2, &w);
883 FD_SET(p1, &w);
884 FD_SET(p2, &w);
885
886 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
887 if (s == -1) {
888 perror("select()");
889 break;
890 } else if (!s) {
891 fprintf(stderr, "unexpected timeout\n");
892 break;
893 }
894
895 for (i = 0; i <= max_fd && s > 0; ++i) {
896 if (!FD_ISSET(i, &w))
897 continue;
898
899 s--;
900
901 rc = recv(i, buf, sizeof(buf), 0);
902 if (rc < 0) {
903 if (errno != EWOULDBLOCK) {
904 perror("recv failed()");
905 return rc;
906 }
907 }
908
909 if (rc == 0) {
910 close(i);
911 break;
912 }
913
914 sc = send(i, buf, rc, 0);
915 if (sc < 0) {
916 perror("send failed()");
917 return sc;
918 }
919 }
920
921 if (rate)
922 sleep(rate);
923
924 if (opt->verbose) {
925 printf(".");
926 fflush(stdout);
927
928 }
929 } while (running);
930
931 return 0;
932 }
933
934 enum {
935 SELFTESTS,
936 PING_PONG,
937 SENDMSG,
938 BASE,
939 BASE_SENDPAGE,
940 SENDPAGE,
941 };
942
run_options(struct sockmap_options * options,int cg_fd,int test)943 static int run_options(struct sockmap_options *options, int cg_fd, int test)
944 {
945 int i, key, next_key, err, zero = 0;
946 struct bpf_program *tx_prog;
947
948 /* If base test skip BPF setup */
949 if (test == BASE || test == BASE_SENDPAGE)
950 goto run;
951
952 /* Attach programs to sockmap */
953 if (!txmsg_omit_skb_parser) {
954 links[0] = bpf_program__attach_sockmap(progs[0], map_fd[0]);
955 if (!links[0]) {
956 fprintf(stderr,
957 "ERROR: bpf_program__attach_sockmap (sockmap %i->%i): (%s)\n",
958 bpf_program__fd(progs[0]), map_fd[0], strerror(errno));
959 return -1;
960 }
961 }
962
963 links[1] = bpf_program__attach_sockmap(progs[1], map_fd[0]);
964 if (!links[1]) {
965 fprintf(stderr, "ERROR: bpf_program__attach_sockmap (sockmap): (%s)\n",
966 strerror(errno));
967 return -1;
968 }
969
970 /* Attach to cgroups */
971 err = bpf_prog_attach(bpf_program__fd(progs[2]), cg_fd, BPF_CGROUP_SOCK_OPS, 0);
972 if (err) {
973 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
974 err, strerror(errno));
975 return err;
976 }
977
978 run:
979 err = sockmap_init_sockets(options->verbose);
980 if (err) {
981 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
982 goto out;
983 }
984
985 /* Attach txmsg program to sockmap */
986 if (txmsg_pass)
987 tx_prog = progs[3];
988 else if (txmsg_redir)
989 tx_prog = progs[4];
990 else if (txmsg_apply)
991 tx_prog = progs[5];
992 else if (txmsg_cork)
993 tx_prog = progs[6];
994 else if (txmsg_drop)
995 tx_prog = progs[7];
996 else
997 tx_prog = NULL;
998
999 if (tx_prog) {
1000 int redir_fd;
1001
1002 links[4] = bpf_program__attach_sockmap(tx_prog, map_fd[1]);
1003 if (!links[4]) {
1004 fprintf(stderr,
1005 "ERROR: bpf_program__attach_sockmap (txmsg): (%s)\n",
1006 strerror(errno));
1007 err = -1;
1008 goto out;
1009 }
1010
1011 i = 0;
1012 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1013 if (err) {
1014 fprintf(stderr,
1015 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1016 err, strerror(errno));
1017 goto out;
1018 }
1019
1020 if (txmsg_redir)
1021 redir_fd = c2;
1022 else
1023 redir_fd = c1;
1024
1025 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1026 if (err) {
1027 fprintf(stderr,
1028 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1029 err, strerror(errno));
1030 goto out;
1031 }
1032
1033 if (txmsg_apply) {
1034 err = bpf_map_update_elem(map_fd[3],
1035 &i, &txmsg_apply, BPF_ANY);
1036 if (err) {
1037 fprintf(stderr,
1038 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
1039 err, strerror(errno));
1040 goto out;
1041 }
1042 }
1043
1044 if (txmsg_cork) {
1045 err = bpf_map_update_elem(map_fd[4],
1046 &i, &txmsg_cork, BPF_ANY);
1047 if (err) {
1048 fprintf(stderr,
1049 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
1050 err, strerror(errno));
1051 goto out;
1052 }
1053 }
1054
1055 if (txmsg_start) {
1056 err = bpf_map_update_elem(map_fd[5],
1057 &i, &txmsg_start, BPF_ANY);
1058 if (err) {
1059 fprintf(stderr,
1060 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
1061 err, strerror(errno));
1062 goto out;
1063 }
1064 }
1065
1066 if (txmsg_end) {
1067 i = 1;
1068 err = bpf_map_update_elem(map_fd[5],
1069 &i, &txmsg_end, BPF_ANY);
1070 if (err) {
1071 fprintf(stderr,
1072 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
1073 err, strerror(errno));
1074 goto out;
1075 }
1076 }
1077
1078 if (txmsg_start_push) {
1079 i = 2;
1080 err = bpf_map_update_elem(map_fd[5],
1081 &i, &txmsg_start_push, BPF_ANY);
1082 if (err) {
1083 fprintf(stderr,
1084 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
1085 err, strerror(errno));
1086 goto out;
1087 }
1088 }
1089
1090 if (txmsg_end_push) {
1091 i = 3;
1092 err = bpf_map_update_elem(map_fd[5],
1093 &i, &txmsg_end_push, BPF_ANY);
1094 if (err) {
1095 fprintf(stderr,
1096 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
1097 txmsg_end_push, i, err, strerror(errno));
1098 goto out;
1099 }
1100 }
1101
1102 if (txmsg_start_pop) {
1103 i = 4;
1104 err = bpf_map_update_elem(map_fd[5],
1105 &i, &txmsg_start_pop, BPF_ANY);
1106 if (err) {
1107 fprintf(stderr,
1108 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
1109 txmsg_start_pop, i, err, strerror(errno));
1110 goto out;
1111 }
1112 } else {
1113 i = 4;
1114 bpf_map_update_elem(map_fd[5],
1115 &i, &txmsg_start_pop, BPF_ANY);
1116 }
1117
1118 if (txmsg_pop) {
1119 i = 5;
1120 err = bpf_map_update_elem(map_fd[5],
1121 &i, &txmsg_pop, BPF_ANY);
1122 if (err) {
1123 fprintf(stderr,
1124 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
1125 txmsg_pop, i, err, strerror(errno));
1126 goto out;
1127 }
1128 } else {
1129 i = 5;
1130 bpf_map_update_elem(map_fd[5],
1131 &i, &txmsg_pop, BPF_ANY);
1132
1133 }
1134
1135 if (txmsg_ingress) {
1136 int in = BPF_F_INGRESS;
1137
1138 i = 0;
1139 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1140 if (err) {
1141 fprintf(stderr,
1142 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1143 err, strerror(errno));
1144 }
1145 i = 1;
1146 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1147 if (err) {
1148 fprintf(stderr,
1149 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1150 err, strerror(errno));
1151 }
1152 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1153 if (err) {
1154 fprintf(stderr,
1155 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1156 err, strerror(errno));
1157 }
1158
1159 i = 2;
1160 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1161 if (err) {
1162 fprintf(stderr,
1163 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1164 err, strerror(errno));
1165 }
1166 }
1167
1168 if (txmsg_redir_skb) {
1169 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1170 p2 : p1;
1171 int ingress = BPF_F_INGRESS;
1172
1173 i = 0;
1174 err = bpf_map_update_elem(map_fd[7],
1175 &i, &ingress, BPF_ANY);
1176 if (err) {
1177 fprintf(stderr,
1178 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1179 err, strerror(errno));
1180 }
1181
1182 i = 3;
1183 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1184 if (err) {
1185 fprintf(stderr,
1186 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1187 err, strerror(errno));
1188 }
1189 }
1190 }
1191
1192 if (skb_use_parser) {
1193 i = 2;
1194 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1195 }
1196
1197 if (txmsg_drop)
1198 options->drop_expected = true;
1199
1200 if (test == PING_PONG)
1201 err = forever_ping_pong(options->rate, options);
1202 else if (test == SENDMSG) {
1203 options->base = false;
1204 options->sendpage = false;
1205 err = sendmsg_test(options);
1206 } else if (test == SENDPAGE) {
1207 options->base = false;
1208 options->sendpage = true;
1209 err = sendmsg_test(options);
1210 } else if (test == BASE) {
1211 options->base = true;
1212 options->sendpage = false;
1213 err = sendmsg_test(options);
1214 } else if (test == BASE_SENDPAGE) {
1215 options->base = true;
1216 options->sendpage = true;
1217 err = sendmsg_test(options);
1218 } else
1219 fprintf(stderr, "unknown test\n");
1220 out:
1221 /* Detach and zero all the maps */
1222 bpf_prog_detach2(bpf_program__fd(progs[2]), cg_fd, BPF_CGROUP_SOCK_OPS);
1223
1224 for (i = 0; i < ARRAY_SIZE(links); i++) {
1225 if (links[i])
1226 bpf_link__detach(links[i]);
1227 }
1228
1229 for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1230 key = next_key = 0;
1231 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1232 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1233 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1234 key = next_key;
1235 }
1236 }
1237
1238 close(s1);
1239 close(s2);
1240 close(p1);
1241 close(p2);
1242 close(c1);
1243 close(c2);
1244 return err;
1245 }
1246
test_to_str(int test)1247 static char *test_to_str(int test)
1248 {
1249 switch (test) {
1250 case SENDMSG:
1251 return "sendmsg";
1252 case SENDPAGE:
1253 return "sendpage";
1254 }
1255 return "unknown";
1256 }
1257
append_str(char * dst,const char * src,size_t dst_cap)1258 static void append_str(char *dst, const char *src, size_t dst_cap)
1259 {
1260 size_t avail = dst_cap - strlen(dst);
1261
1262 if (avail <= 1) /* just zero byte could be written */
1263 return;
1264
1265 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1266 }
1267
1268 #define OPTSTRING 60
test_options(char * options)1269 static void test_options(char *options)
1270 {
1271 char tstr[OPTSTRING];
1272
1273 memset(options, 0, OPTSTRING);
1274
1275 if (txmsg_pass)
1276 append_str(options, "pass,", OPTSTRING);
1277 if (txmsg_redir)
1278 append_str(options, "redir,", OPTSTRING);
1279 if (txmsg_drop)
1280 append_str(options, "drop,", OPTSTRING);
1281 if (txmsg_apply) {
1282 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1283 append_str(options, tstr, OPTSTRING);
1284 }
1285 if (txmsg_cork) {
1286 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1287 append_str(options, tstr, OPTSTRING);
1288 }
1289 if (txmsg_start) {
1290 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1291 append_str(options, tstr, OPTSTRING);
1292 }
1293 if (txmsg_end) {
1294 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1295 append_str(options, tstr, OPTSTRING);
1296 }
1297 if (txmsg_start_pop) {
1298 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1299 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1300 append_str(options, tstr, OPTSTRING);
1301 }
1302 if (txmsg_ingress)
1303 append_str(options, "ingress,", OPTSTRING);
1304 if (txmsg_redir_skb)
1305 append_str(options, "redir_skb,", OPTSTRING);
1306 if (peek_flag)
1307 append_str(options, "peek,", OPTSTRING);
1308 }
1309
__test_exec(int cgrp,int test,struct sockmap_options * opt)1310 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1311 {
1312 char *options = calloc(OPTSTRING, sizeof(char));
1313 int err;
1314
1315 if (test == SENDPAGE)
1316 opt->sendpage = true;
1317 else
1318 opt->sendpage = false;
1319
1320 if (txmsg_drop)
1321 opt->drop_expected = true;
1322 else
1323 opt->drop_expected = false;
1324
1325 test_options(options);
1326
1327 if (opt->verbose) {
1328 fprintf(stdout,
1329 " [TEST %i]: (%i, %i, %i, %s, %s): ",
1330 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1331 test_to_str(test), options);
1332 fflush(stdout);
1333 }
1334 err = run_options(opt, cgrp, test);
1335 if (opt->verbose)
1336 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1337 test_cnt++;
1338 !err ? passed++ : failed++;
1339 free(options);
1340 return err;
1341 }
1342
test_exec(int cgrp,struct sockmap_options * opt)1343 static void test_exec(int cgrp, struct sockmap_options *opt)
1344 {
1345 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1346 int err;
1347
1348 if (type == 0) {
1349 test_start();
1350 err = __test_exec(cgrp, SENDMSG, opt);
1351 if (err)
1352 test_fail();
1353 } else {
1354 test_start();
1355 err = __test_exec(cgrp, SENDPAGE, opt);
1356 if (err)
1357 test_fail();
1358 }
1359 }
1360
test_send_one(struct sockmap_options * opt,int cgrp)1361 static void test_send_one(struct sockmap_options *opt, int cgrp)
1362 {
1363 opt->iov_length = 1;
1364 opt->iov_count = 1;
1365 opt->rate = 1;
1366 test_exec(cgrp, opt);
1367
1368 opt->iov_length = 1;
1369 opt->iov_count = 1024;
1370 opt->rate = 1;
1371 test_exec(cgrp, opt);
1372
1373 opt->iov_length = 1024;
1374 opt->iov_count = 1;
1375 opt->rate = 1;
1376 test_exec(cgrp, opt);
1377
1378 }
1379
test_send_many(struct sockmap_options * opt,int cgrp)1380 static void test_send_many(struct sockmap_options *opt, int cgrp)
1381 {
1382 opt->iov_length = 3;
1383 opt->iov_count = 1;
1384 opt->rate = 512;
1385 test_exec(cgrp, opt);
1386
1387 opt->rate = 100;
1388 opt->iov_count = 1;
1389 opt->iov_length = 5;
1390 test_exec(cgrp, opt);
1391 }
1392
test_send_large(struct sockmap_options * opt,int cgrp)1393 static void test_send_large(struct sockmap_options *opt, int cgrp)
1394 {
1395 opt->iov_length = 8192;
1396 opt->iov_count = 32;
1397 opt->rate = 2;
1398 test_exec(cgrp, opt);
1399 }
1400
test_send(struct sockmap_options * opt,int cgrp)1401 static void test_send(struct sockmap_options *opt, int cgrp)
1402 {
1403 test_send_one(opt, cgrp);
1404 test_send_many(opt, cgrp);
1405 test_send_large(opt, cgrp);
1406 sched_yield();
1407 }
1408
test_txmsg_pass(int cgrp,struct sockmap_options * opt)1409 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1410 {
1411 /* Test small and large iov_count values with pass/redir/apply/cork */
1412 txmsg_pass = 1;
1413 test_send(opt, cgrp);
1414 }
1415
test_txmsg_redir(int cgrp,struct sockmap_options * opt)1416 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1417 {
1418 txmsg_redir = 1;
1419 test_send(opt, cgrp);
1420 }
1421
test_txmsg_redir_wait_sndmem(int cgrp,struct sockmap_options * opt)1422 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
1423 {
1424 opt->tx_wait_mem = true;
1425 txmsg_redir = 1;
1426 test_send_large(opt, cgrp);
1427
1428 txmsg_redir = 1;
1429 txmsg_apply = 4097;
1430 test_send_large(opt, cgrp);
1431 opt->tx_wait_mem = false;
1432 }
1433
test_txmsg_drop(int cgrp,struct sockmap_options * opt)1434 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1435 {
1436 txmsg_drop = 1;
1437 test_send(opt, cgrp);
1438 }
1439
test_txmsg_ingress_redir(int cgrp,struct sockmap_options * opt)1440 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1441 {
1442 txmsg_pass = txmsg_drop = 0;
1443 txmsg_ingress = txmsg_redir = 1;
1444 test_send(opt, cgrp);
1445 }
1446
1447 /* Test cork with hung data. This tests poor usage patterns where
1448 * cork can leave data on the ring if user program is buggy and
1449 * doesn't flush them somehow. They do take some time however
1450 * because they wait for a timeout. Test pass, redir and cork with
1451 * apply logic. Use cork size of 4097 with send_large to avoid
1452 * aligning cork size with send size.
1453 */
test_txmsg_cork_hangs(int cgrp,struct sockmap_options * opt)1454 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1455 {
1456 txmsg_pass = 1;
1457 txmsg_redir = 0;
1458 txmsg_cork = 4097;
1459 txmsg_apply = 4097;
1460 test_send_large(opt, cgrp);
1461
1462 txmsg_pass = 0;
1463 txmsg_redir = 1;
1464 txmsg_apply = 0;
1465 txmsg_cork = 4097;
1466 test_send_large(opt, cgrp);
1467
1468 txmsg_pass = 0;
1469 txmsg_redir = 1;
1470 txmsg_apply = 4097;
1471 txmsg_cork = 4097;
1472 test_send_large(opt, cgrp);
1473 }
1474
test_txmsg_pull(int cgrp,struct sockmap_options * opt)1475 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1476 {
1477 /* Test basic start/end */
1478 txmsg_pass = 1;
1479 txmsg_start = 1;
1480 txmsg_end = 2;
1481 test_send(opt, cgrp);
1482
1483 /* Test >4k pull */
1484 txmsg_pass = 1;
1485 txmsg_start = 4096;
1486 txmsg_end = 9182;
1487 test_send_large(opt, cgrp);
1488
1489 /* Test pull + redirect */
1490 txmsg_redir = 1;
1491 txmsg_start = 1;
1492 txmsg_end = 2;
1493 test_send(opt, cgrp);
1494
1495 /* Test pull + cork */
1496 txmsg_redir = 0;
1497 txmsg_cork = 512;
1498 txmsg_start = 1;
1499 txmsg_end = 2;
1500 test_send_many(opt, cgrp);
1501
1502 /* Test pull + cork + redirect */
1503 txmsg_redir = 1;
1504 txmsg_cork = 512;
1505 txmsg_start = 1;
1506 txmsg_end = 2;
1507 test_send_many(opt, cgrp);
1508 }
1509
test_txmsg_pop(int cgrp,struct sockmap_options * opt)1510 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1511 {
1512 bool data = opt->data_test;
1513
1514 /* Test basic pop */
1515 txmsg_pass = 1;
1516 txmsg_start_pop = 1;
1517 txmsg_pop = 2;
1518 test_send_many(opt, cgrp);
1519
1520 /* Test pop with >4k */
1521 txmsg_pass = 1;
1522 txmsg_start_pop = 4096;
1523 txmsg_pop = 4096;
1524 test_send_large(opt, cgrp);
1525
1526 /* Test pop + redirect */
1527 txmsg_redir = 1;
1528 txmsg_start_pop = 1;
1529 txmsg_pop = 2;
1530 test_send_many(opt, cgrp);
1531
1532 /* TODO: Test for pop + cork should be different,
1533 * - It makes the layout of the received data difficult
1534 * - It makes it hard to calculate the total_bytes in the recvmsg
1535 * Temporarily skip the data integrity test for this case now.
1536 */
1537 opt->data_test = false;
1538 /* Test pop + cork */
1539 txmsg_redir = 0;
1540 txmsg_cork = 512;
1541 txmsg_start_pop = 1;
1542 txmsg_pop = 2;
1543 test_send_many(opt, cgrp);
1544
1545 /* Test pop + redirect + cork */
1546 txmsg_redir = 1;
1547 txmsg_cork = 4;
1548 txmsg_start_pop = 1;
1549 txmsg_pop = 2;
1550 test_send_many(opt, cgrp);
1551 opt->data_test = data;
1552 }
1553
test_txmsg_push(int cgrp,struct sockmap_options * opt)1554 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1555 {
1556 bool data = opt->data_test;
1557
1558 /* Test basic push */
1559 txmsg_pass = 1;
1560 txmsg_start_push = 1;
1561 txmsg_end_push = 1;
1562 test_send(opt, cgrp);
1563
1564 /* Test push 4kB >4k */
1565 txmsg_pass = 1;
1566 txmsg_start_push = 4096;
1567 txmsg_end_push = 4096;
1568 test_send_large(opt, cgrp);
1569
1570 /* Test push + redirect */
1571 txmsg_redir = 1;
1572 txmsg_start_push = 1;
1573 txmsg_end_push = 2;
1574 test_send_many(opt, cgrp);
1575
1576 /* TODO: Test for push + cork should be different,
1577 * - It makes the layout of the received data difficult
1578 * - It makes it hard to calculate the total_bytes in the recvmsg
1579 * Temporarily skip the data integrity test for this case now.
1580 */
1581 opt->data_test = false;
1582 /* Test push + cork */
1583 txmsg_redir = 0;
1584 txmsg_cork = 512;
1585 txmsg_start_push = 1;
1586 txmsg_end_push = 2;
1587 test_send_many(opt, cgrp);
1588 opt->data_test = data;
1589 }
1590
test_txmsg_push_pop(int cgrp,struct sockmap_options * opt)1591 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1592 {
1593 /* Test push/pop range overlapping */
1594 txmsg_pass = 1;
1595 txmsg_start_push = 1;
1596 txmsg_end_push = 10;
1597 txmsg_start_pop = 5;
1598 txmsg_pop = 4;
1599 test_send_large(opt, cgrp);
1600
1601 txmsg_pass = 1;
1602 txmsg_start_push = 1;
1603 txmsg_end_push = 10;
1604 txmsg_start_pop = 5;
1605 txmsg_pop = 16;
1606 test_send_large(opt, cgrp);
1607
1608 txmsg_pass = 1;
1609 txmsg_start_push = 5;
1610 txmsg_end_push = 4;
1611 txmsg_start_pop = 1;
1612 txmsg_pop = 10;
1613 test_send_large(opt, cgrp);
1614
1615 txmsg_pass = 1;
1616 txmsg_start_push = 5;
1617 txmsg_end_push = 16;
1618 txmsg_start_pop = 1;
1619 txmsg_pop = 10;
1620 test_send_large(opt, cgrp);
1621
1622 /* Test push/pop range non-overlapping */
1623 txmsg_pass = 1;
1624 txmsg_start_push = 1;
1625 txmsg_end_push = 10;
1626 txmsg_start_pop = 16;
1627 txmsg_pop = 4;
1628 test_send_large(opt, cgrp);
1629
1630 txmsg_pass = 1;
1631 txmsg_start_push = 16;
1632 txmsg_end_push = 10;
1633 txmsg_start_pop = 5;
1634 txmsg_pop = 4;
1635 test_send_large(opt, cgrp);
1636 }
1637
test_txmsg_apply(int cgrp,struct sockmap_options * opt)1638 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1639 {
1640 txmsg_pass = 1;
1641 txmsg_redir = 0;
1642 txmsg_ingress = 0;
1643 txmsg_apply = 1;
1644 txmsg_cork = 0;
1645 test_send_one(opt, cgrp);
1646
1647 txmsg_pass = 0;
1648 txmsg_redir = 1;
1649 txmsg_ingress = 0;
1650 txmsg_apply = 1;
1651 txmsg_cork = 0;
1652 test_send_one(opt, cgrp);
1653
1654 txmsg_pass = 0;
1655 txmsg_redir = 1;
1656 txmsg_ingress = 1;
1657 txmsg_apply = 1;
1658 txmsg_cork = 0;
1659 test_send_one(opt, cgrp);
1660
1661 txmsg_pass = 1;
1662 txmsg_redir = 0;
1663 txmsg_ingress = 0;
1664 txmsg_apply = 1024;
1665 txmsg_cork = 0;
1666 test_send_large(opt, cgrp);
1667
1668 txmsg_pass = 0;
1669 txmsg_redir = 1;
1670 txmsg_ingress = 0;
1671 txmsg_apply = 1024;
1672 txmsg_cork = 0;
1673 test_send_large(opt, cgrp);
1674
1675 txmsg_pass = 0;
1676 txmsg_redir = 1;
1677 txmsg_ingress = 1;
1678 txmsg_apply = 1024;
1679 txmsg_cork = 0;
1680 test_send_large(opt, cgrp);
1681 }
1682
test_txmsg_cork(int cgrp,struct sockmap_options * opt)1683 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1684 {
1685 txmsg_pass = 1;
1686 txmsg_redir = 0;
1687 txmsg_apply = 0;
1688 txmsg_cork = 1;
1689 test_send(opt, cgrp);
1690
1691 txmsg_pass = 1;
1692 txmsg_redir = 0;
1693 txmsg_apply = 1;
1694 txmsg_cork = 1;
1695 test_send(opt, cgrp);
1696 }
1697
test_txmsg_ingress_parser(int cgrp,struct sockmap_options * opt)1698 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1699 {
1700 txmsg_pass = 1;
1701 skb_use_parser = 512;
1702 opt->iov_length = 256;
1703 opt->iov_count = 1;
1704 opt->rate = 2;
1705 test_exec(cgrp, opt);
1706 }
1707
test_txmsg_ingress_parser2(int cgrp,struct sockmap_options * opt)1708 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1709 {
1710 skb_use_parser = 10;
1711 opt->iov_length = 20;
1712 opt->iov_count = 1;
1713 opt->rate = 1;
1714 opt->check_recved_len = true;
1715 test_exec(cgrp, opt);
1716 opt->check_recved_len = false;
1717 }
1718
1719 char *map_names[] = {
1720 "sock_map",
1721 "sock_map_txmsg",
1722 "sock_map_redir",
1723 "sock_apply_bytes",
1724 "sock_cork_bytes",
1725 "sock_bytes",
1726 "sock_redir_flags",
1727 "sock_skb_opts",
1728 };
1729
populate_progs(char * bpf_file)1730 static int populate_progs(char *bpf_file)
1731 {
1732 struct bpf_program *prog;
1733 struct bpf_object *obj;
1734 int i = 0;
1735 long err;
1736
1737 obj = bpf_object__open(bpf_file);
1738 err = libbpf_get_error(obj);
1739 if (err) {
1740 char err_buf[256];
1741
1742 libbpf_strerror(err, err_buf, sizeof(err_buf));
1743 printf("Unable to load eBPF objects in file '%s' : %s\n",
1744 bpf_file, err_buf);
1745 return -1;
1746 }
1747
1748 i = bpf_object__load(obj);
1749 i = 0;
1750 bpf_object__for_each_program(prog, obj) {
1751 progs[i] = prog;
1752 i++;
1753 }
1754
1755 for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1756 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1757 map_fd[i] = bpf_map__fd(maps[i]);
1758 if (map_fd[i] < 0) {
1759 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1760 map_fd[i], strerror(errno));
1761 return -1;
1762 }
1763 }
1764
1765 for (i = 0; i < ARRAY_SIZE(links); i++)
1766 links[i] = NULL;
1767
1768 return 0;
1769 }
1770
1771 struct _test test[] = {
1772 {"txmsg test passthrough", test_txmsg_pass},
1773 {"txmsg test redirect", test_txmsg_redir},
1774 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
1775 {"txmsg test drop", test_txmsg_drop},
1776 {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1777 {"txmsg test apply", test_txmsg_apply},
1778 {"txmsg test cork", test_txmsg_cork},
1779 {"txmsg test hanging corks", test_txmsg_cork_hangs},
1780 {"txmsg test push_data", test_txmsg_push},
1781 {"txmsg test pull-data", test_txmsg_pull},
1782 {"txmsg test pop-data", test_txmsg_pop},
1783 {"txmsg test push/pop data", test_txmsg_push_pop},
1784 {"txmsg test ingress parser", test_txmsg_ingress_parser},
1785 {"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1786 };
1787
check_whitelist(struct _test * t,struct sockmap_options * opt)1788 static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1789 {
1790 char *entry, *ptr;
1791
1792 if (!opt->whitelist)
1793 return 0;
1794 ptr = strdup(opt->whitelist);
1795 if (!ptr)
1796 return -ENOMEM;
1797 entry = strtok(ptr, ",");
1798 while (entry) {
1799 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1800 strstr(opt->map, entry) != 0 ||
1801 strstr(t->title, entry) != 0) {
1802 free(ptr);
1803 return 0;
1804 }
1805 entry = strtok(NULL, ",");
1806 }
1807 free(ptr);
1808 return -EINVAL;
1809 }
1810
check_blacklist(struct _test * t,struct sockmap_options * opt)1811 static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1812 {
1813 char *entry, *ptr;
1814
1815 if (!opt->blacklist)
1816 return -EINVAL;
1817 ptr = strdup(opt->blacklist);
1818 if (!ptr)
1819 return -ENOMEM;
1820 entry = strtok(ptr, ",");
1821 while (entry) {
1822 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1823 strstr(opt->map, entry) != 0 ||
1824 strstr(t->title, entry) != 0) {
1825 free(ptr);
1826 return 0;
1827 }
1828 entry = strtok(NULL, ",");
1829 }
1830 free(ptr);
1831 return -EINVAL;
1832 }
1833
__test_selftests(int cg_fd,struct sockmap_options * opt)1834 static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1835 {
1836 int i, err;
1837
1838 err = populate_progs(opt->map);
1839 if (err < 0) {
1840 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1841 return err;
1842 }
1843
1844 /* Tests basic commands and APIs */
1845 for (i = 0; i < ARRAY_SIZE(test); i++) {
1846 struct _test t = test[i];
1847
1848 if (check_whitelist(&t, opt) != 0)
1849 continue;
1850 if (check_blacklist(&t, opt) == 0)
1851 continue;
1852
1853 test_start_subtest(&t, opt);
1854 t.tester(cg_fd, opt);
1855 test_end_subtest();
1856 }
1857
1858 return err;
1859 }
1860
test_selftests_sockmap(int cg_fd,struct sockmap_options * opt)1861 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1862 {
1863 opt->map = BPF_SOCKMAP_FILENAME;
1864 __test_selftests(cg_fd, opt);
1865 }
1866
test_selftests_sockhash(int cg_fd,struct sockmap_options * opt)1867 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1868 {
1869 opt->map = BPF_SOCKHASH_FILENAME;
1870 __test_selftests(cg_fd, opt);
1871 }
1872
test_selftest(int cg_fd,struct sockmap_options * opt)1873 static int test_selftest(int cg_fd, struct sockmap_options *opt)
1874 {
1875 test_selftests_sockmap(cg_fd, opt);
1876 test_selftests_sockhash(cg_fd, opt);
1877 test_print_results();
1878 return 0;
1879 }
1880
main(int argc,char ** argv)1881 int main(int argc, char **argv)
1882 {
1883 int iov_count = 1, length = 1024, rate = 1;
1884 struct sockmap_options options = {0};
1885 int opt, longindex, err, cg_fd = 0;
1886 char *bpf_file = BPF_SOCKMAP_FILENAME;
1887 int test = SELFTESTS;
1888 bool cg_created = 0;
1889
1890 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1891 long_options, &longindex)) != -1) {
1892 switch (opt) {
1893 case 's':
1894 txmsg_start = atoi(optarg);
1895 break;
1896 case 'e':
1897 txmsg_end = atoi(optarg);
1898 break;
1899 case 'p':
1900 txmsg_start_push = atoi(optarg);
1901 break;
1902 case 'q':
1903 txmsg_end_push = atoi(optarg);
1904 break;
1905 case 'w':
1906 txmsg_start_pop = atoi(optarg);
1907 break;
1908 case 'x':
1909 txmsg_pop = atoi(optarg);
1910 break;
1911 case 'a':
1912 txmsg_apply = atoi(optarg);
1913 break;
1914 case 'k':
1915 txmsg_cork = atoi(optarg);
1916 break;
1917 case 'c':
1918 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1919 if (cg_fd < 0) {
1920 fprintf(stderr,
1921 "ERROR: (%i) open cg path failed: %s\n",
1922 cg_fd, optarg);
1923 return cg_fd;
1924 }
1925 break;
1926 case 'r':
1927 rate = atoi(optarg);
1928 break;
1929 case 'v':
1930 options.verbose = 1;
1931 if (optarg)
1932 options.verbose = atoi(optarg);
1933 break;
1934 case 'i':
1935 iov_count = atoi(optarg);
1936 break;
1937 case 'l':
1938 length = atoi(optarg);
1939 break;
1940 case 'd':
1941 options.data_test = true;
1942 break;
1943 case 't':
1944 if (strcmp(optarg, "ping") == 0) {
1945 test = PING_PONG;
1946 } else if (strcmp(optarg, "sendmsg") == 0) {
1947 test = SENDMSG;
1948 } else if (strcmp(optarg, "base") == 0) {
1949 test = BASE;
1950 } else if (strcmp(optarg, "base_sendpage") == 0) {
1951 test = BASE_SENDPAGE;
1952 } else if (strcmp(optarg, "sendpage") == 0) {
1953 test = SENDPAGE;
1954 } else {
1955 usage(argv);
1956 return -1;
1957 }
1958 break;
1959 case 'n':
1960 options.whitelist = strdup(optarg);
1961 if (!options.whitelist)
1962 return -ENOMEM;
1963 break;
1964 case 'b':
1965 options.blacklist = strdup(optarg);
1966 if (!options.blacklist)
1967 return -ENOMEM;
1968 case 0:
1969 break;
1970 case 'h':
1971 default:
1972 usage(argv);
1973 return -1;
1974 }
1975 }
1976
1977 if (!cg_fd) {
1978 cg_fd = cgroup_setup_and_join(CG_PATH);
1979 if (cg_fd < 0)
1980 return cg_fd;
1981 cg_created = 1;
1982 }
1983
1984 /* Use libbpf 1.0 API mode */
1985 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1986
1987 if (test == SELFTESTS) {
1988 err = test_selftest(cg_fd, &options);
1989 goto out;
1990 }
1991
1992 err = populate_progs(bpf_file);
1993 if (err) {
1994 fprintf(stderr, "populate program: (%s) %s\n",
1995 bpf_file, strerror(errno));
1996 return 1;
1997 }
1998 running = 1;
1999
2000 /* catch SIGINT */
2001 signal(SIGINT, running_handler);
2002
2003 options.iov_count = iov_count;
2004 options.iov_length = length;
2005 options.rate = rate;
2006
2007 err = run_options(&options, cg_fd, test);
2008 out:
2009 if (options.whitelist)
2010 free(options.whitelist);
2011 if (options.blacklist)
2012 free(options.blacklist);
2013 close(cg_fd);
2014 if (cg_created)
2015 cleanup_cgroup_environment();
2016 return err;
2017 }
2018
running_handler(int a)2019 void running_handler(int a)
2020 {
2021 running = 0;
2022 }
2023