1 /* Copyright (c) 2017 Facebook 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 */ 7 #include <stdio.h> 8 #include <unistd.h> 9 #include <errno.h> 10 #include <string.h> 11 #include <assert.h> 12 #include <stdlib.h> 13 #include <stdarg.h> 14 #include <time.h> 15 16 #include <linux/types.h> 17 typedef __u16 __sum16; 18 #include <arpa/inet.h> 19 #include <linux/if_ether.h> 20 #include <linux/if_packet.h> 21 #include <linux/ip.h> 22 #include <linux/ipv6.h> 23 #include <linux/tcp.h> 24 #include <linux/filter.h> 25 #include <linux/perf_event.h> 26 #include <linux/unistd.h> 27 28 #include <sys/ioctl.h> 29 #include <sys/wait.h> 30 #include <sys/types.h> 31 #include <fcntl.h> 32 #include <pthread.h> 33 #include <linux/bpf.h> 34 #include <linux/err.h> 35 #include <bpf/bpf.h> 36 #include <bpf/libbpf.h> 37 38 #include "test_iptunnel_common.h" 39 #include "bpf_util.h" 40 #include "bpf_endian.h" 41 #include "bpf_rlimit.h" 42 #include "trace_helpers.h" 43 #include "flow_dissector_load.h" 44 45 static int error_cnt, pass_cnt; 46 static bool jit_enabled; 47 48 #define MAGIC_BYTES 123 49 50 /* ipv4 test vector */ 51 static struct { 52 struct ethhdr eth; 53 struct iphdr iph; 54 struct tcphdr tcp; 55 } __packed pkt_v4 = { 56 .eth.h_proto = __bpf_constant_htons(ETH_P_IP), 57 .iph.ihl = 5, 58 .iph.protocol = IPPROTO_TCP, 59 .iph.tot_len = __bpf_constant_htons(MAGIC_BYTES), 60 .tcp.urg_ptr = 123, 61 .tcp.doff = 5, 62 }; 63 64 /* ipv6 test vector */ 65 static struct { 66 struct ethhdr eth; 67 struct ipv6hdr iph; 68 struct tcphdr tcp; 69 } __packed pkt_v6 = { 70 .eth.h_proto = __bpf_constant_htons(ETH_P_IPV6), 71 .iph.nexthdr = IPPROTO_TCP, 72 .iph.payload_len = __bpf_constant_htons(MAGIC_BYTES), 73 .tcp.urg_ptr = 123, 74 .tcp.doff = 5, 75 }; 76 77 #define _CHECK(condition, tag, duration, format...) ({ \ 78 int __ret = !!(condition); \ 79 if (__ret) { \ 80 error_cnt++; \ 81 printf("%s:FAIL:%s ", __func__, tag); \ 82 printf(format); \ 83 } else { \ 84 pass_cnt++; \ 85 printf("%s:PASS:%s %d nsec\n", __func__, tag, duration);\ 86 } \ 87 __ret; \ 88 }) 89 90 #define CHECK(condition, tag, format...) \ 91 _CHECK(condition, tag, duration, format) 92 #define CHECK_ATTR(condition, tag, format...) \ 93 _CHECK(condition, tag, tattr.duration, format) 94 95 static int bpf_find_map(const char *test, struct bpf_object *obj, 96 const char *name) 97 { 98 struct bpf_map *map; 99 100 map = bpf_object__find_map_by_name(obj, name); 101 if (!map) { 102 printf("%s:FAIL:map '%s' not found\n", test, name); 103 error_cnt++; 104 return -1; 105 } 106 return bpf_map__fd(map); 107 } 108 109 static void test_pkt_access(void) 110 { 111 const char *file = "./test_pkt_access.o"; 112 struct bpf_object *obj; 113 __u32 duration, retval; 114 int err, prog_fd; 115 116 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); 117 if (err) { 118 error_cnt++; 119 return; 120 } 121 122 err = bpf_prog_test_run(prog_fd, 100000, &pkt_v4, sizeof(pkt_v4), 123 NULL, NULL, &retval, &duration); 124 CHECK(err || retval, "ipv4", 125 "err %d errno %d retval %d duration %d\n", 126 err, errno, retval, duration); 127 128 err = bpf_prog_test_run(prog_fd, 100000, &pkt_v6, sizeof(pkt_v6), 129 NULL, NULL, &retval, &duration); 130 CHECK(err || retval, "ipv6", 131 "err %d errno %d retval %d duration %d\n", 132 err, errno, retval, duration); 133 bpf_object__close(obj); 134 } 135 136 static void test_prog_run_xattr(void) 137 { 138 const char *file = "./test_pkt_access.o"; 139 struct bpf_object *obj; 140 char buf[10]; 141 int err; 142 struct bpf_prog_test_run_attr tattr = { 143 .repeat = 1, 144 .data_in = &pkt_v4, 145 .data_size_in = sizeof(pkt_v4), 146 .data_out = buf, 147 .data_size_out = 5, 148 }; 149 150 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, 151 &tattr.prog_fd); 152 if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno)) 153 return; 154 155 memset(buf, 0, sizeof(buf)); 156 157 err = bpf_prog_test_run_xattr(&tattr); 158 CHECK_ATTR(err != -1 || errno != ENOSPC || tattr.retval, "run", 159 "err %d errno %d retval %d\n", err, errno, tattr.retval); 160 161 CHECK_ATTR(tattr.data_size_out != sizeof(pkt_v4), "data_size_out", 162 "incorrect output size, want %lu have %u\n", 163 sizeof(pkt_v4), tattr.data_size_out); 164 165 CHECK_ATTR(buf[5] != 0, "overflow", 166 "BPF_PROG_TEST_RUN ignored size hint\n"); 167 168 tattr.data_out = NULL; 169 tattr.data_size_out = 0; 170 errno = 0; 171 172 err = bpf_prog_test_run_xattr(&tattr); 173 CHECK_ATTR(err || errno || tattr.retval, "run_no_output", 174 "err %d errno %d retval %d\n", err, errno, tattr.retval); 175 176 tattr.data_size_out = 1; 177 err = bpf_prog_test_run_xattr(&tattr); 178 CHECK_ATTR(err != -EINVAL, "run_wrong_size_out", "err %d\n", err); 179 180 bpf_object__close(obj); 181 } 182 183 static void test_xdp(void) 184 { 185 struct vip key4 = {.protocol = 6, .family = AF_INET}; 186 struct vip key6 = {.protocol = 6, .family = AF_INET6}; 187 struct iptnl_info value4 = {.family = AF_INET}; 188 struct iptnl_info value6 = {.family = AF_INET6}; 189 const char *file = "./test_xdp.o"; 190 struct bpf_object *obj; 191 char buf[128]; 192 struct ipv6hdr *iph6 = (void *)buf + sizeof(struct ethhdr); 193 struct iphdr *iph = (void *)buf + sizeof(struct ethhdr); 194 __u32 duration, retval, size; 195 int err, prog_fd, map_fd; 196 197 err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); 198 if (err) { 199 error_cnt++; 200 return; 201 } 202 203 map_fd = bpf_find_map(__func__, obj, "vip2tnl"); 204 if (map_fd < 0) 205 goto out; 206 bpf_map_update_elem(map_fd, &key4, &value4, 0); 207 bpf_map_update_elem(map_fd, &key6, &value6, 0); 208 209 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4), 210 buf, &size, &retval, &duration); 211 212 CHECK(err || retval != XDP_TX || size != 74 || 213 iph->protocol != IPPROTO_IPIP, "ipv4", 214 "err %d errno %d retval %d size %d\n", 215 err, errno, retval, size); 216 217 err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6), 218 buf, &size, &retval, &duration); 219 CHECK(err || retval != XDP_TX || size != 114 || 220 iph6->nexthdr != IPPROTO_IPV6, "ipv6", 221 "err %d errno %d retval %d size %d\n", 222 err, errno, retval, size); 223 out: 224 bpf_object__close(obj); 225 } 226 227 static void test_xdp_adjust_tail(void) 228 { 229 const char *file = "./test_adjust_tail.o"; 230 struct bpf_object *obj; 231 char buf[128]; 232 __u32 duration, retval, size; 233 int err, prog_fd; 234 235 err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); 236 if (err) { 237 error_cnt++; 238 return; 239 } 240 241 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4), 242 buf, &size, &retval, &duration); 243 244 CHECK(err || retval != XDP_DROP, 245 "ipv4", "err %d errno %d retval %d size %d\n", 246 err, errno, retval, size); 247 248 err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6), 249 buf, &size, &retval, &duration); 250 CHECK(err || retval != XDP_TX || size != 54, 251 "ipv6", "err %d errno %d retval %d size %d\n", 252 err, errno, retval, size); 253 bpf_object__close(obj); 254 } 255 256 257 258 #define MAGIC_VAL 0x1234 259 #define NUM_ITER 100000 260 #define VIP_NUM 5 261 262 static void test_l4lb(const char *file) 263 { 264 unsigned int nr_cpus = bpf_num_possible_cpus(); 265 struct vip key = {.protocol = 6}; 266 struct vip_meta { 267 __u32 flags; 268 __u32 vip_num; 269 } value = {.vip_num = VIP_NUM}; 270 __u32 stats_key = VIP_NUM; 271 struct vip_stats { 272 __u64 bytes; 273 __u64 pkts; 274 } stats[nr_cpus]; 275 struct real_definition { 276 union { 277 __be32 dst; 278 __be32 dstv6[4]; 279 }; 280 __u8 flags; 281 } real_def = {.dst = MAGIC_VAL}; 282 __u32 ch_key = 11, real_num = 3; 283 __u32 duration, retval, size; 284 int err, i, prog_fd, map_fd; 285 __u64 bytes = 0, pkts = 0; 286 struct bpf_object *obj; 287 char buf[128]; 288 u32 *magic = (u32 *)buf; 289 290 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); 291 if (err) { 292 error_cnt++; 293 return; 294 } 295 296 map_fd = bpf_find_map(__func__, obj, "vip_map"); 297 if (map_fd < 0) 298 goto out; 299 bpf_map_update_elem(map_fd, &key, &value, 0); 300 301 map_fd = bpf_find_map(__func__, obj, "ch_rings"); 302 if (map_fd < 0) 303 goto out; 304 bpf_map_update_elem(map_fd, &ch_key, &real_num, 0); 305 306 map_fd = bpf_find_map(__func__, obj, "reals"); 307 if (map_fd < 0) 308 goto out; 309 bpf_map_update_elem(map_fd, &real_num, &real_def, 0); 310 311 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4), 312 buf, &size, &retval, &duration); 313 CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 54 || 314 *magic != MAGIC_VAL, "ipv4", 315 "err %d errno %d retval %d size %d magic %x\n", 316 err, errno, retval, size, *magic); 317 318 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6), 319 buf, &size, &retval, &duration); 320 CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 74 || 321 *magic != MAGIC_VAL, "ipv6", 322 "err %d errno %d retval %d size %d magic %x\n", 323 err, errno, retval, size, *magic); 324 325 map_fd = bpf_find_map(__func__, obj, "stats"); 326 if (map_fd < 0) 327 goto out; 328 bpf_map_lookup_elem(map_fd, &stats_key, stats); 329 for (i = 0; i < nr_cpus; i++) { 330 bytes += stats[i].bytes; 331 pkts += stats[i].pkts; 332 } 333 if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) { 334 error_cnt++; 335 printf("test_l4lb:FAIL:stats %lld %lld\n", bytes, pkts); 336 } 337 out: 338 bpf_object__close(obj); 339 } 340 341 static void test_l4lb_all(void) 342 { 343 const char *file1 = "./test_l4lb.o"; 344 const char *file2 = "./test_l4lb_noinline.o"; 345 346 test_l4lb(file1); 347 test_l4lb(file2); 348 } 349 350 static void test_xdp_noinline(void) 351 { 352 const char *file = "./test_xdp_noinline.o"; 353 unsigned int nr_cpus = bpf_num_possible_cpus(); 354 struct vip key = {.protocol = 6}; 355 struct vip_meta { 356 __u32 flags; 357 __u32 vip_num; 358 } value = {.vip_num = VIP_NUM}; 359 __u32 stats_key = VIP_NUM; 360 struct vip_stats { 361 __u64 bytes; 362 __u64 pkts; 363 } stats[nr_cpus]; 364 struct real_definition { 365 union { 366 __be32 dst; 367 __be32 dstv6[4]; 368 }; 369 __u8 flags; 370 } real_def = {.dst = MAGIC_VAL}; 371 __u32 ch_key = 11, real_num = 3; 372 __u32 duration, retval, size; 373 int err, i, prog_fd, map_fd; 374 __u64 bytes = 0, pkts = 0; 375 struct bpf_object *obj; 376 char buf[128]; 377 u32 *magic = (u32 *)buf; 378 379 err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); 380 if (err) { 381 error_cnt++; 382 return; 383 } 384 385 map_fd = bpf_find_map(__func__, obj, "vip_map"); 386 if (map_fd < 0) 387 goto out; 388 bpf_map_update_elem(map_fd, &key, &value, 0); 389 390 map_fd = bpf_find_map(__func__, obj, "ch_rings"); 391 if (map_fd < 0) 392 goto out; 393 bpf_map_update_elem(map_fd, &ch_key, &real_num, 0); 394 395 map_fd = bpf_find_map(__func__, obj, "reals"); 396 if (map_fd < 0) 397 goto out; 398 bpf_map_update_elem(map_fd, &real_num, &real_def, 0); 399 400 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4), 401 buf, &size, &retval, &duration); 402 CHECK(err || retval != 1 || size != 54 || 403 *magic != MAGIC_VAL, "ipv4", 404 "err %d errno %d retval %d size %d magic %x\n", 405 err, errno, retval, size, *magic); 406 407 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6), 408 buf, &size, &retval, &duration); 409 CHECK(err || retval != 1 || size != 74 || 410 *magic != MAGIC_VAL, "ipv6", 411 "err %d errno %d retval %d size %d magic %x\n", 412 err, errno, retval, size, *magic); 413 414 map_fd = bpf_find_map(__func__, obj, "stats"); 415 if (map_fd < 0) 416 goto out; 417 bpf_map_lookup_elem(map_fd, &stats_key, stats); 418 for (i = 0; i < nr_cpus; i++) { 419 bytes += stats[i].bytes; 420 pkts += stats[i].pkts; 421 } 422 if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) { 423 error_cnt++; 424 printf("test_xdp_noinline:FAIL:stats %lld %lld\n", bytes, pkts); 425 } 426 out: 427 bpf_object__close(obj); 428 } 429 430 static void test_tcp_estats(void) 431 { 432 const char *file = "./test_tcp_estats.o"; 433 int err, prog_fd; 434 struct bpf_object *obj; 435 __u32 duration = 0; 436 437 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); 438 CHECK(err, "", "err %d errno %d\n", err, errno); 439 if (err) { 440 error_cnt++; 441 return; 442 } 443 444 bpf_object__close(obj); 445 } 446 447 static inline __u64 ptr_to_u64(const void *ptr) 448 { 449 return (__u64) (unsigned long) ptr; 450 } 451 452 static bool is_jit_enabled(void) 453 { 454 const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable"; 455 bool enabled = false; 456 int sysctl_fd; 457 458 sysctl_fd = open(jit_sysctl, 0, O_RDONLY); 459 if (sysctl_fd != -1) { 460 char tmpc; 461 462 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1) 463 enabled = (tmpc != '0'); 464 close(sysctl_fd); 465 } 466 467 return enabled; 468 } 469 470 static void test_bpf_obj_id(void) 471 { 472 const __u64 array_magic_value = 0xfaceb00c; 473 const __u32 array_key = 0; 474 const int nr_iters = 2; 475 const char *file = "./test_obj_id.o"; 476 const char *expected_prog_name = "test_obj_id"; 477 const char *expected_map_name = "test_map_id"; 478 const __u64 nsec_per_sec = 1000000000; 479 480 struct bpf_object *objs[nr_iters]; 481 int prog_fds[nr_iters], map_fds[nr_iters]; 482 /* +1 to test for the info_len returned by kernel */ 483 struct bpf_prog_info prog_infos[nr_iters + 1]; 484 struct bpf_map_info map_infos[nr_iters + 1]; 485 /* Each prog only uses one map. +1 to test nr_map_ids 486 * returned by kernel. 487 */ 488 __u32 map_ids[nr_iters + 1]; 489 char jited_insns[128], xlated_insns[128], zeros[128]; 490 __u32 i, next_id, info_len, nr_id_found, duration = 0; 491 struct timespec real_time_ts, boot_time_ts; 492 int err = 0; 493 __u64 array_value; 494 uid_t my_uid = getuid(); 495 time_t now, load_time; 496 497 err = bpf_prog_get_fd_by_id(0); 498 CHECK(err >= 0 || errno != ENOENT, 499 "get-fd-by-notexist-prog-id", "err %d errno %d\n", err, errno); 500 501 err = bpf_map_get_fd_by_id(0); 502 CHECK(err >= 0 || errno != ENOENT, 503 "get-fd-by-notexist-map-id", "err %d errno %d\n", err, errno); 504 505 for (i = 0; i < nr_iters; i++) 506 objs[i] = NULL; 507 508 /* Check bpf_obj_get_info_by_fd() */ 509 bzero(zeros, sizeof(zeros)); 510 for (i = 0; i < nr_iters; i++) { 511 now = time(NULL); 512 err = bpf_prog_load(file, BPF_PROG_TYPE_SOCKET_FILTER, 513 &objs[i], &prog_fds[i]); 514 /* test_obj_id.o is a dumb prog. It should never fail 515 * to load. 516 */ 517 if (err) 518 error_cnt++; 519 assert(!err); 520 521 /* Insert a magic value to the map */ 522 map_fds[i] = bpf_find_map(__func__, objs[i], "test_map_id"); 523 assert(map_fds[i] >= 0); 524 err = bpf_map_update_elem(map_fds[i], &array_key, 525 &array_magic_value, 0); 526 assert(!err); 527 528 /* Check getting map info */ 529 info_len = sizeof(struct bpf_map_info) * 2; 530 bzero(&map_infos[i], info_len); 531 err = bpf_obj_get_info_by_fd(map_fds[i], &map_infos[i], 532 &info_len); 533 if (CHECK(err || 534 map_infos[i].type != BPF_MAP_TYPE_ARRAY || 535 map_infos[i].key_size != sizeof(__u32) || 536 map_infos[i].value_size != sizeof(__u64) || 537 map_infos[i].max_entries != 1 || 538 map_infos[i].map_flags != 0 || 539 info_len != sizeof(struct bpf_map_info) || 540 strcmp((char *)map_infos[i].name, expected_map_name), 541 "get-map-info(fd)", 542 "err %d errno %d type %d(%d) info_len %u(%Zu) key_size %u value_size %u max_entries %u map_flags %X name %s(%s)\n", 543 err, errno, 544 map_infos[i].type, BPF_MAP_TYPE_ARRAY, 545 info_len, sizeof(struct bpf_map_info), 546 map_infos[i].key_size, 547 map_infos[i].value_size, 548 map_infos[i].max_entries, 549 map_infos[i].map_flags, 550 map_infos[i].name, expected_map_name)) 551 goto done; 552 553 /* Check getting prog info */ 554 info_len = sizeof(struct bpf_prog_info) * 2; 555 bzero(&prog_infos[i], info_len); 556 bzero(jited_insns, sizeof(jited_insns)); 557 bzero(xlated_insns, sizeof(xlated_insns)); 558 prog_infos[i].jited_prog_insns = ptr_to_u64(jited_insns); 559 prog_infos[i].jited_prog_len = sizeof(jited_insns); 560 prog_infos[i].xlated_prog_insns = ptr_to_u64(xlated_insns); 561 prog_infos[i].xlated_prog_len = sizeof(xlated_insns); 562 prog_infos[i].map_ids = ptr_to_u64(map_ids + i); 563 prog_infos[i].nr_map_ids = 2; 564 err = clock_gettime(CLOCK_REALTIME, &real_time_ts); 565 assert(!err); 566 err = clock_gettime(CLOCK_BOOTTIME, &boot_time_ts); 567 assert(!err); 568 err = bpf_obj_get_info_by_fd(prog_fds[i], &prog_infos[i], 569 &info_len); 570 load_time = (real_time_ts.tv_sec - boot_time_ts.tv_sec) 571 + (prog_infos[i].load_time / nsec_per_sec); 572 if (CHECK(err || 573 prog_infos[i].type != BPF_PROG_TYPE_SOCKET_FILTER || 574 info_len != sizeof(struct bpf_prog_info) || 575 (jit_enabled && !prog_infos[i].jited_prog_len) || 576 (jit_enabled && 577 !memcmp(jited_insns, zeros, sizeof(zeros))) || 578 !prog_infos[i].xlated_prog_len || 579 !memcmp(xlated_insns, zeros, sizeof(zeros)) || 580 load_time < now - 60 || load_time > now + 60 || 581 prog_infos[i].created_by_uid != my_uid || 582 prog_infos[i].nr_map_ids != 1 || 583 *(int *)(long)prog_infos[i].map_ids != map_infos[i].id || 584 strcmp((char *)prog_infos[i].name, expected_prog_name), 585 "get-prog-info(fd)", 586 "err %d errno %d i %d type %d(%d) info_len %u(%Zu) jit_enabled %d jited_prog_len %u xlated_prog_len %u jited_prog %d xlated_prog %d load_time %lu(%lu) uid %u(%u) nr_map_ids %u(%u) map_id %u(%u) name %s(%s)\n", 587 err, errno, i, 588 prog_infos[i].type, BPF_PROG_TYPE_SOCKET_FILTER, 589 info_len, sizeof(struct bpf_prog_info), 590 jit_enabled, 591 prog_infos[i].jited_prog_len, 592 prog_infos[i].xlated_prog_len, 593 !!memcmp(jited_insns, zeros, sizeof(zeros)), 594 !!memcmp(xlated_insns, zeros, sizeof(zeros)), 595 load_time, now, 596 prog_infos[i].created_by_uid, my_uid, 597 prog_infos[i].nr_map_ids, 1, 598 *(int *)(long)prog_infos[i].map_ids, map_infos[i].id, 599 prog_infos[i].name, expected_prog_name)) 600 goto done; 601 } 602 603 /* Check bpf_prog_get_next_id() */ 604 nr_id_found = 0; 605 next_id = 0; 606 while (!bpf_prog_get_next_id(next_id, &next_id)) { 607 struct bpf_prog_info prog_info = {}; 608 __u32 saved_map_id; 609 int prog_fd; 610 611 info_len = sizeof(prog_info); 612 613 prog_fd = bpf_prog_get_fd_by_id(next_id); 614 if (prog_fd < 0 && errno == ENOENT) 615 /* The bpf_prog is in the dead row */ 616 continue; 617 if (CHECK(prog_fd < 0, "get-prog-fd(next_id)", 618 "prog_fd %d next_id %d errno %d\n", 619 prog_fd, next_id, errno)) 620 break; 621 622 for (i = 0; i < nr_iters; i++) 623 if (prog_infos[i].id == next_id) 624 break; 625 626 if (i == nr_iters) 627 continue; 628 629 nr_id_found++; 630 631 /* Negative test: 632 * prog_info.nr_map_ids = 1 633 * prog_info.map_ids = NULL 634 */ 635 prog_info.nr_map_ids = 1; 636 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len); 637 if (CHECK(!err || errno != EFAULT, 638 "get-prog-fd-bad-nr-map-ids", "err %d errno %d(%d)", 639 err, errno, EFAULT)) 640 break; 641 bzero(&prog_info, sizeof(prog_info)); 642 info_len = sizeof(prog_info); 643 644 saved_map_id = *(int *)((long)prog_infos[i].map_ids); 645 prog_info.map_ids = prog_infos[i].map_ids; 646 prog_info.nr_map_ids = 2; 647 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len); 648 prog_infos[i].jited_prog_insns = 0; 649 prog_infos[i].xlated_prog_insns = 0; 650 CHECK(err || info_len != sizeof(struct bpf_prog_info) || 651 memcmp(&prog_info, &prog_infos[i], info_len) || 652 *(int *)(long)prog_info.map_ids != saved_map_id, 653 "get-prog-info(next_id->fd)", 654 "err %d errno %d info_len %u(%Zu) memcmp %d map_id %u(%u)\n", 655 err, errno, info_len, sizeof(struct bpf_prog_info), 656 memcmp(&prog_info, &prog_infos[i], info_len), 657 *(int *)(long)prog_info.map_ids, saved_map_id); 658 close(prog_fd); 659 } 660 CHECK(nr_id_found != nr_iters, 661 "check total prog id found by get_next_id", 662 "nr_id_found %u(%u)\n", 663 nr_id_found, nr_iters); 664 665 /* Check bpf_map_get_next_id() */ 666 nr_id_found = 0; 667 next_id = 0; 668 while (!bpf_map_get_next_id(next_id, &next_id)) { 669 struct bpf_map_info map_info = {}; 670 int map_fd; 671 672 info_len = sizeof(map_info); 673 674 map_fd = bpf_map_get_fd_by_id(next_id); 675 if (map_fd < 0 && errno == ENOENT) 676 /* The bpf_map is in the dead row */ 677 continue; 678 if (CHECK(map_fd < 0, "get-map-fd(next_id)", 679 "map_fd %d next_id %u errno %d\n", 680 map_fd, next_id, errno)) 681 break; 682 683 for (i = 0; i < nr_iters; i++) 684 if (map_infos[i].id == next_id) 685 break; 686 687 if (i == nr_iters) 688 continue; 689 690 nr_id_found++; 691 692 err = bpf_map_lookup_elem(map_fd, &array_key, &array_value); 693 assert(!err); 694 695 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &info_len); 696 CHECK(err || info_len != sizeof(struct bpf_map_info) || 697 memcmp(&map_info, &map_infos[i], info_len) || 698 array_value != array_magic_value, 699 "check get-map-info(next_id->fd)", 700 "err %d errno %d info_len %u(%Zu) memcmp %d array_value %llu(%llu)\n", 701 err, errno, info_len, sizeof(struct bpf_map_info), 702 memcmp(&map_info, &map_infos[i], info_len), 703 array_value, array_magic_value); 704 705 close(map_fd); 706 } 707 CHECK(nr_id_found != nr_iters, 708 "check total map id found by get_next_id", 709 "nr_id_found %u(%u)\n", 710 nr_id_found, nr_iters); 711 712 done: 713 for (i = 0; i < nr_iters; i++) 714 bpf_object__close(objs[i]); 715 } 716 717 static void test_pkt_md_access(void) 718 { 719 const char *file = "./test_pkt_md_access.o"; 720 struct bpf_object *obj; 721 __u32 duration, retval; 722 int err, prog_fd; 723 724 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); 725 if (err) { 726 error_cnt++; 727 return; 728 } 729 730 err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4), 731 NULL, NULL, &retval, &duration); 732 CHECK(err || retval, "", 733 "err %d errno %d retval %d duration %d\n", 734 err, errno, retval, duration); 735 736 bpf_object__close(obj); 737 } 738 739 static void test_obj_name(void) 740 { 741 struct { 742 const char *name; 743 int success; 744 int expected_errno; 745 } tests[] = { 746 { "", 1, 0 }, 747 { "_123456789ABCDE", 1, 0 }, 748 { "_123456789ABCDEF", 0, EINVAL }, 749 { "_123456789ABCD\n", 0, EINVAL }, 750 }; 751 struct bpf_insn prog[] = { 752 BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0), 753 BPF_EXIT_INSN(), 754 }; 755 __u32 duration = 0; 756 int i; 757 758 for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { 759 size_t name_len = strlen(tests[i].name) + 1; 760 union bpf_attr attr; 761 size_t ncopy; 762 int fd; 763 764 /* test different attr.prog_name during BPF_PROG_LOAD */ 765 ncopy = name_len < sizeof(attr.prog_name) ? 766 name_len : sizeof(attr.prog_name); 767 bzero(&attr, sizeof(attr)); 768 attr.prog_type = BPF_PROG_TYPE_SCHED_CLS; 769 attr.insn_cnt = 2; 770 attr.insns = ptr_to_u64(prog); 771 attr.license = ptr_to_u64(""); 772 memcpy(attr.prog_name, tests[i].name, ncopy); 773 774 fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr)); 775 CHECK((tests[i].success && fd < 0) || 776 (!tests[i].success && fd != -1) || 777 (!tests[i].success && errno != tests[i].expected_errno), 778 "check-bpf-prog-name", 779 "fd %d(%d) errno %d(%d)\n", 780 fd, tests[i].success, errno, tests[i].expected_errno); 781 782 if (fd != -1) 783 close(fd); 784 785 /* test different attr.map_name during BPF_MAP_CREATE */ 786 ncopy = name_len < sizeof(attr.map_name) ? 787 name_len : sizeof(attr.map_name); 788 bzero(&attr, sizeof(attr)); 789 attr.map_type = BPF_MAP_TYPE_ARRAY; 790 attr.key_size = 4; 791 attr.value_size = 4; 792 attr.max_entries = 1; 793 attr.map_flags = 0; 794 memcpy(attr.map_name, tests[i].name, ncopy); 795 fd = syscall(__NR_bpf, BPF_MAP_CREATE, &attr, sizeof(attr)); 796 CHECK((tests[i].success && fd < 0) || 797 (!tests[i].success && fd != -1) || 798 (!tests[i].success && errno != tests[i].expected_errno), 799 "check-bpf-map-name", 800 "fd %d(%d) errno %d(%d)\n", 801 fd, tests[i].success, errno, tests[i].expected_errno); 802 803 if (fd != -1) 804 close(fd); 805 } 806 } 807 808 static void test_tp_attach_query(void) 809 { 810 const int num_progs = 3; 811 int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs]; 812 __u32 duration = 0, info_len, saved_prog_ids[num_progs]; 813 const char *file = "./test_tracepoint.o"; 814 struct perf_event_query_bpf *query; 815 struct perf_event_attr attr = {}; 816 struct bpf_object *obj[num_progs]; 817 struct bpf_prog_info prog_info; 818 char buf[256]; 819 820 snprintf(buf, sizeof(buf), 821 "/sys/kernel/debug/tracing/events/sched/sched_switch/id"); 822 efd = open(buf, O_RDONLY, 0); 823 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno)) 824 return; 825 bytes = read(efd, buf, sizeof(buf)); 826 close(efd); 827 if (CHECK(bytes <= 0 || bytes >= sizeof(buf), 828 "read", "bytes %d errno %d\n", bytes, errno)) 829 return; 830 831 attr.config = strtol(buf, NULL, 0); 832 attr.type = PERF_TYPE_TRACEPOINT; 833 attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN; 834 attr.sample_period = 1; 835 attr.wakeup_events = 1; 836 837 query = malloc(sizeof(*query) + sizeof(__u32) * num_progs); 838 for (i = 0; i < num_progs; i++) { 839 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj[i], 840 &prog_fd[i]); 841 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) 842 goto cleanup1; 843 844 bzero(&prog_info, sizeof(prog_info)); 845 prog_info.jited_prog_len = 0; 846 prog_info.xlated_prog_len = 0; 847 prog_info.nr_map_ids = 0; 848 info_len = sizeof(prog_info); 849 err = bpf_obj_get_info_by_fd(prog_fd[i], &prog_info, &info_len); 850 if (CHECK(err, "bpf_obj_get_info_by_fd", "err %d errno %d\n", 851 err, errno)) 852 goto cleanup1; 853 saved_prog_ids[i] = prog_info.id; 854 855 pmu_fd[i] = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 856 0 /* cpu 0 */, -1 /* group id */, 857 0 /* flags */); 858 if (CHECK(pmu_fd[i] < 0, "perf_event_open", "err %d errno %d\n", 859 pmu_fd[i], errno)) 860 goto cleanup2; 861 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_ENABLE, 0); 862 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", 863 err, errno)) 864 goto cleanup3; 865 866 if (i == 0) { 867 /* check NULL prog array query */ 868 query->ids_len = num_progs; 869 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query); 870 if (CHECK(err || query->prog_cnt != 0, 871 "perf_event_ioc_query_bpf", 872 "err %d errno %d query->prog_cnt %u\n", 873 err, errno, query->prog_cnt)) 874 goto cleanup3; 875 } 876 877 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_SET_BPF, prog_fd[i]); 878 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", 879 err, errno)) 880 goto cleanup3; 881 882 if (i == 1) { 883 /* try to get # of programs only */ 884 query->ids_len = 0; 885 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query); 886 if (CHECK(err || query->prog_cnt != 2, 887 "perf_event_ioc_query_bpf", 888 "err %d errno %d query->prog_cnt %u\n", 889 err, errno, query->prog_cnt)) 890 goto cleanup3; 891 892 /* try a few negative tests */ 893 /* invalid query pointer */ 894 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, 895 (struct perf_event_query_bpf *)0x1); 896 if (CHECK(!err || errno != EFAULT, 897 "perf_event_ioc_query_bpf", 898 "err %d errno %d\n", err, errno)) 899 goto cleanup3; 900 901 /* no enough space */ 902 query->ids_len = 1; 903 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query); 904 if (CHECK(!err || errno != ENOSPC || query->prog_cnt != 2, 905 "perf_event_ioc_query_bpf", 906 "err %d errno %d query->prog_cnt %u\n", 907 err, errno, query->prog_cnt)) 908 goto cleanup3; 909 } 910 911 query->ids_len = num_progs; 912 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query); 913 if (CHECK(err || query->prog_cnt != (i + 1), 914 "perf_event_ioc_query_bpf", 915 "err %d errno %d query->prog_cnt %u\n", 916 err, errno, query->prog_cnt)) 917 goto cleanup3; 918 for (j = 0; j < i + 1; j++) 919 if (CHECK(saved_prog_ids[j] != query->ids[j], 920 "perf_event_ioc_query_bpf", 921 "#%d saved_prog_id %x query prog_id %x\n", 922 j, saved_prog_ids[j], query->ids[j])) 923 goto cleanup3; 924 } 925 926 i = num_progs - 1; 927 for (; i >= 0; i--) { 928 cleanup3: 929 ioctl(pmu_fd[i], PERF_EVENT_IOC_DISABLE); 930 cleanup2: 931 close(pmu_fd[i]); 932 cleanup1: 933 bpf_object__close(obj[i]); 934 } 935 free(query); 936 } 937 938 static int compare_map_keys(int map1_fd, int map2_fd) 939 { 940 __u32 key, next_key; 941 char val_buf[PERF_MAX_STACK_DEPTH * 942 sizeof(struct bpf_stack_build_id)]; 943 int err; 944 945 err = bpf_map_get_next_key(map1_fd, NULL, &key); 946 if (err) 947 return err; 948 err = bpf_map_lookup_elem(map2_fd, &key, val_buf); 949 if (err) 950 return err; 951 952 while (bpf_map_get_next_key(map1_fd, &key, &next_key) == 0) { 953 err = bpf_map_lookup_elem(map2_fd, &next_key, val_buf); 954 if (err) 955 return err; 956 957 key = next_key; 958 } 959 if (errno != ENOENT) 960 return -1; 961 962 return 0; 963 } 964 965 static int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len) 966 { 967 __u32 key, next_key, *cur_key_p, *next_key_p; 968 char *val_buf1, *val_buf2; 969 int i, err = 0; 970 971 val_buf1 = malloc(stack_trace_len); 972 val_buf2 = malloc(stack_trace_len); 973 cur_key_p = NULL; 974 next_key_p = &key; 975 while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) { 976 err = bpf_map_lookup_elem(smap_fd, next_key_p, val_buf1); 977 if (err) 978 goto out; 979 err = bpf_map_lookup_elem(amap_fd, next_key_p, val_buf2); 980 if (err) 981 goto out; 982 for (i = 0; i < stack_trace_len; i++) { 983 if (val_buf1[i] != val_buf2[i]) { 984 err = -1; 985 goto out; 986 } 987 } 988 key = *next_key_p; 989 cur_key_p = &key; 990 next_key_p = &next_key; 991 } 992 if (errno != ENOENT) 993 err = -1; 994 995 out: 996 free(val_buf1); 997 free(val_buf2); 998 return err; 999 } 1000 1001 static void test_stacktrace_map() 1002 { 1003 int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd; 1004 const char *file = "./test_stacktrace_map.o"; 1005 int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len; 1006 struct perf_event_attr attr = {}; 1007 __u32 key, val, duration = 0; 1008 struct bpf_object *obj; 1009 char buf[256]; 1010 1011 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); 1012 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) 1013 return; 1014 1015 /* Get the ID for the sched/sched_switch tracepoint */ 1016 snprintf(buf, sizeof(buf), 1017 "/sys/kernel/debug/tracing/events/sched/sched_switch/id"); 1018 efd = open(buf, O_RDONLY, 0); 1019 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno)) 1020 goto close_prog; 1021 1022 bytes = read(efd, buf, sizeof(buf)); 1023 close(efd); 1024 if (bytes <= 0 || bytes >= sizeof(buf)) 1025 goto close_prog; 1026 1027 /* Open the perf event and attach bpf progrram */ 1028 attr.config = strtol(buf, NULL, 0); 1029 attr.type = PERF_TYPE_TRACEPOINT; 1030 attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN; 1031 attr.sample_period = 1; 1032 attr.wakeup_events = 1; 1033 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 1034 0 /* cpu 0 */, -1 /* group id */, 1035 0 /* flags */); 1036 if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", 1037 pmu_fd, errno)) 1038 goto close_prog; 1039 1040 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); 1041 if (err) 1042 goto disable_pmu; 1043 1044 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd); 1045 if (err) 1046 goto disable_pmu; 1047 1048 /* find map fds */ 1049 control_map_fd = bpf_find_map(__func__, obj, "control_map"); 1050 if (control_map_fd < 0) 1051 goto disable_pmu; 1052 1053 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap"); 1054 if (stackid_hmap_fd < 0) 1055 goto disable_pmu; 1056 1057 stackmap_fd = bpf_find_map(__func__, obj, "stackmap"); 1058 if (stackmap_fd < 0) 1059 goto disable_pmu; 1060 1061 stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap"); 1062 if (stack_amap_fd < 0) 1063 goto disable_pmu; 1064 1065 /* give some time for bpf program run */ 1066 sleep(1); 1067 1068 /* disable stack trace collection */ 1069 key = 0; 1070 val = 1; 1071 bpf_map_update_elem(control_map_fd, &key, &val, 0); 1072 1073 /* for every element in stackid_hmap, we can find a corresponding one 1074 * in stackmap, and vise versa. 1075 */ 1076 err = compare_map_keys(stackid_hmap_fd, stackmap_fd); 1077 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap", 1078 "err %d errno %d\n", err, errno)) 1079 goto disable_pmu_noerr; 1080 1081 err = compare_map_keys(stackmap_fd, stackid_hmap_fd); 1082 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap", 1083 "err %d errno %d\n", err, errno)) 1084 goto disable_pmu_noerr; 1085 1086 stack_trace_len = PERF_MAX_STACK_DEPTH * sizeof(__u64); 1087 err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len); 1088 if (CHECK(err, "compare_stack_ips stackmap vs. stack_amap", 1089 "err %d errno %d\n", err, errno)) 1090 goto disable_pmu_noerr; 1091 1092 goto disable_pmu_noerr; 1093 disable_pmu: 1094 error_cnt++; 1095 disable_pmu_noerr: 1096 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); 1097 close(pmu_fd); 1098 close_prog: 1099 bpf_object__close(obj); 1100 } 1101 1102 static void test_stacktrace_map_raw_tp() 1103 { 1104 int control_map_fd, stackid_hmap_fd, stackmap_fd; 1105 const char *file = "./test_stacktrace_map.o"; 1106 int efd, err, prog_fd; 1107 __u32 key, val, duration = 0; 1108 struct bpf_object *obj; 1109 1110 err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); 1111 if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno)) 1112 return; 1113 1114 efd = bpf_raw_tracepoint_open("sched_switch", prog_fd); 1115 if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno)) 1116 goto close_prog; 1117 1118 /* find map fds */ 1119 control_map_fd = bpf_find_map(__func__, obj, "control_map"); 1120 if (control_map_fd < 0) 1121 goto close_prog; 1122 1123 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap"); 1124 if (stackid_hmap_fd < 0) 1125 goto close_prog; 1126 1127 stackmap_fd = bpf_find_map(__func__, obj, "stackmap"); 1128 if (stackmap_fd < 0) 1129 goto close_prog; 1130 1131 /* give some time for bpf program run */ 1132 sleep(1); 1133 1134 /* disable stack trace collection */ 1135 key = 0; 1136 val = 1; 1137 bpf_map_update_elem(control_map_fd, &key, &val, 0); 1138 1139 /* for every element in stackid_hmap, we can find a corresponding one 1140 * in stackmap, and vise versa. 1141 */ 1142 err = compare_map_keys(stackid_hmap_fd, stackmap_fd); 1143 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap", 1144 "err %d errno %d\n", err, errno)) 1145 goto close_prog; 1146 1147 err = compare_map_keys(stackmap_fd, stackid_hmap_fd); 1148 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap", 1149 "err %d errno %d\n", err, errno)) 1150 goto close_prog; 1151 1152 goto close_prog_noerr; 1153 close_prog: 1154 error_cnt++; 1155 close_prog_noerr: 1156 bpf_object__close(obj); 1157 } 1158 1159 static int extract_build_id(char *build_id, size_t size) 1160 { 1161 FILE *fp; 1162 char *line = NULL; 1163 size_t len = 0; 1164 1165 fp = popen("readelf -n ./urandom_read | grep 'Build ID'", "r"); 1166 if (fp == NULL) 1167 return -1; 1168 1169 if (getline(&line, &len, fp) == -1) 1170 goto err; 1171 fclose(fp); 1172 1173 if (len > size) 1174 len = size; 1175 memcpy(build_id, line, len); 1176 build_id[len] = '\0'; 1177 return 0; 1178 err: 1179 fclose(fp); 1180 return -1; 1181 } 1182 1183 static void test_stacktrace_build_id(void) 1184 { 1185 int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd; 1186 const char *file = "./test_stacktrace_build_id.o"; 1187 int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len; 1188 struct perf_event_attr attr = {}; 1189 __u32 key, previous_key, val, duration = 0; 1190 struct bpf_object *obj; 1191 char buf[256]; 1192 int i, j; 1193 struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH]; 1194 int build_id_matches = 0; 1195 int retry = 1; 1196 1197 retry: 1198 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); 1199 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) 1200 goto out; 1201 1202 /* Get the ID for the sched/sched_switch tracepoint */ 1203 snprintf(buf, sizeof(buf), 1204 "/sys/kernel/debug/tracing/events/random/urandom_read/id"); 1205 efd = open(buf, O_RDONLY, 0); 1206 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno)) 1207 goto close_prog; 1208 1209 bytes = read(efd, buf, sizeof(buf)); 1210 close(efd); 1211 if (CHECK(bytes <= 0 || bytes >= sizeof(buf), 1212 "read", "bytes %d errno %d\n", bytes, errno)) 1213 goto close_prog; 1214 1215 /* Open the perf event and attach bpf progrram */ 1216 attr.config = strtol(buf, NULL, 0); 1217 attr.type = PERF_TYPE_TRACEPOINT; 1218 attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN; 1219 attr.sample_period = 1; 1220 attr.wakeup_events = 1; 1221 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 1222 0 /* cpu 0 */, -1 /* group id */, 1223 0 /* flags */); 1224 if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", 1225 pmu_fd, errno)) 1226 goto close_prog; 1227 1228 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); 1229 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", 1230 err, errno)) 1231 goto close_pmu; 1232 1233 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd); 1234 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", 1235 err, errno)) 1236 goto disable_pmu; 1237 1238 /* find map fds */ 1239 control_map_fd = bpf_find_map(__func__, obj, "control_map"); 1240 if (CHECK(control_map_fd < 0, "bpf_find_map control_map", 1241 "err %d errno %d\n", err, errno)) 1242 goto disable_pmu; 1243 1244 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap"); 1245 if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap", 1246 "err %d errno %d\n", err, errno)) 1247 goto disable_pmu; 1248 1249 stackmap_fd = bpf_find_map(__func__, obj, "stackmap"); 1250 if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n", 1251 err, errno)) 1252 goto disable_pmu; 1253 1254 stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap"); 1255 if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap", 1256 "err %d errno %d\n", err, errno)) 1257 goto disable_pmu; 1258 1259 assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null") 1260 == 0); 1261 assert(system("./urandom_read") == 0); 1262 /* disable stack trace collection */ 1263 key = 0; 1264 val = 1; 1265 bpf_map_update_elem(control_map_fd, &key, &val, 0); 1266 1267 /* for every element in stackid_hmap, we can find a corresponding one 1268 * in stackmap, and vise versa. 1269 */ 1270 err = compare_map_keys(stackid_hmap_fd, stackmap_fd); 1271 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap", 1272 "err %d errno %d\n", err, errno)) 1273 goto disable_pmu; 1274 1275 err = compare_map_keys(stackmap_fd, stackid_hmap_fd); 1276 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap", 1277 "err %d errno %d\n", err, errno)) 1278 goto disable_pmu; 1279 1280 err = extract_build_id(buf, 256); 1281 1282 if (CHECK(err, "get build_id with readelf", 1283 "err %d errno %d\n", err, errno)) 1284 goto disable_pmu; 1285 1286 err = bpf_map_get_next_key(stackmap_fd, NULL, &key); 1287 if (CHECK(err, "get_next_key from stackmap", 1288 "err %d, errno %d\n", err, errno)) 1289 goto disable_pmu; 1290 1291 do { 1292 char build_id[64]; 1293 1294 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs); 1295 if (CHECK(err, "lookup_elem from stackmap", 1296 "err %d, errno %d\n", err, errno)) 1297 goto disable_pmu; 1298 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i) 1299 if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID && 1300 id_offs[i].offset != 0) { 1301 for (j = 0; j < 20; ++j) 1302 sprintf(build_id + 2 * j, "%02x", 1303 id_offs[i].build_id[j] & 0xff); 1304 if (strstr(buf, build_id) != NULL) 1305 build_id_matches = 1; 1306 } 1307 previous_key = key; 1308 } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0); 1309 1310 /* stack_map_get_build_id_offset() is racy and sometimes can return 1311 * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID; 1312 * try it one more time. 1313 */ 1314 if (build_id_matches < 1 && retry--) { 1315 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); 1316 close(pmu_fd); 1317 bpf_object__close(obj); 1318 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n", 1319 __func__); 1320 goto retry; 1321 } 1322 1323 if (CHECK(build_id_matches < 1, "build id match", 1324 "Didn't find expected build ID from the map\n")) 1325 goto disable_pmu; 1326 1327 stack_trace_len = PERF_MAX_STACK_DEPTH 1328 * sizeof(struct bpf_stack_build_id); 1329 err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len); 1330 CHECK(err, "compare_stack_ips stackmap vs. stack_amap", 1331 "err %d errno %d\n", err, errno); 1332 1333 disable_pmu: 1334 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); 1335 1336 close_pmu: 1337 close(pmu_fd); 1338 1339 close_prog: 1340 bpf_object__close(obj); 1341 1342 out: 1343 return; 1344 } 1345 1346 static void test_stacktrace_build_id_nmi(void) 1347 { 1348 int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd; 1349 const char *file = "./test_stacktrace_build_id.o"; 1350 int err, pmu_fd, prog_fd; 1351 struct perf_event_attr attr = { 1352 .sample_freq = 5000, 1353 .freq = 1, 1354 .type = PERF_TYPE_HARDWARE, 1355 .config = PERF_COUNT_HW_CPU_CYCLES, 1356 }; 1357 __u32 key, previous_key, val, duration = 0; 1358 struct bpf_object *obj; 1359 char buf[256]; 1360 int i, j; 1361 struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH]; 1362 int build_id_matches = 0; 1363 int retry = 1; 1364 1365 retry: 1366 err = bpf_prog_load(file, BPF_PROG_TYPE_PERF_EVENT, &obj, &prog_fd); 1367 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) 1368 return; 1369 1370 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 1371 0 /* cpu 0 */, -1 /* group id */, 1372 0 /* flags */); 1373 if (CHECK(pmu_fd < 0, "perf_event_open", 1374 "err %d errno %d. Does the test host support PERF_COUNT_HW_CPU_CYCLES?\n", 1375 pmu_fd, errno)) 1376 goto close_prog; 1377 1378 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); 1379 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", 1380 err, errno)) 1381 goto close_pmu; 1382 1383 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd); 1384 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", 1385 err, errno)) 1386 goto disable_pmu; 1387 1388 /* find map fds */ 1389 control_map_fd = bpf_find_map(__func__, obj, "control_map"); 1390 if (CHECK(control_map_fd < 0, "bpf_find_map control_map", 1391 "err %d errno %d\n", err, errno)) 1392 goto disable_pmu; 1393 1394 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap"); 1395 if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap", 1396 "err %d errno %d\n", err, errno)) 1397 goto disable_pmu; 1398 1399 stackmap_fd = bpf_find_map(__func__, obj, "stackmap"); 1400 if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n", 1401 err, errno)) 1402 goto disable_pmu; 1403 1404 stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap"); 1405 if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap", 1406 "err %d errno %d\n", err, errno)) 1407 goto disable_pmu; 1408 1409 assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null") 1410 == 0); 1411 assert(system("taskset 0x1 ./urandom_read 100000") == 0); 1412 /* disable stack trace collection */ 1413 key = 0; 1414 val = 1; 1415 bpf_map_update_elem(control_map_fd, &key, &val, 0); 1416 1417 /* for every element in stackid_hmap, we can find a corresponding one 1418 * in stackmap, and vise versa. 1419 */ 1420 err = compare_map_keys(stackid_hmap_fd, stackmap_fd); 1421 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap", 1422 "err %d errno %d\n", err, errno)) 1423 goto disable_pmu; 1424 1425 err = compare_map_keys(stackmap_fd, stackid_hmap_fd); 1426 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap", 1427 "err %d errno %d\n", err, errno)) 1428 goto disable_pmu; 1429 1430 err = extract_build_id(buf, 256); 1431 1432 if (CHECK(err, "get build_id with readelf", 1433 "err %d errno %d\n", err, errno)) 1434 goto disable_pmu; 1435 1436 err = bpf_map_get_next_key(stackmap_fd, NULL, &key); 1437 if (CHECK(err, "get_next_key from stackmap", 1438 "err %d, errno %d\n", err, errno)) 1439 goto disable_pmu; 1440 1441 do { 1442 char build_id[64]; 1443 1444 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs); 1445 if (CHECK(err, "lookup_elem from stackmap", 1446 "err %d, errno %d\n", err, errno)) 1447 goto disable_pmu; 1448 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i) 1449 if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID && 1450 id_offs[i].offset != 0) { 1451 for (j = 0; j < 20; ++j) 1452 sprintf(build_id + 2 * j, "%02x", 1453 id_offs[i].build_id[j] & 0xff); 1454 if (strstr(buf, build_id) != NULL) 1455 build_id_matches = 1; 1456 } 1457 previous_key = key; 1458 } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0); 1459 1460 /* stack_map_get_build_id_offset() is racy and sometimes can return 1461 * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID; 1462 * try it one more time. 1463 */ 1464 if (build_id_matches < 1 && retry--) { 1465 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); 1466 close(pmu_fd); 1467 bpf_object__close(obj); 1468 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n", 1469 __func__); 1470 goto retry; 1471 } 1472 1473 if (CHECK(build_id_matches < 1, "build id match", 1474 "Didn't find expected build ID from the map\n")) 1475 goto disable_pmu; 1476 1477 /* 1478 * We intentionally skip compare_stack_ips(). This is because we 1479 * only support one in_nmi() ips-to-build_id translation per cpu 1480 * at any time, thus stack_amap here will always fallback to 1481 * BPF_STACK_BUILD_ID_IP; 1482 */ 1483 1484 disable_pmu: 1485 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); 1486 1487 close_pmu: 1488 close(pmu_fd); 1489 1490 close_prog: 1491 bpf_object__close(obj); 1492 } 1493 1494 #define MAX_CNT_RAWTP 10ull 1495 #define MAX_STACK_RAWTP 100 1496 struct get_stack_trace_t { 1497 int pid; 1498 int kern_stack_size; 1499 int user_stack_size; 1500 int user_stack_buildid_size; 1501 __u64 kern_stack[MAX_STACK_RAWTP]; 1502 __u64 user_stack[MAX_STACK_RAWTP]; 1503 struct bpf_stack_build_id user_stack_buildid[MAX_STACK_RAWTP]; 1504 }; 1505 1506 static int get_stack_print_output(void *data, int size) 1507 { 1508 bool good_kern_stack = false, good_user_stack = false; 1509 const char *nonjit_func = "___bpf_prog_run"; 1510 struct get_stack_trace_t *e = data; 1511 int i, num_stack; 1512 static __u64 cnt; 1513 struct ksym *ks; 1514 1515 cnt++; 1516 1517 if (size < sizeof(struct get_stack_trace_t)) { 1518 __u64 *raw_data = data; 1519 bool found = false; 1520 1521 num_stack = size / sizeof(__u64); 1522 /* If jit is enabled, we do not have a good way to 1523 * verify the sanity of the kernel stack. So we 1524 * just assume it is good if the stack is not empty. 1525 * This could be improved in the future. 1526 */ 1527 if (jit_enabled) { 1528 found = num_stack > 0; 1529 } else { 1530 for (i = 0; i < num_stack; i++) { 1531 ks = ksym_search(raw_data[i]); 1532 if (strcmp(ks->name, nonjit_func) == 0) { 1533 found = true; 1534 break; 1535 } 1536 } 1537 } 1538 if (found) { 1539 good_kern_stack = true; 1540 good_user_stack = true; 1541 } 1542 } else { 1543 num_stack = e->kern_stack_size / sizeof(__u64); 1544 if (jit_enabled) { 1545 good_kern_stack = num_stack > 0; 1546 } else { 1547 for (i = 0; i < num_stack; i++) { 1548 ks = ksym_search(e->kern_stack[i]); 1549 if (strcmp(ks->name, nonjit_func) == 0) { 1550 good_kern_stack = true; 1551 break; 1552 } 1553 } 1554 } 1555 if (e->user_stack_size > 0 && e->user_stack_buildid_size > 0) 1556 good_user_stack = true; 1557 } 1558 if (!good_kern_stack || !good_user_stack) 1559 return LIBBPF_PERF_EVENT_ERROR; 1560 1561 if (cnt == MAX_CNT_RAWTP) 1562 return LIBBPF_PERF_EVENT_DONE; 1563 1564 return LIBBPF_PERF_EVENT_CONT; 1565 } 1566 1567 static void test_get_stack_raw_tp(void) 1568 { 1569 const char *file = "./test_get_stack_rawtp.o"; 1570 int i, efd, err, prog_fd, pmu_fd, perfmap_fd; 1571 struct perf_event_attr attr = {}; 1572 struct timespec tv = {0, 10}; 1573 __u32 key = 0, duration = 0; 1574 struct bpf_object *obj; 1575 1576 err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); 1577 if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno)) 1578 return; 1579 1580 efd = bpf_raw_tracepoint_open("sys_enter", prog_fd); 1581 if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno)) 1582 goto close_prog; 1583 1584 perfmap_fd = bpf_find_map(__func__, obj, "perfmap"); 1585 if (CHECK(perfmap_fd < 0, "bpf_find_map", "err %d errno %d\n", 1586 perfmap_fd, errno)) 1587 goto close_prog; 1588 1589 err = load_kallsyms(); 1590 if (CHECK(err < 0, "load_kallsyms", "err %d errno %d\n", err, errno)) 1591 goto close_prog; 1592 1593 attr.sample_type = PERF_SAMPLE_RAW; 1594 attr.type = PERF_TYPE_SOFTWARE; 1595 attr.config = PERF_COUNT_SW_BPF_OUTPUT; 1596 pmu_fd = syscall(__NR_perf_event_open, &attr, getpid()/*pid*/, -1/*cpu*/, 1597 -1/*group_fd*/, 0); 1598 if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", pmu_fd, 1599 errno)) 1600 goto close_prog; 1601 1602 err = bpf_map_update_elem(perfmap_fd, &key, &pmu_fd, BPF_ANY); 1603 if (CHECK(err < 0, "bpf_map_update_elem", "err %d errno %d\n", err, 1604 errno)) 1605 goto close_prog; 1606 1607 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); 1608 if (CHECK(err < 0, "ioctl PERF_EVENT_IOC_ENABLE", "err %d errno %d\n", 1609 err, errno)) 1610 goto close_prog; 1611 1612 err = perf_event_mmap(pmu_fd); 1613 if (CHECK(err < 0, "perf_event_mmap", "err %d errno %d\n", err, errno)) 1614 goto close_prog; 1615 1616 /* trigger some syscall action */ 1617 for (i = 0; i < MAX_CNT_RAWTP; i++) 1618 nanosleep(&tv, NULL); 1619 1620 err = perf_event_poller(pmu_fd, get_stack_print_output); 1621 if (CHECK(err < 0, "perf_event_poller", "err %d errno %d\n", err, errno)) 1622 goto close_prog; 1623 1624 goto close_prog_noerr; 1625 close_prog: 1626 error_cnt++; 1627 close_prog_noerr: 1628 bpf_object__close(obj); 1629 } 1630 1631 static void test_task_fd_query_rawtp(void) 1632 { 1633 const char *file = "./test_get_stack_rawtp.o"; 1634 __u64 probe_offset, probe_addr; 1635 __u32 len, prog_id, fd_type; 1636 struct bpf_object *obj; 1637 int efd, err, prog_fd; 1638 __u32 duration = 0; 1639 char buf[256]; 1640 1641 err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); 1642 if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno)) 1643 return; 1644 1645 efd = bpf_raw_tracepoint_open("sys_enter", prog_fd); 1646 if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno)) 1647 goto close_prog; 1648 1649 /* query (getpid(), efd) */ 1650 len = sizeof(buf); 1651 err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id, 1652 &fd_type, &probe_offset, &probe_addr); 1653 if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err, 1654 errno)) 1655 goto close_prog; 1656 1657 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT && 1658 strcmp(buf, "sys_enter") == 0; 1659 if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n", 1660 fd_type, buf)) 1661 goto close_prog; 1662 1663 /* test zero len */ 1664 len = 0; 1665 err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id, 1666 &fd_type, &probe_offset, &probe_addr); 1667 if (CHECK(err < 0, "bpf_task_fd_query (len = 0)", "err %d errno %d\n", 1668 err, errno)) 1669 goto close_prog; 1670 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT && 1671 len == strlen("sys_enter"); 1672 if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len)) 1673 goto close_prog; 1674 1675 /* test empty buffer */ 1676 len = sizeof(buf); 1677 err = bpf_task_fd_query(getpid(), efd, 0, 0, &len, &prog_id, 1678 &fd_type, &probe_offset, &probe_addr); 1679 if (CHECK(err < 0, "bpf_task_fd_query (buf = 0)", "err %d errno %d\n", 1680 err, errno)) 1681 goto close_prog; 1682 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT && 1683 len == strlen("sys_enter"); 1684 if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len)) 1685 goto close_prog; 1686 1687 /* test smaller buffer */ 1688 len = 3; 1689 err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id, 1690 &fd_type, &probe_offset, &probe_addr); 1691 if (CHECK(err >= 0 || errno != ENOSPC, "bpf_task_fd_query (len = 3)", 1692 "err %d errno %d\n", err, errno)) 1693 goto close_prog; 1694 err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT && 1695 len == strlen("sys_enter") && 1696 strcmp(buf, "sy") == 0; 1697 if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len)) 1698 goto close_prog; 1699 1700 goto close_prog_noerr; 1701 close_prog: 1702 error_cnt++; 1703 close_prog_noerr: 1704 bpf_object__close(obj); 1705 } 1706 1707 static void test_task_fd_query_tp_core(const char *probe_name, 1708 const char *tp_name) 1709 { 1710 const char *file = "./test_tracepoint.o"; 1711 int err, bytes, efd, prog_fd, pmu_fd; 1712 struct perf_event_attr attr = {}; 1713 __u64 probe_offset, probe_addr; 1714 __u32 len, prog_id, fd_type; 1715 struct bpf_object *obj; 1716 __u32 duration = 0; 1717 char buf[256]; 1718 1719 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); 1720 if (CHECK(err, "bpf_prog_load", "err %d errno %d\n", err, errno)) 1721 goto close_prog; 1722 1723 snprintf(buf, sizeof(buf), 1724 "/sys/kernel/debug/tracing/events/%s/id", probe_name); 1725 efd = open(buf, O_RDONLY, 0); 1726 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno)) 1727 goto close_prog; 1728 bytes = read(efd, buf, sizeof(buf)); 1729 close(efd); 1730 if (CHECK(bytes <= 0 || bytes >= sizeof(buf), "read", 1731 "bytes %d errno %d\n", bytes, errno)) 1732 goto close_prog; 1733 1734 attr.config = strtol(buf, NULL, 0); 1735 attr.type = PERF_TYPE_TRACEPOINT; 1736 attr.sample_type = PERF_SAMPLE_RAW; 1737 attr.sample_period = 1; 1738 attr.wakeup_events = 1; 1739 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 1740 0 /* cpu 0 */, -1 /* group id */, 1741 0 /* flags */); 1742 if (CHECK(err, "perf_event_open", "err %d errno %d\n", err, errno)) 1743 goto close_pmu; 1744 1745 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); 1746 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", err, 1747 errno)) 1748 goto close_pmu; 1749 1750 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd); 1751 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", err, 1752 errno)) 1753 goto close_pmu; 1754 1755 /* query (getpid(), pmu_fd) */ 1756 len = sizeof(buf); 1757 err = bpf_task_fd_query(getpid(), pmu_fd, 0, buf, &len, &prog_id, 1758 &fd_type, &probe_offset, &probe_addr); 1759 if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err, 1760 errno)) 1761 goto close_pmu; 1762 1763 err = (fd_type == BPF_FD_TYPE_TRACEPOINT) && !strcmp(buf, tp_name); 1764 if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n", 1765 fd_type, buf)) 1766 goto close_pmu; 1767 1768 close(pmu_fd); 1769 goto close_prog_noerr; 1770 1771 close_pmu: 1772 close(pmu_fd); 1773 close_prog: 1774 error_cnt++; 1775 close_prog_noerr: 1776 bpf_object__close(obj); 1777 } 1778 1779 static void test_task_fd_query_tp(void) 1780 { 1781 test_task_fd_query_tp_core("sched/sched_switch", 1782 "sched_switch"); 1783 test_task_fd_query_tp_core("syscalls/sys_enter_read", 1784 "sys_enter_read"); 1785 } 1786 1787 static int libbpf_debug_print(enum libbpf_print_level level, 1788 const char *format, va_list args) 1789 { 1790 if (level == LIBBPF_DEBUG) 1791 return 0; 1792 1793 return vfprintf(stderr, format, args); 1794 } 1795 1796 static void test_reference_tracking() 1797 { 1798 const char *file = "./test_sk_lookup_kern.o"; 1799 struct bpf_object *obj; 1800 struct bpf_program *prog; 1801 __u32 duration = 0; 1802 int err = 0; 1803 1804 obj = bpf_object__open(file); 1805 if (IS_ERR(obj)) { 1806 error_cnt++; 1807 return; 1808 } 1809 1810 bpf_object__for_each_program(prog, obj) { 1811 const char *title; 1812 1813 /* Ignore .text sections */ 1814 title = bpf_program__title(prog, false); 1815 if (strstr(title, ".text") != NULL) 1816 continue; 1817 1818 bpf_program__set_type(prog, BPF_PROG_TYPE_SCHED_CLS); 1819 1820 /* Expect verifier failure if test name has 'fail' */ 1821 if (strstr(title, "fail") != NULL) { 1822 libbpf_set_print(NULL); 1823 err = !bpf_program__load(prog, "GPL", 0); 1824 libbpf_set_print(libbpf_debug_print); 1825 } else { 1826 err = bpf_program__load(prog, "GPL", 0); 1827 } 1828 CHECK(err, title, "\n"); 1829 } 1830 bpf_object__close(obj); 1831 } 1832 1833 enum { 1834 QUEUE, 1835 STACK, 1836 }; 1837 1838 static void test_queue_stack_map(int type) 1839 { 1840 const int MAP_SIZE = 32; 1841 __u32 vals[MAP_SIZE], duration, retval, size, val; 1842 int i, err, prog_fd, map_in_fd, map_out_fd; 1843 char file[32], buf[128]; 1844 struct bpf_object *obj; 1845 struct iphdr *iph = (void *)buf + sizeof(struct ethhdr); 1846 1847 /* Fill test values to be used */ 1848 for (i = 0; i < MAP_SIZE; i++) 1849 vals[i] = rand(); 1850 1851 if (type == QUEUE) 1852 strncpy(file, "./test_queue_map.o", sizeof(file)); 1853 else if (type == STACK) 1854 strncpy(file, "./test_stack_map.o", sizeof(file)); 1855 else 1856 return; 1857 1858 err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); 1859 if (err) { 1860 error_cnt++; 1861 return; 1862 } 1863 1864 map_in_fd = bpf_find_map(__func__, obj, "map_in"); 1865 if (map_in_fd < 0) 1866 goto out; 1867 1868 map_out_fd = bpf_find_map(__func__, obj, "map_out"); 1869 if (map_out_fd < 0) 1870 goto out; 1871 1872 /* Push 32 elements to the input map */ 1873 for (i = 0; i < MAP_SIZE; i++) { 1874 err = bpf_map_update_elem(map_in_fd, NULL, &vals[i], 0); 1875 if (err) { 1876 error_cnt++; 1877 goto out; 1878 } 1879 } 1880 1881 /* The eBPF program pushes iph.saddr in the output map, 1882 * pops the input map and saves this value in iph.daddr 1883 */ 1884 for (i = 0; i < MAP_SIZE; i++) { 1885 if (type == QUEUE) { 1886 val = vals[i]; 1887 pkt_v4.iph.saddr = vals[i] * 5; 1888 } else if (type == STACK) { 1889 val = vals[MAP_SIZE - 1 - i]; 1890 pkt_v4.iph.saddr = vals[MAP_SIZE - 1 - i] * 5; 1891 } 1892 1893 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4), 1894 buf, &size, &retval, &duration); 1895 if (err || retval || size != sizeof(pkt_v4) || 1896 iph->daddr != val) 1897 break; 1898 } 1899 1900 CHECK(err || retval || size != sizeof(pkt_v4) || iph->daddr != val, 1901 "bpf_map_pop_elem", 1902 "err %d errno %d retval %d size %d iph->daddr %u\n", 1903 err, errno, retval, size, iph->daddr); 1904 1905 /* Queue is empty, program should return TC_ACT_SHOT */ 1906 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4), 1907 buf, &size, &retval, &duration); 1908 CHECK(err || retval != 2 /* TC_ACT_SHOT */|| size != sizeof(pkt_v4), 1909 "check-queue-stack-map-empty", 1910 "err %d errno %d retval %d size %d\n", 1911 err, errno, retval, size); 1912 1913 /* Check that the program pushed elements correctly */ 1914 for (i = 0; i < MAP_SIZE; i++) { 1915 err = bpf_map_lookup_and_delete_elem(map_out_fd, NULL, &val); 1916 if (err || val != vals[i] * 5) 1917 break; 1918 } 1919 1920 CHECK(i != MAP_SIZE && (err || val != vals[i] * 5), 1921 "bpf_map_push_elem", "err %d value %u\n", err, val); 1922 1923 out: 1924 pkt_v4.iph.saddr = 0; 1925 bpf_object__close(obj); 1926 } 1927 1928 #define CHECK_FLOW_KEYS(desc, got, expected) \ 1929 CHECK(memcmp(&got, &expected, sizeof(got)) != 0, \ 1930 desc, \ 1931 "nhoff=%u/%u " \ 1932 "thoff=%u/%u " \ 1933 "addr_proto=0x%x/0x%x " \ 1934 "is_frag=%u/%u " \ 1935 "is_first_frag=%u/%u " \ 1936 "is_encap=%u/%u " \ 1937 "n_proto=0x%x/0x%x " \ 1938 "sport=%u/%u " \ 1939 "dport=%u/%u\n", \ 1940 got.nhoff, expected.nhoff, \ 1941 got.thoff, expected.thoff, \ 1942 got.addr_proto, expected.addr_proto, \ 1943 got.is_frag, expected.is_frag, \ 1944 got.is_first_frag, expected.is_first_frag, \ 1945 got.is_encap, expected.is_encap, \ 1946 got.n_proto, expected.n_proto, \ 1947 got.sport, expected.sport, \ 1948 got.dport, expected.dport) 1949 1950 static struct bpf_flow_keys pkt_v4_flow_keys = { 1951 .nhoff = 0, 1952 .thoff = sizeof(struct iphdr), 1953 .addr_proto = ETH_P_IP, 1954 .ip_proto = IPPROTO_TCP, 1955 .n_proto = bpf_htons(ETH_P_IP), 1956 }; 1957 1958 static struct bpf_flow_keys pkt_v6_flow_keys = { 1959 .nhoff = 0, 1960 .thoff = sizeof(struct ipv6hdr), 1961 .addr_proto = ETH_P_IPV6, 1962 .ip_proto = IPPROTO_TCP, 1963 .n_proto = bpf_htons(ETH_P_IPV6), 1964 }; 1965 1966 static void test_flow_dissector(void) 1967 { 1968 struct bpf_flow_keys flow_keys; 1969 struct bpf_object *obj; 1970 __u32 duration, retval; 1971 int err, prog_fd; 1972 __u32 size; 1973 1974 err = bpf_flow_load(&obj, "./bpf_flow.o", "flow_dissector", 1975 "jmp_table", &prog_fd); 1976 if (err) { 1977 error_cnt++; 1978 return; 1979 } 1980 1981 err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4), 1982 &flow_keys, &size, &retval, &duration); 1983 CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv4", 1984 "err %d errno %d retval %d duration %d size %u/%lu\n", 1985 err, errno, retval, duration, size, sizeof(flow_keys)); 1986 CHECK_FLOW_KEYS("ipv4_flow_keys", flow_keys, pkt_v4_flow_keys); 1987 1988 err = bpf_prog_test_run(prog_fd, 10, &pkt_v6, sizeof(pkt_v6), 1989 &flow_keys, &size, &retval, &duration); 1990 CHECK(size != sizeof(flow_keys) || err || retval != 1, "ipv6", 1991 "err %d errno %d retval %d duration %d size %u/%lu\n", 1992 err, errno, retval, duration, size, sizeof(flow_keys)); 1993 CHECK_FLOW_KEYS("ipv6_flow_keys", flow_keys, pkt_v6_flow_keys); 1994 1995 bpf_object__close(obj); 1996 } 1997 1998 static void *test_spin_lock(void *arg) 1999 { 2000 __u32 duration, retval; 2001 int err, prog_fd = *(u32 *) arg; 2002 2003 err = bpf_prog_test_run(prog_fd, 10000, &pkt_v4, sizeof(pkt_v4), 2004 NULL, NULL, &retval, &duration); 2005 CHECK(err || retval, "", 2006 "err %d errno %d retval %d duration %d\n", 2007 err, errno, retval, duration); 2008 pthread_exit(arg); 2009 } 2010 2011 static void test_spinlock(void) 2012 { 2013 const char *file = "./test_spin_lock.o"; 2014 pthread_t thread_id[4]; 2015 struct bpf_object *obj; 2016 int prog_fd; 2017 int err = 0, i; 2018 void *ret; 2019 2020 err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd); 2021 if (err) { 2022 printf("test_spin_lock:bpf_prog_load errno %d\n", errno); 2023 goto close_prog; 2024 } 2025 for (i = 0; i < 4; i++) 2026 assert(pthread_create(&thread_id[i], NULL, 2027 &test_spin_lock, &prog_fd) == 0); 2028 for (i = 0; i < 4; i++) 2029 assert(pthread_join(thread_id[i], &ret) == 0 && 2030 ret == (void *)&prog_fd); 2031 goto close_prog_noerr; 2032 close_prog: 2033 error_cnt++; 2034 close_prog_noerr: 2035 bpf_object__close(obj); 2036 } 2037 2038 static void *parallel_map_access(void *arg) 2039 { 2040 int err, map_fd = *(u32 *) arg; 2041 int vars[17], i, j, rnd, key = 0; 2042 2043 for (i = 0; i < 10000; i++) { 2044 err = bpf_map_lookup_elem_flags(map_fd, &key, vars, BPF_F_LOCK); 2045 if (err) { 2046 printf("lookup failed\n"); 2047 error_cnt++; 2048 goto out; 2049 } 2050 if (vars[0] != 0) { 2051 printf("lookup #%d var[0]=%d\n", i, vars[0]); 2052 error_cnt++; 2053 goto out; 2054 } 2055 rnd = vars[1]; 2056 for (j = 2; j < 17; j++) { 2057 if (vars[j] == rnd) 2058 continue; 2059 printf("lookup #%d var[1]=%d var[%d]=%d\n", 2060 i, rnd, j, vars[j]); 2061 error_cnt++; 2062 goto out; 2063 } 2064 } 2065 out: 2066 pthread_exit(arg); 2067 } 2068 2069 static void test_map_lock(void) 2070 { 2071 const char *file = "./test_map_lock.o"; 2072 int prog_fd, map_fd[2], vars[17] = {}; 2073 pthread_t thread_id[6]; 2074 struct bpf_object *obj; 2075 int err = 0, key = 0, i; 2076 void *ret; 2077 2078 err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd); 2079 if (err) { 2080 printf("test_map_lock:bpf_prog_load errno %d\n", errno); 2081 goto close_prog; 2082 } 2083 map_fd[0] = bpf_find_map(__func__, obj, "hash_map"); 2084 if (map_fd[0] < 0) 2085 goto close_prog; 2086 map_fd[1] = bpf_find_map(__func__, obj, "array_map"); 2087 if (map_fd[1] < 0) 2088 goto close_prog; 2089 2090 bpf_map_update_elem(map_fd[0], &key, vars, BPF_F_LOCK); 2091 2092 for (i = 0; i < 4; i++) 2093 assert(pthread_create(&thread_id[i], NULL, 2094 &test_spin_lock, &prog_fd) == 0); 2095 for (i = 4; i < 6; i++) 2096 assert(pthread_create(&thread_id[i], NULL, 2097 ¶llel_map_access, &map_fd[i - 4]) == 0); 2098 for (i = 0; i < 4; i++) 2099 assert(pthread_join(thread_id[i], &ret) == 0 && 2100 ret == (void *)&prog_fd); 2101 for (i = 4; i < 6; i++) 2102 assert(pthread_join(thread_id[i], &ret) == 0 && 2103 ret == (void *)&map_fd[i - 4]); 2104 goto close_prog_noerr; 2105 close_prog: 2106 error_cnt++; 2107 close_prog_noerr: 2108 bpf_object__close(obj); 2109 } 2110 2111 int main(void) 2112 { 2113 srand(time(NULL)); 2114 2115 jit_enabled = is_jit_enabled(); 2116 2117 test_pkt_access(); 2118 test_prog_run_xattr(); 2119 test_xdp(); 2120 test_xdp_adjust_tail(); 2121 test_l4lb_all(); 2122 test_xdp_noinline(); 2123 test_tcp_estats(); 2124 test_bpf_obj_id(); 2125 test_pkt_md_access(); 2126 test_obj_name(); 2127 test_tp_attach_query(); 2128 test_stacktrace_map(); 2129 test_stacktrace_build_id(); 2130 test_stacktrace_build_id_nmi(); 2131 test_stacktrace_map_raw_tp(); 2132 test_get_stack_raw_tp(); 2133 test_task_fd_query_rawtp(); 2134 test_task_fd_query_tp(); 2135 test_reference_tracking(); 2136 test_queue_stack_map(QUEUE); 2137 test_queue_stack_map(STACK); 2138 test_flow_dissector(); 2139 test_spinlock(); 2140 test_map_lock(); 2141 2142 printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt); 2143 return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS; 2144 } 2145