1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ 3 #include <linux/kernel.h> 4 #include <linux/filter.h> 5 #include "bpf.h" 6 #include "libbpf.h" 7 #include "libbpf_common.h" 8 #include "libbpf_internal.h" 9 10 static inline __u64 ptr_to_u64(const void *ptr) 11 { 12 return (__u64)(unsigned long)ptr; 13 } 14 15 int probe_fd(int fd) 16 { 17 if (fd >= 0) 18 close(fd); 19 return fd >= 0; 20 } 21 22 static int probe_kern_prog_name(int token_fd) 23 { 24 const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); 25 struct bpf_insn insns[] = { 26 BPF_MOV64_IMM(BPF_REG_0, 0), 27 BPF_EXIT_INSN(), 28 }; 29 union bpf_attr attr; 30 int ret; 31 32 memset(&attr, 0, attr_sz); 33 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 34 attr.license = ptr_to_u64("GPL"); 35 attr.insns = ptr_to_u64(insns); 36 attr.insn_cnt = (__u32)ARRAY_SIZE(insns); 37 attr.prog_token_fd = token_fd; 38 if (token_fd) 39 attr.prog_flags |= BPF_F_TOKEN_FD; 40 libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name)); 41 42 /* make sure loading with name works */ 43 ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS); 44 return probe_fd(ret); 45 } 46 47 static int probe_kern_global_data(int token_fd) 48 { 49 struct bpf_insn insns[] = { 50 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 51 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 52 BPF_MOV64_IMM(BPF_REG_0, 0), 53 BPF_EXIT_INSN(), 54 }; 55 LIBBPF_OPTS(bpf_map_create_opts, map_opts, 56 .token_fd = token_fd, 57 .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, 58 ); 59 LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, 60 .token_fd = token_fd, 61 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 62 ); 63 int ret, map, insn_cnt = ARRAY_SIZE(insns); 64 65 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, &map_opts); 66 if (map < 0) { 67 ret = -errno; 68 pr_warn("Error in %s(): %s. Couldn't create simple array map.\n", 69 __func__, errstr(ret)); 70 return ret; 71 } 72 73 insns[0].imm = map; 74 75 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); 76 close(map); 77 return probe_fd(ret); 78 } 79 80 /* 81 * A continuous fd_array binds every map in it to the program, whether an 82 * insn refers to the map or not. Kernels without the attribute reject a 83 * non-zero fd_array_cnt. 84 */ 85 static int probe_kern_fd_array_cnt(int token_fd) 86 { 87 struct bpf_insn insns[] = { 88 BPF_MOV64_IMM(BPF_REG_0, 0), 89 BPF_EXIT_INSN(), 90 }; 91 LIBBPF_OPTS(bpf_map_create_opts, map_opts, 92 .token_fd = token_fd, 93 .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, 94 ); 95 LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, 96 .token_fd = token_fd, 97 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 98 .fd_array_cnt = 1, 99 ); 100 int ret, map, insn_cnt = ARRAY_SIZE(insns); 101 102 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), sizeof(int), 1, 103 &map_opts); 104 if (map < 0) { 105 ret = -errno; 106 pr_warn("Error in %s(): %s. Couldn't create simple array map.\n", 107 __func__, errstr(ret)); 108 return ret; 109 } 110 111 prog_opts.fd_array = ↦ 112 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); 113 close(map); 114 return probe_fd(ret); 115 } 116 117 /* 118 * The kernel checks an exclusive map bound through fd_array_cnt against the 119 * digest of the final program. Earlier kernels compared it as the map was 120 * added, against a digest they had not computed yet, and refused the load for 121 * the very program the map is exclusive to; kernels without exclusive maps 122 * refuse the map. 123 */ 124 static int probe_kern_excl_map_fd_array_cnt(int token_fd) 125 { 126 struct bpf_insn insns[] = { 127 BPF_MOV64_IMM(BPF_REG_0, 0), 128 BPF_EXIT_INSN(), 129 }; 130 __u8 hash[SHA256_DIGEST_LENGTH]; 131 LIBBPF_OPTS(bpf_map_create_opts, map_opts, 132 .token_fd = token_fd, 133 .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, 134 .excl_prog_hash = hash, 135 .excl_prog_hash_size = sizeof(hash), 136 ); 137 LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, 138 .token_fd = token_fd, 139 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 140 .fd_array_cnt = 1, 141 ); 142 int ret, map, insn_cnt = ARRAY_SIZE(insns); 143 144 libbpf_sha256(insns, sizeof(insns), hash); 145 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_excl", sizeof(int), sizeof(int), 1, 146 &map_opts); 147 if (map < 0) 148 return 0; 149 150 prog_opts.fd_array = ↦ 151 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); 152 close(map); 153 return probe_fd(ret); 154 } 155 156 static int probe_kern_btf(int token_fd) 157 { 158 static const char strs[] = "\0int"; 159 __u32 types[] = { 160 /* int */ 161 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 162 }; 163 164 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 165 strs, sizeof(strs), token_fd)); 166 } 167 168 static int probe_kern_btf_func(int token_fd) 169 { 170 static const char strs[] = "\0int\0x\0a"; 171 /* void x(int a) {} */ 172 __u32 types[] = { 173 /* int */ 174 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 175 /* FUNC_PROTO */ /* [2] */ 176 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 177 BTF_PARAM_ENC(7, 1), 178 /* FUNC x */ /* [3] */ 179 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 180 }; 181 182 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 183 strs, sizeof(strs), token_fd)); 184 } 185 186 static int probe_kern_btf_func_global(int token_fd) 187 { 188 static const char strs[] = "\0int\0x\0a"; 189 /* static void x(int a) {} */ 190 __u32 types[] = { 191 /* int */ 192 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 193 /* FUNC_PROTO */ /* [2] */ 194 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 195 BTF_PARAM_ENC(7, 1), 196 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ 197 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), 198 }; 199 200 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 201 strs, sizeof(strs), token_fd)); 202 } 203 204 static int probe_kern_btf_datasec(int token_fd) 205 { 206 static const char strs[] = "\0x\0.data"; 207 /* static int a; */ 208 __u32 types[] = { 209 /* int */ 210 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 211 /* VAR x */ /* [2] */ 212 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 213 BTF_VAR_STATIC, 214 /* DATASEC val */ /* [3] */ 215 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 216 BTF_VAR_SECINFO_ENC(2, 0, 4), 217 }; 218 219 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 220 strs, sizeof(strs), token_fd)); 221 } 222 223 static int probe_kern_btf_qmark_datasec(int token_fd) 224 { 225 static const char strs[] = "\0x\0?.data"; 226 /* static int a; */ 227 __u32 types[] = { 228 /* int */ 229 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 230 /* VAR x */ /* [2] */ 231 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 232 BTF_VAR_STATIC, 233 /* DATASEC ?.data */ /* [3] */ 234 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 235 BTF_VAR_SECINFO_ENC(2, 0, 4), 236 }; 237 238 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 239 strs, sizeof(strs), token_fd)); 240 } 241 242 static int probe_kern_btf_float(int token_fd) 243 { 244 static const char strs[] = "\0float"; 245 __u32 types[] = { 246 /* float */ 247 BTF_TYPE_FLOAT_ENC(1, 4), 248 }; 249 250 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 251 strs, sizeof(strs), token_fd)); 252 } 253 254 static int probe_kern_btf_decl_tag(int token_fd) 255 { 256 static const char strs[] = "\0tag"; 257 __u32 types[] = { 258 /* int */ 259 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 260 /* VAR x */ /* [2] */ 261 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 262 BTF_VAR_STATIC, 263 /* attr */ 264 BTF_TYPE_DECL_TAG_ENC(1, 2, -1), 265 }; 266 267 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 268 strs, sizeof(strs), token_fd)); 269 } 270 271 static int probe_kern_btf_type_tag(int token_fd) 272 { 273 static const char strs[] = "\0tag"; 274 __u32 types[] = { 275 /* int */ 276 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 277 /* attr */ 278 BTF_TYPE_TYPE_TAG_ENC(1, 1), /* [2] */ 279 /* ptr */ 280 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2), /* [3] */ 281 }; 282 283 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 284 strs, sizeof(strs), token_fd)); 285 } 286 287 static int probe_kern_array_mmap(int token_fd) 288 { 289 LIBBPF_OPTS(bpf_map_create_opts, opts, 290 .map_flags = BPF_F_MMAPABLE | (token_fd ? BPF_F_TOKEN_FD : 0), 291 .token_fd = token_fd, 292 ); 293 int fd; 294 295 fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts); 296 return probe_fd(fd); 297 } 298 299 static int probe_kern_exp_attach_type(int token_fd) 300 { 301 LIBBPF_OPTS(bpf_prog_load_opts, opts, 302 .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE, 303 .token_fd = token_fd, 304 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 305 ); 306 struct bpf_insn insns[] = { 307 BPF_MOV64_IMM(BPF_REG_0, 0), 308 BPF_EXIT_INSN(), 309 }; 310 int fd, insn_cnt = ARRAY_SIZE(insns); 311 312 /* use any valid combination of program type and (optional) 313 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) 314 * to see if kernel supports expected_attach_type field for 315 * BPF_PROG_LOAD command 316 */ 317 fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts); 318 return probe_fd(fd); 319 } 320 321 static int probe_kern_probe_read_kernel(int token_fd) 322 { 323 LIBBPF_OPTS(bpf_prog_load_opts, opts, 324 .token_fd = token_fd, 325 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 326 ); 327 struct bpf_insn insns[] = { 328 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ 329 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ 330 BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ 331 BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ 332 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), 333 BPF_EXIT_INSN(), 334 }; 335 int fd, insn_cnt = ARRAY_SIZE(insns); 336 337 fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); 338 return probe_fd(fd); 339 } 340 341 static int probe_prog_bind_map(int token_fd) 342 { 343 struct bpf_insn insns[] = { 344 BPF_MOV64_IMM(BPF_REG_0, 0), 345 BPF_EXIT_INSN(), 346 }; 347 LIBBPF_OPTS(bpf_map_create_opts, map_opts, 348 .token_fd = token_fd, 349 .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, 350 ); 351 LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, 352 .token_fd = token_fd, 353 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 354 ); 355 int ret, map, prog, insn_cnt = ARRAY_SIZE(insns); 356 357 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, &map_opts); 358 if (map < 0) { 359 ret = -errno; 360 pr_warn("Error in %s(): %s. Couldn't create simple array map.\n", 361 __func__, errstr(ret)); 362 return ret; 363 } 364 365 prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); 366 if (prog < 0) { 367 close(map); 368 return 0; 369 } 370 371 ret = bpf_prog_bind_map(prog, map, NULL); 372 373 close(map); 374 close(prog); 375 376 return ret >= 0; 377 } 378 379 static int probe_module_btf(int token_fd) 380 { 381 static const char strs[] = "\0int"; 382 __u32 types[] = { 383 /* int */ 384 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 385 }; 386 struct bpf_btf_info info; 387 __u32 len = sizeof(info); 388 char name[16]; 389 int fd, err; 390 391 fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs), token_fd); 392 if (fd < 0) 393 return 0; /* BTF not supported at all */ 394 395 memset(&info, 0, sizeof(info)); 396 info.name = ptr_to_u64(name); 397 info.name_len = sizeof(name); 398 399 /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; 400 * kernel's module BTF support coincides with support for 401 * name/name_len fields in struct bpf_btf_info. 402 */ 403 err = bpf_btf_get_info_by_fd(fd, &info, &len); 404 close(fd); 405 return !err; 406 } 407 408 static int probe_perf_link(int token_fd) 409 { 410 struct bpf_insn insns[] = { 411 BPF_MOV64_IMM(BPF_REG_0, 0), 412 BPF_EXIT_INSN(), 413 }; 414 LIBBPF_OPTS(bpf_prog_load_opts, opts, 415 .token_fd = token_fd, 416 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 417 ); 418 int prog_fd, link_fd, err; 419 420 prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", 421 insns, ARRAY_SIZE(insns), &opts); 422 if (prog_fd < 0) 423 return -errno; 424 425 /* use invalid perf_event FD to get EBADF, if link is supported; 426 * otherwise EINVAL should be returned 427 */ 428 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); 429 err = -errno; /* close() can clobber errno */ 430 431 if (link_fd >= 0) 432 close(link_fd); 433 close(prog_fd); 434 435 return link_fd < 0 && err == -EBADF; 436 } 437 438 static int probe_uprobe_multi_link(int token_fd) 439 { 440 LIBBPF_OPTS(bpf_prog_load_opts, load_opts, 441 .expected_attach_type = BPF_TRACE_UPROBE_MULTI, 442 .token_fd = token_fd, 443 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 444 ); 445 LIBBPF_OPTS(bpf_link_create_opts, link_opts); 446 struct bpf_insn insns[] = { 447 BPF_MOV64_IMM(BPF_REG_0, 0), 448 BPF_EXIT_INSN(), 449 }; 450 int prog_fd, link_fd, err; 451 unsigned long offset = 0; 452 453 prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", 454 insns, ARRAY_SIZE(insns), &load_opts); 455 if (prog_fd < 0) 456 return -errno; 457 458 /* Creating uprobe in '/' binary should fail with -EBADF. */ 459 link_opts.uprobe_multi.path = "/"; 460 link_opts.uprobe_multi.offsets = &offset; 461 link_opts.uprobe_multi.cnt = 1; 462 463 link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts); 464 err = -errno; /* close() can clobber errno */ 465 466 if (link_fd >= 0 || err != -EBADF) { 467 if (link_fd >= 0) 468 close(link_fd); 469 close(prog_fd); 470 return 0; 471 } 472 473 /* Initial multi-uprobe support in kernel didn't handle PID filtering 474 * correctly (it was doing thread filtering, not process filtering). 475 * So now we'll detect if PID filtering logic was fixed, and, if not, 476 * we'll pretend multi-uprobes are not supported, if not. 477 * Multi-uprobes are used in USDT attachment logic, and we need to be 478 * conservative here, because multi-uprobe selection happens early at 479 * load time, while the use of PID filtering is known late at 480 * attachment time, at which point it's too late to undo multi-uprobe 481 * selection. 482 * 483 * Creating uprobe with pid == -1 for (invalid) '/' binary will fail 484 * early with -EINVAL on kernels with fixed PID filtering logic; 485 * otherwise -ESRCH would be returned if passed correct binary path 486 * (but we'll just get -BADF, of course). 487 */ 488 link_opts.uprobe_multi.pid = -1; /* invalid PID */ 489 link_opts.uprobe_multi.path = "/"; /* invalid path */ 490 link_opts.uprobe_multi.offsets = &offset; 491 link_opts.uprobe_multi.cnt = 1; 492 493 link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts); 494 err = -errno; /* close() can clobber errno */ 495 496 if (link_fd >= 0) 497 close(link_fd); 498 close(prog_fd); 499 500 return link_fd < 0 && err == -EINVAL; 501 } 502 503 static int probe_kern_bpf_cookie(int token_fd) 504 { 505 struct bpf_insn insns[] = { 506 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie), 507 BPF_EXIT_INSN(), 508 }; 509 LIBBPF_OPTS(bpf_prog_load_opts, opts, 510 .token_fd = token_fd, 511 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 512 ); 513 int ret, insn_cnt = ARRAY_SIZE(insns); 514 515 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); 516 return probe_fd(ret); 517 } 518 519 static int probe_kern_btf_enum64(int token_fd) 520 { 521 static const char strs[] = "\0enum64"; 522 __u32 types[] = { 523 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8), 524 }; 525 526 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 527 strs, sizeof(strs), token_fd)); 528 } 529 530 static int probe_kern_arg_ctx_tag(int token_fd) 531 { 532 static const char strs[] = "\0a\0b\0arg:ctx\0"; 533 const __u32 types[] = { 534 /* [1] INT */ 535 BTF_TYPE_INT_ENC(1 /* "a" */, BTF_INT_SIGNED, 0, 32, 4), 536 /* [2] PTR -> VOID */ 537 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 0), 538 /* [3] FUNC_PROTO `int(void *a)` */ 539 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 1), 540 BTF_PARAM_ENC(1 /* "a" */, 2), 541 /* [4] FUNC 'a' -> FUNC_PROTO (main prog) */ 542 BTF_TYPE_ENC(1 /* "a" */, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 3), 543 /* [5] FUNC_PROTO `int(void *b __arg_ctx)` */ 544 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 1), 545 BTF_PARAM_ENC(3 /* "b" */, 2), 546 /* [6] FUNC 'b' -> FUNC_PROTO (subprog) */ 547 BTF_TYPE_ENC(3 /* "b" */, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 5), 548 /* [7] DECL_TAG 'arg:ctx' -> func 'b' arg 'b' */ 549 BTF_TYPE_DECL_TAG_ENC(5 /* "arg:ctx" */, 6, 0), 550 }; 551 const struct bpf_insn insns[] = { 552 /* main prog */ 553 BPF_CALL_REL(+1), 554 BPF_EXIT_INSN(), 555 /* global subprog */ 556 BPF_EMIT_CALL(BPF_FUNC_get_func_ip), /* needs PTR_TO_CTX */ 557 BPF_EXIT_INSN(), 558 }; 559 const struct bpf_func_info_min func_infos[] = { 560 { 0, 4 }, /* main prog -> FUNC 'a' */ 561 { 2, 6 }, /* subprog -> FUNC 'b' */ 562 }; 563 LIBBPF_OPTS(bpf_prog_load_opts, opts, 564 .token_fd = token_fd, 565 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 566 ); 567 int prog_fd, btf_fd, insn_cnt = ARRAY_SIZE(insns); 568 569 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs), token_fd); 570 if (btf_fd < 0) 571 return 0; 572 573 opts.prog_btf_fd = btf_fd; 574 opts.func_info = &func_infos; 575 opts.func_info_cnt = ARRAY_SIZE(func_infos); 576 opts.func_info_rec_size = sizeof(func_infos[0]); 577 578 prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, "det_arg_ctx", 579 "GPL", insns, insn_cnt, &opts); 580 close(btf_fd); 581 582 return probe_fd(prog_fd); 583 } 584 585 static int probe_ldimm64_full_range_off(int token_fd) 586 { 587 char log_buf[1024]; 588 int prog_fd, map_fd; 589 int ret; 590 LIBBPF_OPTS(bpf_map_create_opts, map_opts, 591 .token_fd = token_fd, 592 .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, 593 ); 594 LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, 595 .token_fd = token_fd, 596 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 597 .log_buf = log_buf, 598 .log_size = sizeof(log_buf), 599 ); 600 struct bpf_insn insns[] = { 601 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 1UL << 30), 602 BPF_EXIT_INSN(), 603 }; 604 int insn_cnt = ARRAY_SIZE(insns); 605 606 map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr", sizeof(int), 1, 1, &map_opts); 607 if (map_fd < 0) { 608 ret = -errno; 609 pr_warn("Error in %s(): %s. Couldn't create simple array map.\n", 610 __func__, errstr(ret)); 611 return ret; 612 } 613 insns[0].imm = map_fd; 614 615 log_buf[0] = '\0'; 616 prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "global_reloc", "GPL", insns, insn_cnt, &prog_opts); 617 ret = -errno; 618 619 close(map_fd); 620 621 if (prog_fd >= 0) { 622 pr_warn("Error in %s(): Program loading unexpectedly succeeded.\n", __func__); 623 close(prog_fd); 624 return -EINVAL; 625 } 626 627 /* 628 * Feature is allowed if we're not failing with the error message 629 * "direct value offset of %u is not allowed" removed in 630 * 12a1fe6e12db ("bpf/verifier: Do not limit maximum direct offset into arena map"). 631 * We should instead fail with "invalid access to map value pointer". 632 * Ensure we match with one of the two and we're not failing with a 633 * different, unexpected message. 634 */ 635 if (strstr(log_buf, "direct value offset of")) 636 return 0; 637 638 if (!strstr(log_buf, "invalid access to map value pointer")) { 639 pr_warn("Error in %s(): Program unexpectedly failed with message: %s.\n", 640 __func__, log_buf); 641 return ret; 642 } 643 644 return 1; 645 } 646 647 #ifdef __x86_64__ 648 649 #ifndef __NR_uprobe 650 #define __NR_uprobe 336 651 #endif 652 653 static int probe_uprobe_syscall(int token_fd) 654 { 655 /* 656 * If kernel supports uprobe() syscall, it will return -EPROTO when called 657 * from the outside of a kernel-generated uprobe trampoline. 658 */ 659 return syscall(__NR_uprobe) < 0 && errno == EPROTO; 660 } 661 #else 662 static int probe_uprobe_syscall(int token_fd) 663 { 664 return 0; 665 } 666 #endif 667 668 static int probe_kern_btf_layout(int token_fd) 669 { 670 static const char strs[] = "\0int"; 671 __u32 types[] = { 672 /* int */ 673 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 674 }; 675 struct btf_layout layout[] = { 676 { 0, 0, 0 }, 677 { sizeof(__u32), 0, 0 }, 678 }; 679 struct btf_header hdr = { 680 .magic = BTF_MAGIC, 681 .version = BTF_VERSION, 682 .hdr_len = sizeof(struct btf_header), 683 .type_len = sizeof(types), 684 .str_off = sizeof(types) + sizeof(layout), 685 .str_len = sizeof(strs), 686 .layout_off = sizeof(types), 687 .layout_len = sizeof(layout), 688 }; 689 690 return probe_fd(libbpf__load_raw_btf_hdr(&hdr, (char *)types, strs, 691 (char *)layout, token_fd)); 692 } 693 694 static int probe_bpf_syscall_common_attrs(int token_fd) 695 { 696 return probe_sys_bpf_ext(); 697 } 698 699 static int probe_kern_percpu_data(int token_fd) 700 { 701 struct bpf_insn insns[] = { 702 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), 703 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), 704 BPF_EXIT_INSN(), 705 }; 706 LIBBPF_OPTS(bpf_map_create_opts, map_opts, 707 .token_fd = token_fd, 708 .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, 709 ); 710 LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, 711 .token_fd = token_fd, 712 .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, 713 ); 714 int ret, map, insn_cnt = ARRAY_SIZE(insns); 715 716 map = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "libbpf_percpu", sizeof(int), 8, 1, 717 &map_opts); 718 if (map < 0) { 719 pr_warn("Error in %s(): %s. Couldn't create simple percpu_array map.\n", 720 __func__, errstr(map)); 721 return map; 722 } 723 724 insns[0].imm = map; 725 726 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); 727 close(map); 728 return probe_fd(ret); 729 } 730 731 typedef int (*feature_probe_fn)(int /* token_fd */); 732 733 static struct kern_feature_cache feature_cache; 734 735 static struct kern_feature_desc { 736 const char *desc; 737 feature_probe_fn probe; 738 } feature_probes[__FEAT_CNT] = { 739 [FEAT_PROG_NAME] = { 740 "BPF program name", probe_kern_prog_name, 741 }, 742 [FEAT_GLOBAL_DATA] = { 743 "global variables", probe_kern_global_data, 744 }, 745 [FEAT_BTF] = { 746 "minimal BTF", probe_kern_btf, 747 }, 748 [FEAT_BTF_FUNC] = { 749 "BTF functions", probe_kern_btf_func, 750 }, 751 [FEAT_BTF_GLOBAL_FUNC] = { 752 "BTF global function", probe_kern_btf_func_global, 753 }, 754 [FEAT_BTF_DATASEC] = { 755 "BTF data section and variable", probe_kern_btf_datasec, 756 }, 757 [FEAT_ARRAY_MMAP] = { 758 "ARRAY map mmap()", probe_kern_array_mmap, 759 }, 760 [FEAT_EXP_ATTACH_TYPE] = { 761 "BPF_PROG_LOAD expected_attach_type attribute", 762 probe_kern_exp_attach_type, 763 }, 764 [FEAT_PROBE_READ_KERN] = { 765 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, 766 }, 767 [FEAT_PROG_BIND_MAP] = { 768 "BPF_PROG_BIND_MAP support", probe_prog_bind_map, 769 }, 770 [FEAT_MODULE_BTF] = { 771 "module BTF support", probe_module_btf, 772 }, 773 [FEAT_BTF_FLOAT] = { 774 "BTF_KIND_FLOAT support", probe_kern_btf_float, 775 }, 776 [FEAT_PERF_LINK] = { 777 "BPF perf link support", probe_perf_link, 778 }, 779 [FEAT_BTF_DECL_TAG] = { 780 "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag, 781 }, 782 [FEAT_BTF_TYPE_TAG] = { 783 "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag, 784 }, 785 [FEAT_MEMCG_ACCOUNT] = { 786 "memcg-based memory accounting", probe_memcg_account, 787 }, 788 [FEAT_BPF_COOKIE] = { 789 "BPF cookie support", probe_kern_bpf_cookie, 790 }, 791 [FEAT_BTF_ENUM64] = { 792 "BTF_KIND_ENUM64 support", probe_kern_btf_enum64, 793 }, 794 [FEAT_SYSCALL_WRAPPER] = { 795 "Kernel using syscall wrapper", probe_kern_syscall_wrapper, 796 }, 797 [FEAT_UPROBE_MULTI_LINK] = { 798 "BPF multi-uprobe link support", probe_uprobe_multi_link, 799 }, 800 [FEAT_ARG_CTX_TAG] = { 801 "kernel-side __arg_ctx tag", probe_kern_arg_ctx_tag, 802 }, 803 [FEAT_BTF_QMARK_DATASEC] = { 804 "BTF DATASEC names starting from '?'", probe_kern_btf_qmark_datasec, 805 }, 806 [FEAT_LDIMM64_FULL_RANGE_OFF] = { 807 "full range LDIMM64 support", probe_ldimm64_full_range_off, 808 }, 809 [FEAT_UPROBE_SYSCALL] = { 810 "kernel supports uprobe syscall", probe_uprobe_syscall, 811 }, 812 [FEAT_BTF_LAYOUT] = { 813 "kernel supports BTF layout", probe_kern_btf_layout, 814 }, 815 [FEAT_BPF_SYSCALL_COMMON_ATTRS] = { 816 "BPF syscall common attributes support", probe_bpf_syscall_common_attrs, 817 }, 818 [FEAT_PERCPU_DATA] = { 819 "kernel supports percpu data", probe_kern_percpu_data, 820 }, 821 [FEAT_FD_ARRAY_CNT] = { 822 "continuous fd_array (fd_array_cnt)", probe_kern_fd_array_cnt, 823 }, 824 [FEAT_EXCL_MAP_FD_ARRAY_CNT] = { 825 "exclusive map bound through fd_array_cnt", probe_kern_excl_map_fd_array_cnt, 826 }, 827 }; 828 829 bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) 830 { 831 struct kern_feature_desc *feat = &feature_probes[feat_id]; 832 int ret; 833 834 /* assume global feature cache, unless custom one is provided */ 835 if (!cache) 836 cache = &feature_cache; 837 838 if (READ_ONCE(cache->res[feat_id]) == FEAT_UNKNOWN) { 839 ret = feat->probe(cache->token_fd); 840 if (ret > 0) { 841 WRITE_ONCE(cache->res[feat_id], FEAT_SUPPORTED); 842 } else if (ret == 0) { 843 WRITE_ONCE(cache->res[feat_id], FEAT_MISSING); 844 } else { 845 pr_warn("Detection of kernel %s support failed: %s\n", 846 feat->desc, errstr(ret)); 847 WRITE_ONCE(cache->res[feat_id], FEAT_MISSING); 848 } 849 } 850 851 return READ_ONCE(cache->res[feat_id]) == FEAT_SUPPORTED; 852 } 853