1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ 2 3 /* 4 * Internal libbpf helpers. 5 * 6 * Copyright (c) 2019 Facebook 7 */ 8 9 #ifndef __LIBBPF_LIBBPF_INTERNAL_H 10 #define __LIBBPF_LIBBPF_INTERNAL_H 11 12 #include <stdlib.h> 13 #include <byteswap.h> 14 #include <limits.h> 15 #include <errno.h> 16 #include <linux/err.h> 17 #include <fcntl.h> 18 #include <unistd.h> 19 #include <sys/syscall.h> 20 #include <libelf.h> 21 #include "relo_core.h" 22 23 /* Android's libc doesn't support AT_EACCESS in faccessat() implementation 24 * ([0]), and just returns -EINVAL even if file exists and is accessible. 25 * See [1] for issues caused by this. 26 * 27 * So just redefine it to 0 on Android. 28 * 29 * [0] https://android.googlesource.com/platform/bionic/+/refs/heads/android13-release/libc/bionic/faccessat.cpp#50 30 * [1] https://github.com/libbpf/libbpf-bootstrap/issues/250#issuecomment-1911324250 31 */ 32 #ifdef __ANDROID__ 33 #undef AT_EACCESS 34 #define AT_EACCESS 0 35 #endif 36 37 /* make sure libbpf doesn't use kernel-only integer typedefs */ 38 #pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 39 40 /* prevent accidental re-addition of reallocarray() */ 41 #pragma GCC poison reallocarray 42 43 #include "libbpf.h" 44 #include "btf.h" 45 46 #ifndef EM_BPF 47 #define EM_BPF 247 48 #endif 49 50 #ifndef R_BPF_64_64 51 #define R_BPF_64_64 1 52 #endif 53 #ifndef R_BPF_64_ABS64 54 #define R_BPF_64_ABS64 2 55 #endif 56 #ifndef R_BPF_64_ABS32 57 #define R_BPF_64_ABS32 3 58 #endif 59 #ifndef R_BPF_64_32 60 #define R_BPF_64_32 10 61 #endif 62 63 #ifndef SHT_LLVM_ADDRSIG 64 #define SHT_LLVM_ADDRSIG 0x6FFF4C03 65 #endif 66 67 /* if libelf is old and doesn't support mmap(), fall back to read() */ 68 #ifndef ELF_C_READ_MMAP 69 #define ELF_C_READ_MMAP ELF_C_READ 70 #endif 71 72 /* Older libelf all end up in this expression, for both 32 and 64 bit */ 73 #ifndef ELF64_ST_VISIBILITY 74 #define ELF64_ST_VISIBILITY(o) ((o) & 0x03) 75 #endif 76 77 #define BTF_INFO_ENC(kind, kind_flag, vlen) \ 78 ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) 79 #define BTF_TYPE_ENC(name, info, size_or_type) (name), (info), (size_or_type) 80 #define BTF_INT_ENC(encoding, bits_offset, nr_bits) \ 81 ((encoding) << 24 | (bits_offset) << 16 | (nr_bits)) 82 #define BTF_TYPE_INT_ENC(name, encoding, bits_offset, bits, sz) \ 83 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \ 84 BTF_INT_ENC(encoding, bits_offset, bits) 85 #define BTF_MEMBER_ENC(name, type, bits_offset) (name), (type), (bits_offset) 86 #define BTF_PARAM_ENC(name, type) (name), (type) 87 #define BTF_VAR_SECINFO_ENC(type, offset, size) (type), (offset), (size) 88 #define BTF_TYPE_FLOAT_ENC(name, sz) \ 89 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz) 90 #define BTF_TYPE_DECL_TAG_ENC(value, type, component_idx) \ 91 BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 0, 0), type), (component_idx) 92 #define BTF_TYPE_TYPE_TAG_ENC(value, type) \ 93 BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TYPE_TAG, 0, 0), type) 94 95 #ifndef likely 96 #define likely(x) __builtin_expect(!!(x), 1) 97 #endif 98 #ifndef unlikely 99 #define unlikely(x) __builtin_expect(!!(x), 0) 100 #endif 101 #ifndef min 102 # define min(x, y) ((x) < (y) ? (x) : (y)) 103 #endif 104 #ifndef max 105 # define max(x, y) ((x) < (y) ? (y) : (x)) 106 #endif 107 #ifndef offsetofend 108 # define offsetofend(TYPE, FIELD) \ 109 (offsetof(TYPE, FIELD) + sizeof(((TYPE *)0)->FIELD)) 110 #endif 111 #ifndef __alias 112 #define __alias(symbol) __attribute__((alias(#symbol))) 113 #endif 114 115 /* Check whether a string `str` has prefix `pfx`, regardless if `pfx` is 116 * a string literal known at compilation time or char * pointer known only at 117 * runtime. 118 */ 119 #define str_has_pfx(str, pfx) \ 120 (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0) 121 122 /* suffix check */ 123 static inline bool str_has_sfx(const char *str, const char *sfx) 124 { 125 size_t str_len = strlen(str); 126 size_t sfx_len = strlen(sfx); 127 128 if (sfx_len > str_len) 129 return false; 130 return strcmp(str + str_len - sfx_len, sfx) == 0; 131 } 132 133 /* Symbol versioning is different between static and shared library. 134 * Properly versioned symbols are needed for shared library, but 135 * only the symbol of the new version is needed for static library. 136 * Starting with GNU C 10, use symver attribute instead of .symver assembler 137 * directive, which works better with GCC LTO builds. 138 */ 139 #if defined(SHARED) && defined(__GNUC__) && __GNUC__ >= 10 140 141 #define DEFAULT_VERSION(internal_name, api_name, version) \ 142 __attribute__((symver(#api_name "@@" #version))) 143 #define COMPAT_VERSION(internal_name, api_name, version) \ 144 __attribute__((symver(#api_name "@" #version))) 145 146 #elif defined(SHARED) 147 148 #define COMPAT_VERSION(internal_name, api_name, version) \ 149 asm(".symver " #internal_name "," #api_name "@" #version); 150 #define DEFAULT_VERSION(internal_name, api_name, version) \ 151 asm(".symver " #internal_name "," #api_name "@@" #version); 152 153 #else /* !SHARED */ 154 155 #define COMPAT_VERSION(internal_name, api_name, version) 156 #define DEFAULT_VERSION(internal_name, api_name, version) \ 157 extern typeof(internal_name) api_name \ 158 __attribute__((alias(#internal_name))); 159 160 #endif 161 162 extern void libbpf_print(enum libbpf_print_level level, 163 const char *format, ...) 164 __attribute__((format(printf, 2, 3))); 165 166 #define __pr(level, fmt, ...) \ 167 do { \ 168 libbpf_print(level, "libbpf: " fmt, ##__VA_ARGS__); \ 169 } while (0) 170 171 #define pr_warn(fmt, ...) __pr(LIBBPF_WARN, fmt, ##__VA_ARGS__) 172 #define pr_info(fmt, ...) __pr(LIBBPF_INFO, fmt, ##__VA_ARGS__) 173 #define pr_debug(fmt, ...) __pr(LIBBPF_DEBUG, fmt, ##__VA_ARGS__) 174 175 /** 176 * @brief **libbpf_errstr()** returns string corresponding to numeric errno 177 * @param err negative numeric errno 178 * @return pointer to string representation of the errno, that is invalidated 179 * upon the next call. 180 */ 181 const char *libbpf_errstr(int err); 182 183 #define errstr(err) libbpf_errstr(err) 184 185 #ifndef __has_builtin 186 #define __has_builtin(x) 0 187 #endif 188 189 struct bpf_link { 190 int (*detach)(struct bpf_link *link); 191 void (*dealloc)(struct bpf_link *link); 192 char *pin_path; /* NULL, if not pinned */ 193 int fd; /* hook FD, -1 if not applicable */ 194 bool disconnected; 195 }; 196 197 /* 198 * Re-implement glibc's reallocarray() for libbpf internal-only use. 199 * reallocarray(), unfortunately, is not available in all versions of glibc, 200 * so requires extra feature detection and using reallocarray() stub from 201 * <tools/libc_compat.h> and COMPAT_NEED_REALLOCARRAY. All this complicates 202 * build of libbpf unnecessarily and is just a maintenance burden. Instead, 203 * it's trivial to implement libbpf-specific internal version and use it 204 * throughout libbpf. 205 */ 206 static inline void *libbpf_reallocarray(void *ptr, size_t nmemb, size_t size) 207 { 208 size_t total; 209 210 #if __has_builtin(__builtin_mul_overflow) 211 if (unlikely(__builtin_mul_overflow(nmemb, size, &total))) 212 return NULL; 213 #else 214 if (size == 0 || nmemb > ULONG_MAX / size) 215 return NULL; 216 total = nmemb * size; 217 #endif 218 return realloc(ptr, total); 219 } 220 221 /* Copy up to sz - 1 bytes from zero-terminated src string and ensure that dst 222 * is zero-terminated string no matter what (unless sz == 0, in which case 223 * it's a no-op). It's conceptually close to FreeBSD's strlcpy(), but differs 224 * in what is returned. Given this is internal helper, it's trivial to extend 225 * this, when necessary. Use this instead of strncpy inside libbpf source code. 226 */ 227 static inline void libbpf_strlcpy(char *dst, const char *src, size_t sz) 228 { 229 size_t i; 230 231 if (sz == 0) 232 return; 233 234 sz--; 235 for (i = 0; i < sz && src[i]; i++) 236 dst[i] = src[i]; 237 dst[i] = '\0'; 238 } 239 240 __u32 get_kernel_version(void); 241 242 struct btf; 243 struct btf_type; 244 245 struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id); 246 const char *btf_kind_str(const struct btf_type *t); 247 const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id); 248 const struct btf_header *btf_header(const struct btf *btf); 249 void btf_set_base_btf(struct btf *btf, const struct btf *base_btf); 250 int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **id_map); 251 252 static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t) 253 { 254 return (enum btf_func_linkage)(int)btf_vlen(t); 255 } 256 257 static inline __u32 btf_type_info(int kind, int vlen, int kflag) 258 { 259 return (kflag << 31) | (kind << 24) | vlen; 260 } 261 262 enum map_def_parts { 263 MAP_DEF_MAP_TYPE = 0x001, 264 MAP_DEF_KEY_TYPE = 0x002, 265 MAP_DEF_KEY_SIZE = 0x004, 266 MAP_DEF_VALUE_TYPE = 0x008, 267 MAP_DEF_VALUE_SIZE = 0x010, 268 MAP_DEF_MAX_ENTRIES = 0x020, 269 MAP_DEF_MAP_FLAGS = 0x040, 270 MAP_DEF_NUMA_NODE = 0x080, 271 MAP_DEF_PINNING = 0x100, 272 MAP_DEF_INNER_MAP = 0x200, 273 MAP_DEF_MAP_EXTRA = 0x400, 274 275 MAP_DEF_ALL = 0x7ff, /* combination of all above */ 276 }; 277 278 struct btf_map_def { 279 enum map_def_parts parts; 280 __u32 map_type; 281 __u32 key_type_id; 282 __u32 key_size; 283 __u32 value_type_id; 284 __u32 value_size; 285 __u32 max_entries; 286 __u32 map_flags; 287 __u32 numa_node; 288 __u32 pinning; 289 __u64 map_extra; 290 }; 291 292 int parse_btf_map_def(const char *map_name, struct btf *btf, 293 const struct btf_type *def_t, bool strict, 294 struct btf_map_def *map_def, struct btf_map_def *inner_def); 295 296 void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, 297 size_t cur_cnt, size_t max_cnt, size_t add_cnt); 298 int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt); 299 300 static inline bool libbpf_is_mem_zeroed(const char *p, ssize_t len) 301 { 302 while (len > 0) { 303 if (*p) 304 return false; 305 p++; 306 len--; 307 } 308 return true; 309 } 310 311 static inline bool libbpf_validate_opts(const char *opts, 312 size_t opts_sz, size_t user_sz, 313 const char *type_name) 314 { 315 if (user_sz < sizeof(size_t)) { 316 pr_warn("%s size (%zu) is too small\n", type_name, user_sz); 317 return false; 318 } 319 if (!libbpf_is_mem_zeroed(opts + opts_sz, (ssize_t)user_sz - opts_sz)) { 320 pr_warn("%s has non-zero extra bytes\n", type_name); 321 return false; 322 } 323 return true; 324 } 325 326 #define OPTS_VALID(opts, type) \ 327 (!(opts) || libbpf_validate_opts((const char *)opts, \ 328 offsetofend(struct type, \ 329 type##__last_field), \ 330 (opts)->sz, #type)) 331 #define OPTS_HAS(opts, field) \ 332 ((opts) && opts->sz >= offsetofend(typeof(*(opts)), field)) 333 #define OPTS_GET(opts, field, fallback_value) \ 334 (OPTS_HAS(opts, field) ? (opts)->field : fallback_value) 335 #define OPTS_SET(opts, field, value) \ 336 do { \ 337 if (OPTS_HAS(opts, field)) \ 338 (opts)->field = value; \ 339 } while (0) 340 341 #define OPTS_ZEROED(opts, last_nonzero_field) \ 342 ({ \ 343 ssize_t __off = offsetofend(typeof(*(opts)), last_nonzero_field); \ 344 !(opts) || libbpf_is_mem_zeroed((const void *)opts + __off, \ 345 (opts)->sz - __off); \ 346 }) 347 348 enum kern_feature_id { 349 /* v4.14: kernel support for program & map names. */ 350 FEAT_PROG_NAME, 351 /* v5.2: kernel support for global data sections. */ 352 FEAT_GLOBAL_DATA, 353 /* BTF support */ 354 FEAT_BTF, 355 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ 356 FEAT_BTF_FUNC, 357 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ 358 FEAT_BTF_DATASEC, 359 /* BTF_FUNC_GLOBAL is supported */ 360 FEAT_BTF_GLOBAL_FUNC, 361 /* BPF_F_MMAPABLE is supported for arrays */ 362 FEAT_ARRAY_MMAP, 363 /* kernel support for expected_attach_type in BPF_PROG_LOAD */ 364 FEAT_EXP_ATTACH_TYPE, 365 /* bpf_probe_read_{kernel,user}[_str] helpers */ 366 FEAT_PROBE_READ_KERN, 367 /* BPF_PROG_BIND_MAP is supported */ 368 FEAT_PROG_BIND_MAP, 369 /* Kernel support for module BTFs */ 370 FEAT_MODULE_BTF, 371 /* BTF_KIND_FLOAT support */ 372 FEAT_BTF_FLOAT, 373 /* BPF perf link support */ 374 FEAT_PERF_LINK, 375 /* BTF_KIND_DECL_TAG support */ 376 FEAT_BTF_DECL_TAG, 377 /* BTF_KIND_TYPE_TAG support */ 378 FEAT_BTF_TYPE_TAG, 379 /* memcg-based accounting for BPF maps and progs */ 380 FEAT_MEMCG_ACCOUNT, 381 /* BPF cookie (bpf_get_attach_cookie() BPF helper) support */ 382 FEAT_BPF_COOKIE, 383 /* BTF_KIND_ENUM64 support and BTF_KIND_ENUM kflag support */ 384 FEAT_BTF_ENUM64, 385 /* Kernel uses syscall wrapper (CONFIG_ARCH_HAS_SYSCALL_WRAPPER) */ 386 FEAT_SYSCALL_WRAPPER, 387 /* BPF multi-uprobe link support */ 388 FEAT_UPROBE_MULTI_LINK, 389 /* Kernel supports arg:ctx tag (__arg_ctx) for global subprogs natively */ 390 FEAT_ARG_CTX_TAG, 391 /* Kernel supports '?' at the front of datasec names */ 392 FEAT_BTF_QMARK_DATASEC, 393 __FEAT_CNT, 394 }; 395 396 enum kern_feature_result { 397 FEAT_UNKNOWN = 0, 398 FEAT_SUPPORTED = 1, 399 FEAT_MISSING = 2, 400 }; 401 402 struct kern_feature_cache { 403 enum kern_feature_result res[__FEAT_CNT]; 404 int token_fd; 405 }; 406 407 bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id); 408 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id); 409 410 int probe_kern_syscall_wrapper(int token_fd); 411 int probe_memcg_account(int token_fd); 412 int bump_rlimit_memlock(void); 413 414 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz); 415 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); 416 int libbpf__load_raw_btf(const char *raw_types, size_t types_len, 417 const char *str_sec, size_t str_len, 418 int token_fd); 419 int btf_load_into_kernel(struct btf *btf, 420 char *log_buf, size_t log_sz, __u32 log_level, 421 int token_fd); 422 struct btf *btf_load_from_kernel(__u32 id, struct btf *base_btf, int token_fd); 423 424 struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf); 425 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, 426 const char **prefix, int *kind); 427 428 struct btf_ext_info { 429 /* 430 * info points to the individual info section (e.g. func_info and 431 * line_info) from the .BTF.ext. It does not include the __u32 rec_size. 432 */ 433 void *info; 434 __u32 rec_size; 435 __u32 len; 436 /* optional (maintained internally by libbpf) mapping between .BTF.ext 437 * section and corresponding ELF section. This is used to join 438 * information like CO-RE relocation records with corresponding BPF 439 * programs defined in ELF sections 440 */ 441 __u32 *sec_idxs; 442 int sec_cnt; 443 }; 444 445 #define for_each_btf_ext_sec(seg, sec) \ 446 for (sec = (seg)->info; \ 447 (void *)sec < (seg)->info + (seg)->len; \ 448 sec = (void *)sec + sizeof(struct btf_ext_info_sec) + \ 449 (seg)->rec_size * sec->num_info) 450 451 #define for_each_btf_ext_rec(seg, sec, i, rec) \ 452 for (i = 0, rec = (void *)&(sec)->data; \ 453 i < (sec)->num_info; \ 454 i++, rec = (void *)rec + (seg)->rec_size) 455 456 /* 457 * The .BTF.ext ELF section layout defined as 458 * struct btf_ext_header 459 * func_info subsection 460 * 461 * The func_info subsection layout: 462 * record size for struct bpf_func_info in the func_info subsection 463 * struct btf_ext_info_sec for section #1 464 * a list of bpf_func_info records for section #1 465 * where struct bpf_func_info mimics one in include/uapi/linux/bpf.h 466 * but may not be identical 467 * struct btf_ext_info_sec for section #2 468 * a list of bpf_func_info records for section #2 469 * ...... 470 * 471 * Note that the bpf_func_info record size in .BTF.ext may not 472 * be the same as the one defined in include/uapi/linux/bpf.h. 473 * The loader should ensure that record_size meets minimum 474 * requirement and pass the record as is to the kernel. The 475 * kernel will handle the func_info properly based on its contents. 476 */ 477 struct btf_ext_header { 478 __u16 magic; 479 __u8 version; 480 __u8 flags; 481 __u32 hdr_len; 482 483 /* All offsets are in bytes relative to the end of this header */ 484 __u32 func_info_off; 485 __u32 func_info_len; 486 __u32 line_info_off; 487 __u32 line_info_len; 488 489 /* optional part of .BTF.ext header */ 490 __u32 core_relo_off; 491 __u32 core_relo_len; 492 }; 493 494 struct btf_ext { 495 union { 496 struct btf_ext_header *hdr; 497 void *data; 498 }; 499 void *data_swapped; 500 bool swapped_endian; 501 struct btf_ext_info func_info; 502 struct btf_ext_info line_info; 503 struct btf_ext_info core_relo_info; 504 __u32 data_size; 505 }; 506 507 struct btf_ext_info_sec { 508 __u32 sec_name_off; 509 __u32 num_info; 510 /* Followed by num_info * record_size number of bytes */ 511 __u8 data[]; 512 }; 513 514 /* The minimum bpf_func_info checked by the loader */ 515 struct bpf_func_info_min { 516 __u32 insn_off; 517 __u32 type_id; 518 }; 519 520 /* The minimum bpf_line_info checked by the loader */ 521 struct bpf_line_info_min { 522 __u32 insn_off; 523 __u32 file_name_off; 524 __u32 line_off; 525 __u32 line_col; 526 }; 527 528 /* Functions to byte-swap info records */ 529 530 typedef void (*info_rec_bswap_fn)(void *); 531 532 static inline void bpf_func_info_bswap(struct bpf_func_info *i) 533 { 534 i->insn_off = bswap_32(i->insn_off); 535 i->type_id = bswap_32(i->type_id); 536 } 537 538 static inline void bpf_line_info_bswap(struct bpf_line_info *i) 539 { 540 i->insn_off = bswap_32(i->insn_off); 541 i->file_name_off = bswap_32(i->file_name_off); 542 i->line_off = bswap_32(i->line_off); 543 i->line_col = bswap_32(i->line_col); 544 } 545 546 static inline void bpf_core_relo_bswap(struct bpf_core_relo *i) 547 { 548 i->insn_off = bswap_32(i->insn_off); 549 i->type_id = bswap_32(i->type_id); 550 i->access_str_off = bswap_32(i->access_str_off); 551 i->kind = bswap_32(i->kind); 552 } 553 554 enum btf_field_iter_kind { 555 BTF_FIELD_ITER_IDS, 556 BTF_FIELD_ITER_STRS, 557 }; 558 559 struct btf_field_desc { 560 /* once-per-type offsets */ 561 int t_off_cnt, t_offs[2]; 562 /* member struct size, or zero, if no members */ 563 int m_sz; 564 /* repeated per-member offsets */ 565 int m_off_cnt, m_offs[1]; 566 }; 567 568 struct btf_field_iter { 569 struct btf_field_desc desc; 570 void *p; 571 int m_idx; 572 int off_idx; 573 int vlen; 574 }; 575 576 int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, enum btf_field_iter_kind iter_kind); 577 __u32 *btf_field_iter_next(struct btf_field_iter *it); 578 579 typedef int (*type_id_visit_fn)(__u32 *type_id, void *ctx); 580 typedef int (*str_off_visit_fn)(__u32 *str_off, void *ctx); 581 int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx); 582 int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx); 583 __s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, 584 __u32 kind); 585 586 /* handle direct returned errors */ 587 static inline int libbpf_err(int ret) 588 { 589 if (ret < 0) 590 errno = -ret; 591 return ret; 592 } 593 594 /* handle errno-based (e.g., syscall or libc) errors according to libbpf's 595 * strict mode settings 596 */ 597 static inline int libbpf_err_errno(int ret) 598 { 599 /* errno is already assumed to be set on error */ 600 return ret < 0 ? -errno : ret; 601 } 602 603 /* handle error for pointer-returning APIs, err is assumed to be < 0 always */ 604 static inline void *libbpf_err_ptr(int err) 605 { 606 /* set errno on error, this doesn't break anything */ 607 errno = -err; 608 return NULL; 609 } 610 611 /* handle pointer-returning APIs' error handling */ 612 static inline void *libbpf_ptr(void *ret) 613 { 614 /* set errno on error, this doesn't break anything */ 615 if (IS_ERR(ret)) 616 errno = -PTR_ERR(ret); 617 618 return IS_ERR(ret) ? NULL : ret; 619 } 620 621 static inline bool str_is_empty(const char *s) 622 { 623 return !s || !s[0]; 624 } 625 626 static inline bool is_ldimm64_insn(struct bpf_insn *insn) 627 { 628 return insn->code == (BPF_LD | BPF_IMM | BPF_DW); 629 } 630 631 static inline void bpf_insn_bswap(struct bpf_insn *insn) 632 { 633 __u8 tmp_reg = insn->dst_reg; 634 635 insn->dst_reg = insn->src_reg; 636 insn->src_reg = tmp_reg; 637 insn->off = bswap_16(insn->off); 638 insn->imm = bswap_32(insn->imm); 639 } 640 641 /* Unconditionally dup FD, ensuring it doesn't use [0, 2] range. 642 * Original FD is not closed or altered in any other way. 643 * Preserves original FD value, if it's invalid (negative). 644 */ 645 static inline int dup_good_fd(int fd) 646 { 647 if (fd < 0) 648 return fd; 649 return fcntl(fd, F_DUPFD_CLOEXEC, 3); 650 } 651 652 /* if fd is stdin, stdout, or stderr, dup to a fd greater than 2 653 * Takes ownership of the fd passed in, and closes it if calling 654 * fcntl(fd, F_DUPFD_CLOEXEC, 3). 655 */ 656 static inline int ensure_good_fd(int fd) 657 { 658 int old_fd = fd, saved_errno; 659 660 if (fd < 0) 661 return fd; 662 if (fd < 3) { 663 fd = dup_good_fd(fd); 664 saved_errno = errno; 665 close(old_fd); 666 errno = saved_errno; 667 if (fd < 0) { 668 pr_warn("failed to dup FD %d to FD > 2: %d\n", old_fd, -saved_errno); 669 errno = saved_errno; 670 } 671 } 672 return fd; 673 } 674 675 static inline int sys_dup3(int oldfd, int newfd, int flags) 676 { 677 return syscall(__NR_dup3, oldfd, newfd, flags); 678 } 679 680 /* Some versions of Android don't provide memfd_create() in their libc 681 * implementation, so avoid complications and just go straight to Linux 682 * syscall. 683 */ 684 static inline int sys_memfd_create(const char *name, unsigned flags) 685 { 686 return syscall(__NR_memfd_create, name, flags); 687 } 688 689 /* Point *fixed_fd* to the same file that *tmp_fd* points to. 690 * Regardless of success, *tmp_fd* is closed. 691 * Whatever *fixed_fd* pointed to is closed silently. 692 */ 693 static inline int reuse_fd(int fixed_fd, int tmp_fd) 694 { 695 int err; 696 697 err = sys_dup3(tmp_fd, fixed_fd, O_CLOEXEC); 698 err = err < 0 ? -errno : 0; 699 close(tmp_fd); /* clean up temporary FD */ 700 return err; 701 } 702 703 /* The following two functions are exposed to bpftool */ 704 int bpf_core_add_cands(struct bpf_core_cand *local_cand, 705 size_t local_essent_len, 706 const struct btf *targ_btf, 707 const char *targ_btf_name, 708 int targ_start_id, 709 struct bpf_core_cand_list *cands); 710 void bpf_core_free_cands(struct bpf_core_cand_list *cands); 711 712 struct usdt_manager *usdt_manager_new(struct bpf_object *obj); 713 void usdt_manager_free(struct usdt_manager *man); 714 struct bpf_link * usdt_manager_attach_usdt(struct usdt_manager *man, 715 const struct bpf_program *prog, 716 pid_t pid, const char *path, 717 const char *usdt_provider, const char *usdt_name, 718 __u64 usdt_cookie); 719 720 static inline bool is_pow_of_2(size_t x) 721 { 722 return x && (x & (x - 1)) == 0; 723 } 724 725 static inline __u32 ror32(__u32 v, int bits) 726 { 727 return (v >> bits) | (v << (32 - bits)); 728 } 729 730 #define PROG_LOAD_ATTEMPTS 5 731 int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts); 732 733 bool glob_match(const char *str, const char *pat); 734 735 long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name); 736 long elf_find_func_offset_from_file(const char *binary_path, const char *name); 737 738 struct elf_fd { 739 Elf *elf; 740 int fd; 741 }; 742 743 int elf_open(const char *binary_path, struct elf_fd *elf_fd); 744 void elf_close(struct elf_fd *elf_fd); 745 746 int elf_resolve_syms_offsets(const char *binary_path, int cnt, 747 const char **syms, unsigned long **poffsets, 748 int st_type); 749 int elf_resolve_pattern_offsets(const char *binary_path, const char *pattern, 750 unsigned long **poffsets, size_t *pcnt); 751 752 int probe_fd(int fd); 753 754 #define SHA256_DIGEST_LENGTH 32 755 #define SHA256_DWORD_SIZE SHA256_DIGEST_LENGTH / sizeof(__u64) 756 757 void libbpf_sha256(const void *data, size_t len, __u8 out[SHA256_DIGEST_LENGTH]); 758 #endif /* __LIBBPF_LIBBPF_INTERNAL_H */ 759