1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <limits.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <endian.h> 25 #include <fcntl.h> 26 #include <errno.h> 27 #include <ctype.h> 28 #include <asm/unistd.h> 29 #include <linux/err.h> 30 #include <linux/kernel.h> 31 #include <linux/bpf.h> 32 #include <linux/btf.h> 33 #include <linux/filter.h> 34 #include <linux/limits.h> 35 #include <linux/perf_event.h> 36 #include <linux/ring_buffer.h> 37 #include <sys/epoll.h> 38 #include <sys/ioctl.h> 39 #include <sys/mman.h> 40 #include <sys/stat.h> 41 #include <sys/types.h> 42 #include <sys/vfs.h> 43 #include <sys/utsname.h> 44 #include <sys/resource.h> 45 #include <libelf.h> 46 #include <gelf.h> 47 #include <zlib.h> 48 49 #include "libbpf.h" 50 #include "bpf.h" 51 #include "btf.h" 52 #include "str_error.h" 53 #include "libbpf_internal.h" 54 #include "hashmap.h" 55 #include "bpf_gen_internal.h" 56 #include "zip.h" 57 58 #ifndef BPF_FS_MAGIC 59 #define BPF_FS_MAGIC 0xcafe4a11 60 #endif 61 62 #define BPF_INSN_SZ (sizeof(struct bpf_insn)) 63 64 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 65 * compilation if user enables corresponding warning. Disable it explicitly. 66 */ 67 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 68 69 #define __printf(a, b) __attribute__((format(printf, a, b))) 70 71 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); 72 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog); 73 74 static const char * const attach_type_name[] = { 75 [BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress", 76 [BPF_CGROUP_INET_EGRESS] = "cgroup_inet_egress", 77 [BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create", 78 [BPF_CGROUP_INET_SOCK_RELEASE] = "cgroup_inet_sock_release", 79 [BPF_CGROUP_SOCK_OPS] = "cgroup_sock_ops", 80 [BPF_CGROUP_DEVICE] = "cgroup_device", 81 [BPF_CGROUP_INET4_BIND] = "cgroup_inet4_bind", 82 [BPF_CGROUP_INET6_BIND] = "cgroup_inet6_bind", 83 [BPF_CGROUP_INET4_CONNECT] = "cgroup_inet4_connect", 84 [BPF_CGROUP_INET6_CONNECT] = "cgroup_inet6_connect", 85 [BPF_CGROUP_INET4_POST_BIND] = "cgroup_inet4_post_bind", 86 [BPF_CGROUP_INET6_POST_BIND] = "cgroup_inet6_post_bind", 87 [BPF_CGROUP_INET4_GETPEERNAME] = "cgroup_inet4_getpeername", 88 [BPF_CGROUP_INET6_GETPEERNAME] = "cgroup_inet6_getpeername", 89 [BPF_CGROUP_INET4_GETSOCKNAME] = "cgroup_inet4_getsockname", 90 [BPF_CGROUP_INET6_GETSOCKNAME] = "cgroup_inet6_getsockname", 91 [BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg", 92 [BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg", 93 [BPF_CGROUP_SYSCTL] = "cgroup_sysctl", 94 [BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg", 95 [BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg", 96 [BPF_CGROUP_GETSOCKOPT] = "cgroup_getsockopt", 97 [BPF_CGROUP_SETSOCKOPT] = "cgroup_setsockopt", 98 [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser", 99 [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict", 100 [BPF_SK_SKB_VERDICT] = "sk_skb_verdict", 101 [BPF_SK_MSG_VERDICT] = "sk_msg_verdict", 102 [BPF_LIRC_MODE2] = "lirc_mode2", 103 [BPF_FLOW_DISSECTOR] = "flow_dissector", 104 [BPF_TRACE_RAW_TP] = "trace_raw_tp", 105 [BPF_TRACE_FENTRY] = "trace_fentry", 106 [BPF_TRACE_FEXIT] = "trace_fexit", 107 [BPF_MODIFY_RETURN] = "modify_return", 108 [BPF_LSM_MAC] = "lsm_mac", 109 [BPF_LSM_CGROUP] = "lsm_cgroup", 110 [BPF_SK_LOOKUP] = "sk_lookup", 111 [BPF_TRACE_ITER] = "trace_iter", 112 [BPF_XDP_DEVMAP] = "xdp_devmap", 113 [BPF_XDP_CPUMAP] = "xdp_cpumap", 114 [BPF_XDP] = "xdp", 115 [BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select", 116 [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_reuseport_select_or_migrate", 117 [BPF_PERF_EVENT] = "perf_event", 118 [BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi", 119 [BPF_STRUCT_OPS] = "struct_ops", 120 [BPF_NETFILTER] = "netfilter", 121 [BPF_TCX_INGRESS] = "tcx_ingress", 122 [BPF_TCX_EGRESS] = "tcx_egress", 123 [BPF_TRACE_UPROBE_MULTI] = "trace_uprobe_multi", 124 }; 125 126 static const char * const link_type_name[] = { 127 [BPF_LINK_TYPE_UNSPEC] = "unspec", 128 [BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", 129 [BPF_LINK_TYPE_TRACING] = "tracing", 130 [BPF_LINK_TYPE_CGROUP] = "cgroup", 131 [BPF_LINK_TYPE_ITER] = "iter", 132 [BPF_LINK_TYPE_NETNS] = "netns", 133 [BPF_LINK_TYPE_XDP] = "xdp", 134 [BPF_LINK_TYPE_PERF_EVENT] = "perf_event", 135 [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi", 136 [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops", 137 [BPF_LINK_TYPE_NETFILTER] = "netfilter", 138 [BPF_LINK_TYPE_TCX] = "tcx", 139 [BPF_LINK_TYPE_UPROBE_MULTI] = "uprobe_multi", 140 }; 141 142 static const char * const map_type_name[] = { 143 [BPF_MAP_TYPE_UNSPEC] = "unspec", 144 [BPF_MAP_TYPE_HASH] = "hash", 145 [BPF_MAP_TYPE_ARRAY] = "array", 146 [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array", 147 [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array", 148 [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash", 149 [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array", 150 [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace", 151 [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array", 152 [BPF_MAP_TYPE_LRU_HASH] = "lru_hash", 153 [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash", 154 [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie", 155 [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps", 156 [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps", 157 [BPF_MAP_TYPE_DEVMAP] = "devmap", 158 [BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash", 159 [BPF_MAP_TYPE_SOCKMAP] = "sockmap", 160 [BPF_MAP_TYPE_CPUMAP] = "cpumap", 161 [BPF_MAP_TYPE_XSKMAP] = "xskmap", 162 [BPF_MAP_TYPE_SOCKHASH] = "sockhash", 163 [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage", 164 [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray", 165 [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage", 166 [BPF_MAP_TYPE_QUEUE] = "queue", 167 [BPF_MAP_TYPE_STACK] = "stack", 168 [BPF_MAP_TYPE_SK_STORAGE] = "sk_storage", 169 [BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops", 170 [BPF_MAP_TYPE_RINGBUF] = "ringbuf", 171 [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage", 172 [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage", 173 [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter", 174 [BPF_MAP_TYPE_USER_RINGBUF] = "user_ringbuf", 175 [BPF_MAP_TYPE_CGRP_STORAGE] = "cgrp_storage", 176 }; 177 178 static const char * const prog_type_name[] = { 179 [BPF_PROG_TYPE_UNSPEC] = "unspec", 180 [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", 181 [BPF_PROG_TYPE_KPROBE] = "kprobe", 182 [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls", 183 [BPF_PROG_TYPE_SCHED_ACT] = "sched_act", 184 [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint", 185 [BPF_PROG_TYPE_XDP] = "xdp", 186 [BPF_PROG_TYPE_PERF_EVENT] = "perf_event", 187 [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb", 188 [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock", 189 [BPF_PROG_TYPE_LWT_IN] = "lwt_in", 190 [BPF_PROG_TYPE_LWT_OUT] = "lwt_out", 191 [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", 192 [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", 193 [BPF_PROG_TYPE_SK_SKB] = "sk_skb", 194 [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device", 195 [BPF_PROG_TYPE_SK_MSG] = "sk_msg", 196 [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", 197 [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr", 198 [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local", 199 [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2", 200 [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport", 201 [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector", 202 [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl", 203 [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable", 204 [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt", 205 [BPF_PROG_TYPE_TRACING] = "tracing", 206 [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops", 207 [BPF_PROG_TYPE_EXT] = "ext", 208 [BPF_PROG_TYPE_LSM] = "lsm", 209 [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup", 210 [BPF_PROG_TYPE_SYSCALL] = "syscall", 211 [BPF_PROG_TYPE_NETFILTER] = "netfilter", 212 }; 213 214 static int __base_pr(enum libbpf_print_level level, const char *format, 215 va_list args) 216 { 217 if (level == LIBBPF_DEBUG) 218 return 0; 219 220 return vfprintf(stderr, format, args); 221 } 222 223 static libbpf_print_fn_t __libbpf_pr = __base_pr; 224 225 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) 226 { 227 libbpf_print_fn_t old_print_fn; 228 229 old_print_fn = __atomic_exchange_n(&__libbpf_pr, fn, __ATOMIC_RELAXED); 230 231 return old_print_fn; 232 } 233 234 __printf(2, 3) 235 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 236 { 237 va_list args; 238 int old_errno; 239 libbpf_print_fn_t print_fn; 240 241 print_fn = __atomic_load_n(&__libbpf_pr, __ATOMIC_RELAXED); 242 if (!print_fn) 243 return; 244 245 old_errno = errno; 246 247 va_start(args, format); 248 __libbpf_pr(level, format, args); 249 va_end(args); 250 251 errno = old_errno; 252 } 253 254 static void pr_perm_msg(int err) 255 { 256 struct rlimit limit; 257 char buf[100]; 258 259 if (err != -EPERM || geteuid() != 0) 260 return; 261 262 err = getrlimit(RLIMIT_MEMLOCK, &limit); 263 if (err) 264 return; 265 266 if (limit.rlim_cur == RLIM_INFINITY) 267 return; 268 269 if (limit.rlim_cur < 1024) 270 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); 271 else if (limit.rlim_cur < 1024*1024) 272 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); 273 else 274 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); 275 276 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", 277 buf); 278 } 279 280 #define STRERR_BUFSIZE 128 281 282 /* Copied from tools/perf/util/util.h */ 283 #ifndef zfree 284 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 285 #endif 286 287 #ifndef zclose 288 # define zclose(fd) ({ \ 289 int ___err = 0; \ 290 if ((fd) >= 0) \ 291 ___err = close((fd)); \ 292 fd = -1; \ 293 ___err; }) 294 #endif 295 296 static inline __u64 ptr_to_u64(const void *ptr) 297 { 298 return (__u64) (unsigned long) ptr; 299 } 300 301 int libbpf_set_strict_mode(enum libbpf_strict_mode mode) 302 { 303 /* as of v1.0 libbpf_set_strict_mode() is a no-op */ 304 return 0; 305 } 306 307 __u32 libbpf_major_version(void) 308 { 309 return LIBBPF_MAJOR_VERSION; 310 } 311 312 __u32 libbpf_minor_version(void) 313 { 314 return LIBBPF_MINOR_VERSION; 315 } 316 317 const char *libbpf_version_string(void) 318 { 319 #define __S(X) #X 320 #define _S(X) __S(X) 321 return "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION); 322 #undef _S 323 #undef __S 324 } 325 326 enum reloc_type { 327 RELO_LD64, 328 RELO_CALL, 329 RELO_DATA, 330 RELO_EXTERN_LD64, 331 RELO_EXTERN_CALL, 332 RELO_SUBPROG_ADDR, 333 RELO_CORE, 334 }; 335 336 struct reloc_desc { 337 enum reloc_type type; 338 int insn_idx; 339 union { 340 const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */ 341 struct { 342 int map_idx; 343 int sym_off; 344 int ext_idx; 345 }; 346 }; 347 }; 348 349 /* stored as sec_def->cookie for all libbpf-supported SEC()s */ 350 enum sec_def_flags { 351 SEC_NONE = 0, 352 /* expected_attach_type is optional, if kernel doesn't support that */ 353 SEC_EXP_ATTACH_OPT = 1, 354 /* legacy, only used by libbpf_get_type_names() and 355 * libbpf_attach_type_by_name(), not used by libbpf itself at all. 356 * This used to be associated with cgroup (and few other) BPF programs 357 * that were attachable through BPF_PROG_ATTACH command. Pretty 358 * meaningless nowadays, though. 359 */ 360 SEC_ATTACHABLE = 2, 361 SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT, 362 /* attachment target is specified through BTF ID in either kernel or 363 * other BPF program's BTF object 364 */ 365 SEC_ATTACH_BTF = 4, 366 /* BPF program type allows sleeping/blocking in kernel */ 367 SEC_SLEEPABLE = 8, 368 /* BPF program support non-linear XDP buffer */ 369 SEC_XDP_FRAGS = 16, 370 /* Setup proper attach type for usdt probes. */ 371 SEC_USDT = 32, 372 }; 373 374 struct bpf_sec_def { 375 char *sec; 376 enum bpf_prog_type prog_type; 377 enum bpf_attach_type expected_attach_type; 378 long cookie; 379 int handler_id; 380 381 libbpf_prog_setup_fn_t prog_setup_fn; 382 libbpf_prog_prepare_load_fn_t prog_prepare_load_fn; 383 libbpf_prog_attach_fn_t prog_attach_fn; 384 }; 385 386 /* 387 * bpf_prog should be a better name but it has been used in 388 * linux/filter.h. 389 */ 390 struct bpf_program { 391 char *name; 392 char *sec_name; 393 size_t sec_idx; 394 const struct bpf_sec_def *sec_def; 395 /* this program's instruction offset (in number of instructions) 396 * within its containing ELF section 397 */ 398 size_t sec_insn_off; 399 /* number of original instructions in ELF section belonging to this 400 * program, not taking into account subprogram instructions possible 401 * appended later during relocation 402 */ 403 size_t sec_insn_cnt; 404 /* Offset (in number of instructions) of the start of instruction 405 * belonging to this BPF program within its containing main BPF 406 * program. For the entry-point (main) BPF program, this is always 407 * zero. For a sub-program, this gets reset before each of main BPF 408 * programs are processed and relocated and is used to determined 409 * whether sub-program was already appended to the main program, and 410 * if yes, at which instruction offset. 411 */ 412 size_t sub_insn_off; 413 414 /* instructions that belong to BPF program; insns[0] is located at 415 * sec_insn_off instruction within its ELF section in ELF file, so 416 * when mapping ELF file instruction index to the local instruction, 417 * one needs to subtract sec_insn_off; and vice versa. 418 */ 419 struct bpf_insn *insns; 420 /* actual number of instruction in this BPF program's image; for 421 * entry-point BPF programs this includes the size of main program 422 * itself plus all the used sub-programs, appended at the end 423 */ 424 size_t insns_cnt; 425 426 struct reloc_desc *reloc_desc; 427 int nr_reloc; 428 429 /* BPF verifier log settings */ 430 char *log_buf; 431 size_t log_size; 432 __u32 log_level; 433 434 struct bpf_object *obj; 435 436 int fd; 437 bool autoload; 438 bool autoattach; 439 bool mark_btf_static; 440 enum bpf_prog_type type; 441 enum bpf_attach_type expected_attach_type; 442 443 int prog_ifindex; 444 __u32 attach_btf_obj_fd; 445 __u32 attach_btf_id; 446 __u32 attach_prog_fd; 447 448 void *func_info; 449 __u32 func_info_rec_size; 450 __u32 func_info_cnt; 451 452 void *line_info; 453 __u32 line_info_rec_size; 454 __u32 line_info_cnt; 455 __u32 prog_flags; 456 }; 457 458 struct bpf_struct_ops { 459 const char *tname; 460 const struct btf_type *type; 461 struct bpf_program **progs; 462 __u32 *kern_func_off; 463 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ 464 void *data; 465 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in 466 * btf_vmlinux's format. 467 * struct bpf_struct_ops_tcp_congestion_ops { 468 * [... some other kernel fields ...] 469 * struct tcp_congestion_ops data; 470 * } 471 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) 472 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" 473 * from "data". 474 */ 475 void *kern_vdata; 476 __u32 type_id; 477 }; 478 479 #define DATA_SEC ".data" 480 #define BSS_SEC ".bss" 481 #define RODATA_SEC ".rodata" 482 #define KCONFIG_SEC ".kconfig" 483 #define KSYMS_SEC ".ksyms" 484 #define STRUCT_OPS_SEC ".struct_ops" 485 #define STRUCT_OPS_LINK_SEC ".struct_ops.link" 486 487 enum libbpf_map_type { 488 LIBBPF_MAP_UNSPEC, 489 LIBBPF_MAP_DATA, 490 LIBBPF_MAP_BSS, 491 LIBBPF_MAP_RODATA, 492 LIBBPF_MAP_KCONFIG, 493 }; 494 495 struct bpf_map_def { 496 unsigned int type; 497 unsigned int key_size; 498 unsigned int value_size; 499 unsigned int max_entries; 500 unsigned int map_flags; 501 }; 502 503 struct bpf_map { 504 struct bpf_object *obj; 505 char *name; 506 /* real_name is defined for special internal maps (.rodata*, 507 * .data*, .bss, .kconfig) and preserves their original ELF section 508 * name. This is important to be able to find corresponding BTF 509 * DATASEC information. 510 */ 511 char *real_name; 512 int fd; 513 int sec_idx; 514 size_t sec_offset; 515 int map_ifindex; 516 int inner_map_fd; 517 struct bpf_map_def def; 518 __u32 numa_node; 519 __u32 btf_var_idx; 520 __u32 btf_key_type_id; 521 __u32 btf_value_type_id; 522 __u32 btf_vmlinux_value_type_id; 523 enum libbpf_map_type libbpf_type; 524 void *mmaped; 525 struct bpf_struct_ops *st_ops; 526 struct bpf_map *inner_map; 527 void **init_slots; 528 int init_slots_sz; 529 char *pin_path; 530 bool pinned; 531 bool reused; 532 bool autocreate; 533 __u64 map_extra; 534 }; 535 536 enum extern_type { 537 EXT_UNKNOWN, 538 EXT_KCFG, 539 EXT_KSYM, 540 }; 541 542 enum kcfg_type { 543 KCFG_UNKNOWN, 544 KCFG_CHAR, 545 KCFG_BOOL, 546 KCFG_INT, 547 KCFG_TRISTATE, 548 KCFG_CHAR_ARR, 549 }; 550 551 struct extern_desc { 552 enum extern_type type; 553 int sym_idx; 554 int btf_id; 555 int sec_btf_id; 556 const char *name; 557 char *essent_name; 558 bool is_set; 559 bool is_weak; 560 union { 561 struct { 562 enum kcfg_type type; 563 int sz; 564 int align; 565 int data_off; 566 bool is_signed; 567 } kcfg; 568 struct { 569 unsigned long long addr; 570 571 /* target btf_id of the corresponding kernel var. */ 572 int kernel_btf_obj_fd; 573 int kernel_btf_id; 574 575 /* local btf_id of the ksym extern's type. */ 576 __u32 type_id; 577 /* BTF fd index to be patched in for insn->off, this is 578 * 0 for vmlinux BTF, index in obj->fd_array for module 579 * BTF 580 */ 581 __s16 btf_fd_idx; 582 } ksym; 583 }; 584 }; 585 586 struct module_btf { 587 struct btf *btf; 588 char *name; 589 __u32 id; 590 int fd; 591 int fd_array_idx; 592 }; 593 594 enum sec_type { 595 SEC_UNUSED = 0, 596 SEC_RELO, 597 SEC_BSS, 598 SEC_DATA, 599 SEC_RODATA, 600 }; 601 602 struct elf_sec_desc { 603 enum sec_type sec_type; 604 Elf64_Shdr *shdr; 605 Elf_Data *data; 606 }; 607 608 struct elf_state { 609 int fd; 610 const void *obj_buf; 611 size_t obj_buf_sz; 612 Elf *elf; 613 Elf64_Ehdr *ehdr; 614 Elf_Data *symbols; 615 Elf_Data *st_ops_data; 616 Elf_Data *st_ops_link_data; 617 size_t shstrndx; /* section index for section name strings */ 618 size_t strtabidx; 619 struct elf_sec_desc *secs; 620 size_t sec_cnt; 621 int btf_maps_shndx; 622 __u32 btf_maps_sec_btf_id; 623 int text_shndx; 624 int symbols_shndx; 625 int st_ops_shndx; 626 int st_ops_link_shndx; 627 }; 628 629 struct usdt_manager; 630 631 struct bpf_object { 632 char name[BPF_OBJ_NAME_LEN]; 633 char license[64]; 634 __u32 kern_version; 635 636 struct bpf_program *programs; 637 size_t nr_programs; 638 struct bpf_map *maps; 639 size_t nr_maps; 640 size_t maps_cap; 641 642 char *kconfig; 643 struct extern_desc *externs; 644 int nr_extern; 645 int kconfig_map_idx; 646 647 bool loaded; 648 bool has_subcalls; 649 bool has_rodata; 650 651 struct bpf_gen *gen_loader; 652 653 /* Information when doing ELF related work. Only valid if efile.elf is not NULL */ 654 struct elf_state efile; 655 656 struct btf *btf; 657 struct btf_ext *btf_ext; 658 659 /* Parse and load BTF vmlinux if any of the programs in the object need 660 * it at load time. 661 */ 662 struct btf *btf_vmlinux; 663 /* Path to the custom BTF to be used for BPF CO-RE relocations as an 664 * override for vmlinux BTF. 665 */ 666 char *btf_custom_path; 667 /* vmlinux BTF override for CO-RE relocations */ 668 struct btf *btf_vmlinux_override; 669 /* Lazily initialized kernel module BTFs */ 670 struct module_btf *btf_modules; 671 bool btf_modules_loaded; 672 size_t btf_module_cnt; 673 size_t btf_module_cap; 674 675 /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */ 676 char *log_buf; 677 size_t log_size; 678 __u32 log_level; 679 680 int *fd_array; 681 size_t fd_array_cap; 682 size_t fd_array_cnt; 683 684 struct usdt_manager *usdt_man; 685 686 char path[]; 687 }; 688 689 static const char *elf_sym_str(const struct bpf_object *obj, size_t off); 690 static const char *elf_sec_str(const struct bpf_object *obj, size_t off); 691 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx); 692 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name); 693 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn); 694 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn); 695 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn); 696 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx); 697 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx); 698 699 void bpf_program__unload(struct bpf_program *prog) 700 { 701 if (!prog) 702 return; 703 704 zclose(prog->fd); 705 706 zfree(&prog->func_info); 707 zfree(&prog->line_info); 708 } 709 710 static void bpf_program__exit(struct bpf_program *prog) 711 { 712 if (!prog) 713 return; 714 715 bpf_program__unload(prog); 716 zfree(&prog->name); 717 zfree(&prog->sec_name); 718 zfree(&prog->insns); 719 zfree(&prog->reloc_desc); 720 721 prog->nr_reloc = 0; 722 prog->insns_cnt = 0; 723 prog->sec_idx = -1; 724 } 725 726 static bool insn_is_subprog_call(const struct bpf_insn *insn) 727 { 728 return BPF_CLASS(insn->code) == BPF_JMP && 729 BPF_OP(insn->code) == BPF_CALL && 730 BPF_SRC(insn->code) == BPF_K && 731 insn->src_reg == BPF_PSEUDO_CALL && 732 insn->dst_reg == 0 && 733 insn->off == 0; 734 } 735 736 static bool is_call_insn(const struct bpf_insn *insn) 737 { 738 return insn->code == (BPF_JMP | BPF_CALL); 739 } 740 741 static bool insn_is_pseudo_func(struct bpf_insn *insn) 742 { 743 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; 744 } 745 746 static int 747 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, 748 const char *name, size_t sec_idx, const char *sec_name, 749 size_t sec_off, void *insn_data, size_t insn_data_sz) 750 { 751 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) { 752 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n", 753 sec_name, name, sec_off, insn_data_sz); 754 return -EINVAL; 755 } 756 757 memset(prog, 0, sizeof(*prog)); 758 prog->obj = obj; 759 760 prog->sec_idx = sec_idx; 761 prog->sec_insn_off = sec_off / BPF_INSN_SZ; 762 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; 763 /* insns_cnt can later be increased by appending used subprograms */ 764 prog->insns_cnt = prog->sec_insn_cnt; 765 766 prog->type = BPF_PROG_TYPE_UNSPEC; 767 prog->fd = -1; 768 769 /* libbpf's convention for SEC("?abc...") is that it's just like 770 * SEC("abc...") but the corresponding bpf_program starts out with 771 * autoload set to false. 772 */ 773 if (sec_name[0] == '?') { 774 prog->autoload = false; 775 /* from now on forget there was ? in section name */ 776 sec_name++; 777 } else { 778 prog->autoload = true; 779 } 780 781 prog->autoattach = true; 782 783 /* inherit object's log_level */ 784 prog->log_level = obj->log_level; 785 786 prog->sec_name = strdup(sec_name); 787 if (!prog->sec_name) 788 goto errout; 789 790 prog->name = strdup(name); 791 if (!prog->name) 792 goto errout; 793 794 prog->insns = malloc(insn_data_sz); 795 if (!prog->insns) 796 goto errout; 797 memcpy(prog->insns, insn_data, insn_data_sz); 798 799 return 0; 800 errout: 801 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name); 802 bpf_program__exit(prog); 803 return -ENOMEM; 804 } 805 806 static int 807 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, 808 const char *sec_name, int sec_idx) 809 { 810 Elf_Data *symbols = obj->efile.symbols; 811 struct bpf_program *prog, *progs; 812 void *data = sec_data->d_buf; 813 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; 814 int nr_progs, err, i; 815 const char *name; 816 Elf64_Sym *sym; 817 818 progs = obj->programs; 819 nr_progs = obj->nr_programs; 820 nr_syms = symbols->d_size / sizeof(Elf64_Sym); 821 822 for (i = 0; i < nr_syms; i++) { 823 sym = elf_sym_by_idx(obj, i); 824 825 if (sym->st_shndx != sec_idx) 826 continue; 827 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) 828 continue; 829 830 prog_sz = sym->st_size; 831 sec_off = sym->st_value; 832 833 name = elf_sym_str(obj, sym->st_name); 834 if (!name) { 835 pr_warn("sec '%s': failed to get symbol name for offset %zu\n", 836 sec_name, sec_off); 837 return -LIBBPF_ERRNO__FORMAT; 838 } 839 840 if (sec_off + prog_sz > sec_sz) { 841 pr_warn("sec '%s': program at offset %zu crosses section boundary\n", 842 sec_name, sec_off); 843 return -LIBBPF_ERRNO__FORMAT; 844 } 845 846 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { 847 pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name); 848 return -ENOTSUP; 849 } 850 851 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n", 852 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz); 853 854 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs)); 855 if (!progs) { 856 /* 857 * In this case the original obj->programs 858 * is still valid, so don't need special treat for 859 * bpf_close_object(). 860 */ 861 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n", 862 sec_name, name); 863 return -ENOMEM; 864 } 865 obj->programs = progs; 866 867 prog = &progs[nr_progs]; 868 869 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, 870 sec_off, data + sec_off, prog_sz); 871 if (err) 872 return err; 873 874 /* if function is a global/weak symbol, but has restricted 875 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC 876 * as static to enable more permissive BPF verification mode 877 * with more outside context available to BPF verifier 878 */ 879 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL 880 && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN 881 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) 882 prog->mark_btf_static = true; 883 884 nr_progs++; 885 obj->nr_programs = nr_progs; 886 } 887 888 return 0; 889 } 890 891 static const struct btf_member * 892 find_member_by_offset(const struct btf_type *t, __u32 bit_offset) 893 { 894 struct btf_member *m; 895 int i; 896 897 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 898 if (btf_member_bit_offset(t, i) == bit_offset) 899 return m; 900 } 901 902 return NULL; 903 } 904 905 static const struct btf_member * 906 find_member_by_name(const struct btf *btf, const struct btf_type *t, 907 const char *name) 908 { 909 struct btf_member *m; 910 int i; 911 912 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 913 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) 914 return m; 915 } 916 917 return NULL; 918 } 919 920 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" 921 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 922 const char *name, __u32 kind); 923 924 static int 925 find_struct_ops_kern_types(const struct btf *btf, const char *tname, 926 const struct btf_type **type, __u32 *type_id, 927 const struct btf_type **vtype, __u32 *vtype_id, 928 const struct btf_member **data_member) 929 { 930 const struct btf_type *kern_type, *kern_vtype; 931 const struct btf_member *kern_data_member; 932 __s32 kern_vtype_id, kern_type_id; 933 __u32 i; 934 935 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT); 936 if (kern_type_id < 0) { 937 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", 938 tname); 939 return kern_type_id; 940 } 941 kern_type = btf__type_by_id(btf, kern_type_id); 942 943 /* Find the corresponding "map_value" type that will be used 944 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example, 945 * find "struct bpf_struct_ops_tcp_congestion_ops" from the 946 * btf_vmlinux. 947 */ 948 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX, 949 tname, BTF_KIND_STRUCT); 950 if (kern_vtype_id < 0) { 951 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n", 952 STRUCT_OPS_VALUE_PREFIX, tname); 953 return kern_vtype_id; 954 } 955 kern_vtype = btf__type_by_id(btf, kern_vtype_id); 956 957 /* Find "struct tcp_congestion_ops" from 958 * struct bpf_struct_ops_tcp_congestion_ops { 959 * [ ... ] 960 * struct tcp_congestion_ops data; 961 * } 962 */ 963 kern_data_member = btf_members(kern_vtype); 964 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { 965 if (kern_data_member->type == kern_type_id) 966 break; 967 } 968 if (i == btf_vlen(kern_vtype)) { 969 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n", 970 tname, STRUCT_OPS_VALUE_PREFIX, tname); 971 return -EINVAL; 972 } 973 974 *type = kern_type; 975 *type_id = kern_type_id; 976 *vtype = kern_vtype; 977 *vtype_id = kern_vtype_id; 978 *data_member = kern_data_member; 979 980 return 0; 981 } 982 983 static bool bpf_map__is_struct_ops(const struct bpf_map *map) 984 { 985 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; 986 } 987 988 /* Init the map's fields that depend on kern_btf */ 989 static int bpf_map__init_kern_struct_ops(struct bpf_map *map, 990 const struct btf *btf, 991 const struct btf *kern_btf) 992 { 993 const struct btf_member *member, *kern_member, *kern_data_member; 994 const struct btf_type *type, *kern_type, *kern_vtype; 995 __u32 i, kern_type_id, kern_vtype_id, kern_data_off; 996 struct bpf_struct_ops *st_ops; 997 void *data, *kern_data; 998 const char *tname; 999 int err; 1000 1001 st_ops = map->st_ops; 1002 type = st_ops->type; 1003 tname = st_ops->tname; 1004 err = find_struct_ops_kern_types(kern_btf, tname, 1005 &kern_type, &kern_type_id, 1006 &kern_vtype, &kern_vtype_id, 1007 &kern_data_member); 1008 if (err) 1009 return err; 1010 1011 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", 1012 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); 1013 1014 map->def.value_size = kern_vtype->size; 1015 map->btf_vmlinux_value_type_id = kern_vtype_id; 1016 1017 st_ops->kern_vdata = calloc(1, kern_vtype->size); 1018 if (!st_ops->kern_vdata) 1019 return -ENOMEM; 1020 1021 data = st_ops->data; 1022 kern_data_off = kern_data_member->offset / 8; 1023 kern_data = st_ops->kern_vdata + kern_data_off; 1024 1025 member = btf_members(type); 1026 for (i = 0; i < btf_vlen(type); i++, member++) { 1027 const struct btf_type *mtype, *kern_mtype; 1028 __u32 mtype_id, kern_mtype_id; 1029 void *mdata, *kern_mdata; 1030 __s64 msize, kern_msize; 1031 __u32 moff, kern_moff; 1032 __u32 kern_member_idx; 1033 const char *mname; 1034 1035 mname = btf__name_by_offset(btf, member->name_off); 1036 kern_member = find_member_by_name(kern_btf, kern_type, mname); 1037 if (!kern_member) { 1038 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", 1039 map->name, mname); 1040 return -ENOTSUP; 1041 } 1042 1043 kern_member_idx = kern_member - btf_members(kern_type); 1044 if (btf_member_bitfield_size(type, i) || 1045 btf_member_bitfield_size(kern_type, kern_member_idx)) { 1046 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", 1047 map->name, mname); 1048 return -ENOTSUP; 1049 } 1050 1051 moff = member->offset / 8; 1052 kern_moff = kern_member->offset / 8; 1053 1054 mdata = data + moff; 1055 kern_mdata = kern_data + kern_moff; 1056 1057 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); 1058 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, 1059 &kern_mtype_id); 1060 if (BTF_INFO_KIND(mtype->info) != 1061 BTF_INFO_KIND(kern_mtype->info)) { 1062 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", 1063 map->name, mname, BTF_INFO_KIND(mtype->info), 1064 BTF_INFO_KIND(kern_mtype->info)); 1065 return -ENOTSUP; 1066 } 1067 1068 if (btf_is_ptr(mtype)) { 1069 struct bpf_program *prog; 1070 1071 prog = st_ops->progs[i]; 1072 if (!prog) 1073 continue; 1074 1075 kern_mtype = skip_mods_and_typedefs(kern_btf, 1076 kern_mtype->type, 1077 &kern_mtype_id); 1078 1079 /* mtype->type must be a func_proto which was 1080 * guaranteed in bpf_object__collect_st_ops_relos(), 1081 * so only check kern_mtype for func_proto here. 1082 */ 1083 if (!btf_is_func_proto(kern_mtype)) { 1084 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n", 1085 map->name, mname); 1086 return -ENOTSUP; 1087 } 1088 1089 prog->attach_btf_id = kern_type_id; 1090 prog->expected_attach_type = kern_member_idx; 1091 1092 st_ops->kern_func_off[i] = kern_data_off + kern_moff; 1093 1094 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", 1095 map->name, mname, prog->name, moff, 1096 kern_moff); 1097 1098 continue; 1099 } 1100 1101 msize = btf__resolve_size(btf, mtype_id); 1102 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); 1103 if (msize < 0 || kern_msize < 0 || msize != kern_msize) { 1104 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", 1105 map->name, mname, (ssize_t)msize, 1106 (ssize_t)kern_msize); 1107 return -ENOTSUP; 1108 } 1109 1110 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", 1111 map->name, mname, (unsigned int)msize, 1112 moff, kern_moff); 1113 memcpy(kern_mdata, mdata, msize); 1114 } 1115 1116 return 0; 1117 } 1118 1119 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) 1120 { 1121 struct bpf_map *map; 1122 size_t i; 1123 int err; 1124 1125 for (i = 0; i < obj->nr_maps; i++) { 1126 map = &obj->maps[i]; 1127 1128 if (!bpf_map__is_struct_ops(map)) 1129 continue; 1130 1131 err = bpf_map__init_kern_struct_ops(map, obj->btf, 1132 obj->btf_vmlinux); 1133 if (err) 1134 return err; 1135 } 1136 1137 return 0; 1138 } 1139 1140 static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name, 1141 int shndx, Elf_Data *data, __u32 map_flags) 1142 { 1143 const struct btf_type *type, *datasec; 1144 const struct btf_var_secinfo *vsi; 1145 struct bpf_struct_ops *st_ops; 1146 const char *tname, *var_name; 1147 __s32 type_id, datasec_id; 1148 const struct btf *btf; 1149 struct bpf_map *map; 1150 __u32 i; 1151 1152 if (shndx == -1) 1153 return 0; 1154 1155 btf = obj->btf; 1156 datasec_id = btf__find_by_name_kind(btf, sec_name, 1157 BTF_KIND_DATASEC); 1158 if (datasec_id < 0) { 1159 pr_warn("struct_ops init: DATASEC %s not found\n", 1160 sec_name); 1161 return -EINVAL; 1162 } 1163 1164 datasec = btf__type_by_id(btf, datasec_id); 1165 vsi = btf_var_secinfos(datasec); 1166 for (i = 0; i < btf_vlen(datasec); i++, vsi++) { 1167 type = btf__type_by_id(obj->btf, vsi->type); 1168 var_name = btf__name_by_offset(obj->btf, type->name_off); 1169 1170 type_id = btf__resolve_type(obj->btf, vsi->type); 1171 if (type_id < 0) { 1172 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", 1173 vsi->type, sec_name); 1174 return -EINVAL; 1175 } 1176 1177 type = btf__type_by_id(obj->btf, type_id); 1178 tname = btf__name_by_offset(obj->btf, type->name_off); 1179 if (!tname[0]) { 1180 pr_warn("struct_ops init: anonymous type is not supported\n"); 1181 return -ENOTSUP; 1182 } 1183 if (!btf_is_struct(type)) { 1184 pr_warn("struct_ops init: %s is not a struct\n", tname); 1185 return -EINVAL; 1186 } 1187 1188 map = bpf_object__add_map(obj); 1189 if (IS_ERR(map)) 1190 return PTR_ERR(map); 1191 1192 map->sec_idx = shndx; 1193 map->sec_offset = vsi->offset; 1194 map->name = strdup(var_name); 1195 if (!map->name) 1196 return -ENOMEM; 1197 1198 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; 1199 map->def.key_size = sizeof(int); 1200 map->def.value_size = type->size; 1201 map->def.max_entries = 1; 1202 map->def.map_flags = map_flags; 1203 1204 map->st_ops = calloc(1, sizeof(*map->st_ops)); 1205 if (!map->st_ops) 1206 return -ENOMEM; 1207 st_ops = map->st_ops; 1208 st_ops->data = malloc(type->size); 1209 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); 1210 st_ops->kern_func_off = malloc(btf_vlen(type) * 1211 sizeof(*st_ops->kern_func_off)); 1212 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) 1213 return -ENOMEM; 1214 1215 if (vsi->offset + type->size > data->d_size) { 1216 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", 1217 var_name, sec_name); 1218 return -EINVAL; 1219 } 1220 1221 memcpy(st_ops->data, 1222 data->d_buf + vsi->offset, 1223 type->size); 1224 st_ops->tname = tname; 1225 st_ops->type = type; 1226 st_ops->type_id = type_id; 1227 1228 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", 1229 tname, type_id, var_name, vsi->offset); 1230 } 1231 1232 return 0; 1233 } 1234 1235 static int bpf_object_init_struct_ops(struct bpf_object *obj) 1236 { 1237 int err; 1238 1239 err = init_struct_ops_maps(obj, STRUCT_OPS_SEC, obj->efile.st_ops_shndx, 1240 obj->efile.st_ops_data, 0); 1241 err = err ?: init_struct_ops_maps(obj, STRUCT_OPS_LINK_SEC, 1242 obj->efile.st_ops_link_shndx, 1243 obj->efile.st_ops_link_data, 1244 BPF_F_LINK); 1245 return err; 1246 } 1247 1248 static struct bpf_object *bpf_object__new(const char *path, 1249 const void *obj_buf, 1250 size_t obj_buf_sz, 1251 const char *obj_name) 1252 { 1253 struct bpf_object *obj; 1254 char *end; 1255 1256 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 1257 if (!obj) { 1258 pr_warn("alloc memory failed for %s\n", path); 1259 return ERR_PTR(-ENOMEM); 1260 } 1261 1262 strcpy(obj->path, path); 1263 if (obj_name) { 1264 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); 1265 } else { 1266 /* Using basename() GNU version which doesn't modify arg. */ 1267 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); 1268 end = strchr(obj->name, '.'); 1269 if (end) 1270 *end = 0; 1271 } 1272 1273 obj->efile.fd = -1; 1274 /* 1275 * Caller of this function should also call 1276 * bpf_object__elf_finish() after data collection to return 1277 * obj_buf to user. If not, we should duplicate the buffer to 1278 * avoid user freeing them before elf finish. 1279 */ 1280 obj->efile.obj_buf = obj_buf; 1281 obj->efile.obj_buf_sz = obj_buf_sz; 1282 obj->efile.btf_maps_shndx = -1; 1283 obj->efile.st_ops_shndx = -1; 1284 obj->efile.st_ops_link_shndx = -1; 1285 obj->kconfig_map_idx = -1; 1286 1287 obj->kern_version = get_kernel_version(); 1288 obj->loaded = false; 1289 1290 return obj; 1291 } 1292 1293 static void bpf_object__elf_finish(struct bpf_object *obj) 1294 { 1295 if (!obj->efile.elf) 1296 return; 1297 1298 elf_end(obj->efile.elf); 1299 obj->efile.elf = NULL; 1300 obj->efile.symbols = NULL; 1301 obj->efile.st_ops_data = NULL; 1302 obj->efile.st_ops_link_data = NULL; 1303 1304 zfree(&obj->efile.secs); 1305 obj->efile.sec_cnt = 0; 1306 zclose(obj->efile.fd); 1307 obj->efile.obj_buf = NULL; 1308 obj->efile.obj_buf_sz = 0; 1309 } 1310 1311 static int bpf_object__elf_init(struct bpf_object *obj) 1312 { 1313 Elf64_Ehdr *ehdr; 1314 int err = 0; 1315 Elf *elf; 1316 1317 if (obj->efile.elf) { 1318 pr_warn("elf: init internal error\n"); 1319 return -LIBBPF_ERRNO__LIBELF; 1320 } 1321 1322 if (obj->efile.obj_buf_sz > 0) { 1323 /* obj_buf should have been validated by bpf_object__open_mem(). */ 1324 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz); 1325 } else { 1326 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC); 1327 if (obj->efile.fd < 0) { 1328 char errmsg[STRERR_BUFSIZE], *cp; 1329 1330 err = -errno; 1331 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 1332 pr_warn("elf: failed to open %s: %s\n", obj->path, cp); 1333 return err; 1334 } 1335 1336 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); 1337 } 1338 1339 if (!elf) { 1340 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); 1341 err = -LIBBPF_ERRNO__LIBELF; 1342 goto errout; 1343 } 1344 1345 obj->efile.elf = elf; 1346 1347 if (elf_kind(elf) != ELF_K_ELF) { 1348 err = -LIBBPF_ERRNO__FORMAT; 1349 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path); 1350 goto errout; 1351 } 1352 1353 if (gelf_getclass(elf) != ELFCLASS64) { 1354 err = -LIBBPF_ERRNO__FORMAT; 1355 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path); 1356 goto errout; 1357 } 1358 1359 obj->efile.ehdr = ehdr = elf64_getehdr(elf); 1360 if (!obj->efile.ehdr) { 1361 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); 1362 err = -LIBBPF_ERRNO__FORMAT; 1363 goto errout; 1364 } 1365 1366 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) { 1367 pr_warn("elf: failed to get section names section index for %s: %s\n", 1368 obj->path, elf_errmsg(-1)); 1369 err = -LIBBPF_ERRNO__FORMAT; 1370 goto errout; 1371 } 1372 1373 /* ELF is corrupted/truncated, avoid calling elf_strptr. */ 1374 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) { 1375 pr_warn("elf: failed to get section names strings from %s: %s\n", 1376 obj->path, elf_errmsg(-1)); 1377 err = -LIBBPF_ERRNO__FORMAT; 1378 goto errout; 1379 } 1380 1381 /* Old LLVM set e_machine to EM_NONE */ 1382 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) { 1383 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); 1384 err = -LIBBPF_ERRNO__FORMAT; 1385 goto errout; 1386 } 1387 1388 return 0; 1389 errout: 1390 bpf_object__elf_finish(obj); 1391 return err; 1392 } 1393 1394 static int bpf_object__check_endianness(struct bpf_object *obj) 1395 { 1396 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1397 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB) 1398 return 0; 1399 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 1400 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB) 1401 return 0; 1402 #else 1403 # error "Unrecognized __BYTE_ORDER__" 1404 #endif 1405 pr_warn("elf: endianness mismatch in %s.\n", obj->path); 1406 return -LIBBPF_ERRNO__ENDIAN; 1407 } 1408 1409 static int 1410 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 1411 { 1412 if (!data) { 1413 pr_warn("invalid license section in %s\n", obj->path); 1414 return -LIBBPF_ERRNO__FORMAT; 1415 } 1416 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't 1417 * go over allowed ELF data section buffer 1418 */ 1419 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license))); 1420 pr_debug("license of %s is %s\n", obj->path, obj->license); 1421 return 0; 1422 } 1423 1424 static int 1425 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 1426 { 1427 __u32 kver; 1428 1429 if (!data || size != sizeof(kver)) { 1430 pr_warn("invalid kver section in %s\n", obj->path); 1431 return -LIBBPF_ERRNO__FORMAT; 1432 } 1433 memcpy(&kver, data, sizeof(kver)); 1434 obj->kern_version = kver; 1435 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 1436 return 0; 1437 } 1438 1439 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 1440 { 1441 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 1442 type == BPF_MAP_TYPE_HASH_OF_MAPS) 1443 return true; 1444 return false; 1445 } 1446 1447 static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size) 1448 { 1449 Elf_Data *data; 1450 Elf_Scn *scn; 1451 1452 if (!name) 1453 return -EINVAL; 1454 1455 scn = elf_sec_by_name(obj, name); 1456 data = elf_sec_data(obj, scn); 1457 if (data) { 1458 *size = data->d_size; 1459 return 0; /* found it */ 1460 } 1461 1462 return -ENOENT; 1463 } 1464 1465 static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name) 1466 { 1467 Elf_Data *symbols = obj->efile.symbols; 1468 const char *sname; 1469 size_t si; 1470 1471 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) { 1472 Elf64_Sym *sym = elf_sym_by_idx(obj, si); 1473 1474 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT) 1475 continue; 1476 1477 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL && 1478 ELF64_ST_BIND(sym->st_info) != STB_WEAK) 1479 continue; 1480 1481 sname = elf_sym_str(obj, sym->st_name); 1482 if (!sname) { 1483 pr_warn("failed to get sym name string for var %s\n", name); 1484 return ERR_PTR(-EIO); 1485 } 1486 if (strcmp(name, sname) == 0) 1487 return sym; 1488 } 1489 1490 return ERR_PTR(-ENOENT); 1491 } 1492 1493 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 1494 { 1495 struct bpf_map *map; 1496 int err; 1497 1498 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap, 1499 sizeof(*obj->maps), obj->nr_maps + 1); 1500 if (err) 1501 return ERR_PTR(err); 1502 1503 map = &obj->maps[obj->nr_maps++]; 1504 map->obj = obj; 1505 map->fd = -1; 1506 map->inner_map_fd = -1; 1507 map->autocreate = true; 1508 1509 return map; 1510 } 1511 1512 static size_t bpf_map_mmap_sz(unsigned int value_sz, unsigned int max_entries) 1513 { 1514 const long page_sz = sysconf(_SC_PAGE_SIZE); 1515 size_t map_sz; 1516 1517 map_sz = (size_t)roundup(value_sz, 8) * max_entries; 1518 map_sz = roundup(map_sz, page_sz); 1519 return map_sz; 1520 } 1521 1522 static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz) 1523 { 1524 void *mmaped; 1525 1526 if (!map->mmaped) 1527 return -EINVAL; 1528 1529 if (old_sz == new_sz) 1530 return 0; 1531 1532 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1533 if (mmaped == MAP_FAILED) 1534 return -errno; 1535 1536 memcpy(mmaped, map->mmaped, min(old_sz, new_sz)); 1537 munmap(map->mmaped, old_sz); 1538 map->mmaped = mmaped; 1539 return 0; 1540 } 1541 1542 static char *internal_map_name(struct bpf_object *obj, const char *real_name) 1543 { 1544 char map_name[BPF_OBJ_NAME_LEN], *p; 1545 int pfx_len, sfx_len = max((size_t)7, strlen(real_name)); 1546 1547 /* This is one of the more confusing parts of libbpf for various 1548 * reasons, some of which are historical. The original idea for naming 1549 * internal names was to include as much of BPF object name prefix as 1550 * possible, so that it can be distinguished from similar internal 1551 * maps of a different BPF object. 1552 * As an example, let's say we have bpf_object named 'my_object_name' 1553 * and internal map corresponding to '.rodata' ELF section. The final 1554 * map name advertised to user and to the kernel will be 1555 * 'my_objec.rodata', taking first 8 characters of object name and 1556 * entire 7 characters of '.rodata'. 1557 * Somewhat confusingly, if internal map ELF section name is shorter 1558 * than 7 characters, e.g., '.bss', we still reserve 7 characters 1559 * for the suffix, even though we only have 4 actual characters, and 1560 * resulting map will be called 'my_objec.bss', not even using all 15 1561 * characters allowed by the kernel. Oh well, at least the truncated 1562 * object name is somewhat consistent in this case. But if the map 1563 * name is '.kconfig', we'll still have entirety of '.kconfig' added 1564 * (8 chars) and thus will be left with only first 7 characters of the 1565 * object name ('my_obje'). Happy guessing, user, that the final map 1566 * name will be "my_obje.kconfig". 1567 * Now, with libbpf starting to support arbitrarily named .rodata.* 1568 * and .data.* data sections, it's possible that ELF section name is 1569 * longer than allowed 15 chars, so we now need to be careful to take 1570 * only up to 15 first characters of ELF name, taking no BPF object 1571 * name characters at all. So '.rodata.abracadabra' will result in 1572 * '.rodata.abracad' kernel and user-visible name. 1573 * We need to keep this convoluted logic intact for .data, .bss and 1574 * .rodata maps, but for new custom .data.custom and .rodata.custom 1575 * maps we use their ELF names as is, not prepending bpf_object name 1576 * in front. We still need to truncate them to 15 characters for the 1577 * kernel. Full name can be recovered for such maps by using DATASEC 1578 * BTF type associated with such map's value type, though. 1579 */ 1580 if (sfx_len >= BPF_OBJ_NAME_LEN) 1581 sfx_len = BPF_OBJ_NAME_LEN - 1; 1582 1583 /* if there are two or more dots in map name, it's a custom dot map */ 1584 if (strchr(real_name + 1, '.') != NULL) 1585 pfx_len = 0; 1586 else 1587 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name)); 1588 1589 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, 1590 sfx_len, real_name); 1591 1592 /* sanitise map name to characters allowed by kernel */ 1593 for (p = map_name; *p && p < map_name + sizeof(map_name); p++) 1594 if (!isalnum(*p) && *p != '_' && *p != '.') 1595 *p = '_'; 1596 1597 return strdup(map_name); 1598 } 1599 1600 static int 1601 map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map); 1602 1603 /* Internal BPF map is mmap()'able only if at least one of corresponding 1604 * DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL 1605 * variable and it's not marked as __hidden (which turns it into, effectively, 1606 * a STATIC variable). 1607 */ 1608 static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map) 1609 { 1610 const struct btf_type *t, *vt; 1611 struct btf_var_secinfo *vsi; 1612 int i, n; 1613 1614 if (!map->btf_value_type_id) 1615 return false; 1616 1617 t = btf__type_by_id(obj->btf, map->btf_value_type_id); 1618 if (!btf_is_datasec(t)) 1619 return false; 1620 1621 vsi = btf_var_secinfos(t); 1622 for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) { 1623 vt = btf__type_by_id(obj->btf, vsi->type); 1624 if (!btf_is_var(vt)) 1625 continue; 1626 1627 if (btf_var(vt)->linkage != BTF_VAR_STATIC) 1628 return true; 1629 } 1630 1631 return false; 1632 } 1633 1634 static int 1635 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 1636 const char *real_name, int sec_idx, void *data, size_t data_sz) 1637 { 1638 struct bpf_map_def *def; 1639 struct bpf_map *map; 1640 size_t mmap_sz; 1641 int err; 1642 1643 map = bpf_object__add_map(obj); 1644 if (IS_ERR(map)) 1645 return PTR_ERR(map); 1646 1647 map->libbpf_type = type; 1648 map->sec_idx = sec_idx; 1649 map->sec_offset = 0; 1650 map->real_name = strdup(real_name); 1651 map->name = internal_map_name(obj, real_name); 1652 if (!map->real_name || !map->name) { 1653 zfree(&map->real_name); 1654 zfree(&map->name); 1655 return -ENOMEM; 1656 } 1657 1658 def = &map->def; 1659 def->type = BPF_MAP_TYPE_ARRAY; 1660 def->key_size = sizeof(int); 1661 def->value_size = data_sz; 1662 def->max_entries = 1; 1663 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG 1664 ? BPF_F_RDONLY_PROG : 0; 1665 1666 /* failures are fine because of maps like .rodata.str1.1 */ 1667 (void) map_fill_btf_type_info(obj, map); 1668 1669 if (map_is_mmapable(obj, map)) 1670 def->map_flags |= BPF_F_MMAPABLE; 1671 1672 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", 1673 map->name, map->sec_idx, map->sec_offset, def->map_flags); 1674 1675 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 1676 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 1677 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1678 if (map->mmaped == MAP_FAILED) { 1679 err = -errno; 1680 map->mmaped = NULL; 1681 pr_warn("failed to alloc map '%s' content buffer: %d\n", 1682 map->name, err); 1683 zfree(&map->real_name); 1684 zfree(&map->name); 1685 return err; 1686 } 1687 1688 if (data) 1689 memcpy(map->mmaped, data, data_sz); 1690 1691 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 1692 return 0; 1693 } 1694 1695 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 1696 { 1697 struct elf_sec_desc *sec_desc; 1698 const char *sec_name; 1699 int err = 0, sec_idx; 1700 1701 /* 1702 * Populate obj->maps with libbpf internal maps. 1703 */ 1704 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) { 1705 sec_desc = &obj->efile.secs[sec_idx]; 1706 1707 /* Skip recognized sections with size 0. */ 1708 if (!sec_desc->data || sec_desc->data->d_size == 0) 1709 continue; 1710 1711 switch (sec_desc->sec_type) { 1712 case SEC_DATA: 1713 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 1714 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 1715 sec_name, sec_idx, 1716 sec_desc->data->d_buf, 1717 sec_desc->data->d_size); 1718 break; 1719 case SEC_RODATA: 1720 obj->has_rodata = true; 1721 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 1722 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 1723 sec_name, sec_idx, 1724 sec_desc->data->d_buf, 1725 sec_desc->data->d_size); 1726 break; 1727 case SEC_BSS: 1728 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 1729 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 1730 sec_name, sec_idx, 1731 NULL, 1732 sec_desc->data->d_size); 1733 break; 1734 default: 1735 /* skip */ 1736 break; 1737 } 1738 if (err) 1739 return err; 1740 } 1741 return 0; 1742 } 1743 1744 1745 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, 1746 const void *name) 1747 { 1748 int i; 1749 1750 for (i = 0; i < obj->nr_extern; i++) { 1751 if (strcmp(obj->externs[i].name, name) == 0) 1752 return &obj->externs[i]; 1753 } 1754 return NULL; 1755 } 1756 1757 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, 1758 char value) 1759 { 1760 switch (ext->kcfg.type) { 1761 case KCFG_BOOL: 1762 if (value == 'm') { 1763 pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n", 1764 ext->name, value); 1765 return -EINVAL; 1766 } 1767 *(bool *)ext_val = value == 'y' ? true : false; 1768 break; 1769 case KCFG_TRISTATE: 1770 if (value == 'y') 1771 *(enum libbpf_tristate *)ext_val = TRI_YES; 1772 else if (value == 'm') 1773 *(enum libbpf_tristate *)ext_val = TRI_MODULE; 1774 else /* value == 'n' */ 1775 *(enum libbpf_tristate *)ext_val = TRI_NO; 1776 break; 1777 case KCFG_CHAR: 1778 *(char *)ext_val = value; 1779 break; 1780 case KCFG_UNKNOWN: 1781 case KCFG_INT: 1782 case KCFG_CHAR_ARR: 1783 default: 1784 pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n", 1785 ext->name, value); 1786 return -EINVAL; 1787 } 1788 ext->is_set = true; 1789 return 0; 1790 } 1791 1792 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, 1793 const char *value) 1794 { 1795 size_t len; 1796 1797 if (ext->kcfg.type != KCFG_CHAR_ARR) { 1798 pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n", 1799 ext->name, value); 1800 return -EINVAL; 1801 } 1802 1803 len = strlen(value); 1804 if (value[len - 1] != '"') { 1805 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n", 1806 ext->name, value); 1807 return -EINVAL; 1808 } 1809 1810 /* strip quotes */ 1811 len -= 2; 1812 if (len >= ext->kcfg.sz) { 1813 pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n", 1814 ext->name, value, len, ext->kcfg.sz - 1); 1815 len = ext->kcfg.sz - 1; 1816 } 1817 memcpy(ext_val, value + 1, len); 1818 ext_val[len] = '\0'; 1819 ext->is_set = true; 1820 return 0; 1821 } 1822 1823 static int parse_u64(const char *value, __u64 *res) 1824 { 1825 char *value_end; 1826 int err; 1827 1828 errno = 0; 1829 *res = strtoull(value, &value_end, 0); 1830 if (errno) { 1831 err = -errno; 1832 pr_warn("failed to parse '%s' as integer: %d\n", value, err); 1833 return err; 1834 } 1835 if (*value_end) { 1836 pr_warn("failed to parse '%s' as integer completely\n", value); 1837 return -EINVAL; 1838 } 1839 return 0; 1840 } 1841 1842 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) 1843 { 1844 int bit_sz = ext->kcfg.sz * 8; 1845 1846 if (ext->kcfg.sz == 8) 1847 return true; 1848 1849 /* Validate that value stored in u64 fits in integer of `ext->sz` 1850 * bytes size without any loss of information. If the target integer 1851 * is signed, we rely on the following limits of integer type of 1852 * Y bits and subsequent transformation: 1853 * 1854 * -2^(Y-1) <= X <= 2^(Y-1) - 1 1855 * 0 <= X + 2^(Y-1) <= 2^Y - 1 1856 * 0 <= X + 2^(Y-1) < 2^Y 1857 * 1858 * For unsigned target integer, check that all the (64 - Y) bits are 1859 * zero. 1860 */ 1861 if (ext->kcfg.is_signed) 1862 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); 1863 else 1864 return (v >> bit_sz) == 0; 1865 } 1866 1867 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, 1868 __u64 value) 1869 { 1870 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR && 1871 ext->kcfg.type != KCFG_BOOL) { 1872 pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n", 1873 ext->name, (unsigned long long)value); 1874 return -EINVAL; 1875 } 1876 if (ext->kcfg.type == KCFG_BOOL && value > 1) { 1877 pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n", 1878 ext->name, (unsigned long long)value); 1879 return -EINVAL; 1880 1881 } 1882 if (!is_kcfg_value_in_range(ext, value)) { 1883 pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n", 1884 ext->name, (unsigned long long)value, ext->kcfg.sz); 1885 return -ERANGE; 1886 } 1887 switch (ext->kcfg.sz) { 1888 case 1: 1889 *(__u8 *)ext_val = value; 1890 break; 1891 case 2: 1892 *(__u16 *)ext_val = value; 1893 break; 1894 case 4: 1895 *(__u32 *)ext_val = value; 1896 break; 1897 case 8: 1898 *(__u64 *)ext_val = value; 1899 break; 1900 default: 1901 return -EINVAL; 1902 } 1903 ext->is_set = true; 1904 return 0; 1905 } 1906 1907 static int bpf_object__process_kconfig_line(struct bpf_object *obj, 1908 char *buf, void *data) 1909 { 1910 struct extern_desc *ext; 1911 char *sep, *value; 1912 int len, err = 0; 1913 void *ext_val; 1914 __u64 num; 1915 1916 if (!str_has_pfx(buf, "CONFIG_")) 1917 return 0; 1918 1919 sep = strchr(buf, '='); 1920 if (!sep) { 1921 pr_warn("failed to parse '%s': no separator\n", buf); 1922 return -EINVAL; 1923 } 1924 1925 /* Trim ending '\n' */ 1926 len = strlen(buf); 1927 if (buf[len - 1] == '\n') 1928 buf[len - 1] = '\0'; 1929 /* Split on '=' and ensure that a value is present. */ 1930 *sep = '\0'; 1931 if (!sep[1]) { 1932 *sep = '='; 1933 pr_warn("failed to parse '%s': no value\n", buf); 1934 return -EINVAL; 1935 } 1936 1937 ext = find_extern_by_name(obj, buf); 1938 if (!ext || ext->is_set) 1939 return 0; 1940 1941 ext_val = data + ext->kcfg.data_off; 1942 value = sep + 1; 1943 1944 switch (*value) { 1945 case 'y': case 'n': case 'm': 1946 err = set_kcfg_value_tri(ext, ext_val, *value); 1947 break; 1948 case '"': 1949 err = set_kcfg_value_str(ext, ext_val, value); 1950 break; 1951 default: 1952 /* assume integer */ 1953 err = parse_u64(value, &num); 1954 if (err) { 1955 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value); 1956 return err; 1957 } 1958 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { 1959 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value); 1960 return -EINVAL; 1961 } 1962 err = set_kcfg_value_num(ext, ext_val, num); 1963 break; 1964 } 1965 if (err) 1966 return err; 1967 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value); 1968 return 0; 1969 } 1970 1971 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) 1972 { 1973 char buf[PATH_MAX]; 1974 struct utsname uts; 1975 int len, err = 0; 1976 gzFile file; 1977 1978 uname(&uts); 1979 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); 1980 if (len < 0) 1981 return -EINVAL; 1982 else if (len >= PATH_MAX) 1983 return -ENAMETOOLONG; 1984 1985 /* gzopen also accepts uncompressed files. */ 1986 file = gzopen(buf, "re"); 1987 if (!file) 1988 file = gzopen("/proc/config.gz", "re"); 1989 1990 if (!file) { 1991 pr_warn("failed to open system Kconfig\n"); 1992 return -ENOENT; 1993 } 1994 1995 while (gzgets(file, buf, sizeof(buf))) { 1996 err = bpf_object__process_kconfig_line(obj, buf, data); 1997 if (err) { 1998 pr_warn("error parsing system Kconfig line '%s': %d\n", 1999 buf, err); 2000 goto out; 2001 } 2002 } 2003 2004 out: 2005 gzclose(file); 2006 return err; 2007 } 2008 2009 static int bpf_object__read_kconfig_mem(struct bpf_object *obj, 2010 const char *config, void *data) 2011 { 2012 char buf[PATH_MAX]; 2013 int err = 0; 2014 FILE *file; 2015 2016 file = fmemopen((void *)config, strlen(config), "r"); 2017 if (!file) { 2018 err = -errno; 2019 pr_warn("failed to open in-memory Kconfig: %d\n", err); 2020 return err; 2021 } 2022 2023 while (fgets(buf, sizeof(buf), file)) { 2024 err = bpf_object__process_kconfig_line(obj, buf, data); 2025 if (err) { 2026 pr_warn("error parsing in-memory Kconfig line '%s': %d\n", 2027 buf, err); 2028 break; 2029 } 2030 } 2031 2032 fclose(file); 2033 return err; 2034 } 2035 2036 static int bpf_object__init_kconfig_map(struct bpf_object *obj) 2037 { 2038 struct extern_desc *last_ext = NULL, *ext; 2039 size_t map_sz; 2040 int i, err; 2041 2042 for (i = 0; i < obj->nr_extern; i++) { 2043 ext = &obj->externs[i]; 2044 if (ext->type == EXT_KCFG) 2045 last_ext = ext; 2046 } 2047 2048 if (!last_ext) 2049 return 0; 2050 2051 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; 2052 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, 2053 ".kconfig", obj->efile.symbols_shndx, 2054 NULL, map_sz); 2055 if (err) 2056 return err; 2057 2058 obj->kconfig_map_idx = obj->nr_maps - 1; 2059 2060 return 0; 2061 } 2062 2063 const struct btf_type * 2064 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) 2065 { 2066 const struct btf_type *t = btf__type_by_id(btf, id); 2067 2068 if (res_id) 2069 *res_id = id; 2070 2071 while (btf_is_mod(t) || btf_is_typedef(t)) { 2072 if (res_id) 2073 *res_id = t->type; 2074 t = btf__type_by_id(btf, t->type); 2075 } 2076 2077 return t; 2078 } 2079 2080 static const struct btf_type * 2081 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) 2082 { 2083 const struct btf_type *t; 2084 2085 t = skip_mods_and_typedefs(btf, id, NULL); 2086 if (!btf_is_ptr(t)) 2087 return NULL; 2088 2089 t = skip_mods_and_typedefs(btf, t->type, res_id); 2090 2091 return btf_is_func_proto(t) ? t : NULL; 2092 } 2093 2094 static const char *__btf_kind_str(__u16 kind) 2095 { 2096 switch (kind) { 2097 case BTF_KIND_UNKN: return "void"; 2098 case BTF_KIND_INT: return "int"; 2099 case BTF_KIND_PTR: return "ptr"; 2100 case BTF_KIND_ARRAY: return "array"; 2101 case BTF_KIND_STRUCT: return "struct"; 2102 case BTF_KIND_UNION: return "union"; 2103 case BTF_KIND_ENUM: return "enum"; 2104 case BTF_KIND_FWD: return "fwd"; 2105 case BTF_KIND_TYPEDEF: return "typedef"; 2106 case BTF_KIND_VOLATILE: return "volatile"; 2107 case BTF_KIND_CONST: return "const"; 2108 case BTF_KIND_RESTRICT: return "restrict"; 2109 case BTF_KIND_FUNC: return "func"; 2110 case BTF_KIND_FUNC_PROTO: return "func_proto"; 2111 case BTF_KIND_VAR: return "var"; 2112 case BTF_KIND_DATASEC: return "datasec"; 2113 case BTF_KIND_FLOAT: return "float"; 2114 case BTF_KIND_DECL_TAG: return "decl_tag"; 2115 case BTF_KIND_TYPE_TAG: return "type_tag"; 2116 case BTF_KIND_ENUM64: return "enum64"; 2117 default: return "unknown"; 2118 } 2119 } 2120 2121 const char *btf_kind_str(const struct btf_type *t) 2122 { 2123 return __btf_kind_str(btf_kind(t)); 2124 } 2125 2126 /* 2127 * Fetch integer attribute of BTF map definition. Such attributes are 2128 * represented using a pointer to an array, in which dimensionality of array 2129 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 2130 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 2131 * type definition, while using only sizeof(void *) space in ELF data section. 2132 */ 2133 static bool get_map_field_int(const char *map_name, const struct btf *btf, 2134 const struct btf_member *m, __u32 *res) 2135 { 2136 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); 2137 const char *name = btf__name_by_offset(btf, m->name_off); 2138 const struct btf_array *arr_info; 2139 const struct btf_type *arr_t; 2140 2141 if (!btf_is_ptr(t)) { 2142 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n", 2143 map_name, name, btf_kind_str(t)); 2144 return false; 2145 } 2146 2147 arr_t = btf__type_by_id(btf, t->type); 2148 if (!arr_t) { 2149 pr_warn("map '%s': attr '%s': type [%u] not found.\n", 2150 map_name, name, t->type); 2151 return false; 2152 } 2153 if (!btf_is_array(arr_t)) { 2154 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n", 2155 map_name, name, btf_kind_str(arr_t)); 2156 return false; 2157 } 2158 arr_info = btf_array(arr_t); 2159 *res = arr_info->nelems; 2160 return true; 2161 } 2162 2163 static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name) 2164 { 2165 int len; 2166 2167 len = snprintf(buf, buf_sz, "%s/%s", path, name); 2168 if (len < 0) 2169 return -EINVAL; 2170 if (len >= buf_sz) 2171 return -ENAMETOOLONG; 2172 2173 return 0; 2174 } 2175 2176 static int build_map_pin_path(struct bpf_map *map, const char *path) 2177 { 2178 char buf[PATH_MAX]; 2179 int err; 2180 2181 if (!path) 2182 path = "/sys/fs/bpf"; 2183 2184 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 2185 if (err) 2186 return err; 2187 2188 return bpf_map__set_pin_path(map, buf); 2189 } 2190 2191 /* should match definition in bpf_helpers.h */ 2192 enum libbpf_pin_type { 2193 LIBBPF_PIN_NONE, 2194 /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ 2195 LIBBPF_PIN_BY_NAME, 2196 }; 2197 2198 int parse_btf_map_def(const char *map_name, struct btf *btf, 2199 const struct btf_type *def_t, bool strict, 2200 struct btf_map_def *map_def, struct btf_map_def *inner_def) 2201 { 2202 const struct btf_type *t; 2203 const struct btf_member *m; 2204 bool is_inner = inner_def == NULL; 2205 int vlen, i; 2206 2207 vlen = btf_vlen(def_t); 2208 m = btf_members(def_t); 2209 for (i = 0; i < vlen; i++, m++) { 2210 const char *name = btf__name_by_offset(btf, m->name_off); 2211 2212 if (!name) { 2213 pr_warn("map '%s': invalid field #%d.\n", map_name, i); 2214 return -EINVAL; 2215 } 2216 if (strcmp(name, "type") == 0) { 2217 if (!get_map_field_int(map_name, btf, m, &map_def->map_type)) 2218 return -EINVAL; 2219 map_def->parts |= MAP_DEF_MAP_TYPE; 2220 } else if (strcmp(name, "max_entries") == 0) { 2221 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries)) 2222 return -EINVAL; 2223 map_def->parts |= MAP_DEF_MAX_ENTRIES; 2224 } else if (strcmp(name, "map_flags") == 0) { 2225 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags)) 2226 return -EINVAL; 2227 map_def->parts |= MAP_DEF_MAP_FLAGS; 2228 } else if (strcmp(name, "numa_node") == 0) { 2229 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node)) 2230 return -EINVAL; 2231 map_def->parts |= MAP_DEF_NUMA_NODE; 2232 } else if (strcmp(name, "key_size") == 0) { 2233 __u32 sz; 2234 2235 if (!get_map_field_int(map_name, btf, m, &sz)) 2236 return -EINVAL; 2237 if (map_def->key_size && map_def->key_size != sz) { 2238 pr_warn("map '%s': conflicting key size %u != %u.\n", 2239 map_name, map_def->key_size, sz); 2240 return -EINVAL; 2241 } 2242 map_def->key_size = sz; 2243 map_def->parts |= MAP_DEF_KEY_SIZE; 2244 } else if (strcmp(name, "key") == 0) { 2245 __s64 sz; 2246 2247 t = btf__type_by_id(btf, m->type); 2248 if (!t) { 2249 pr_warn("map '%s': key type [%d] not found.\n", 2250 map_name, m->type); 2251 return -EINVAL; 2252 } 2253 if (!btf_is_ptr(t)) { 2254 pr_warn("map '%s': key spec is not PTR: %s.\n", 2255 map_name, btf_kind_str(t)); 2256 return -EINVAL; 2257 } 2258 sz = btf__resolve_size(btf, t->type); 2259 if (sz < 0) { 2260 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", 2261 map_name, t->type, (ssize_t)sz); 2262 return sz; 2263 } 2264 if (map_def->key_size && map_def->key_size != sz) { 2265 pr_warn("map '%s': conflicting key size %u != %zd.\n", 2266 map_name, map_def->key_size, (ssize_t)sz); 2267 return -EINVAL; 2268 } 2269 map_def->key_size = sz; 2270 map_def->key_type_id = t->type; 2271 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE; 2272 } else if (strcmp(name, "value_size") == 0) { 2273 __u32 sz; 2274 2275 if (!get_map_field_int(map_name, btf, m, &sz)) 2276 return -EINVAL; 2277 if (map_def->value_size && map_def->value_size != sz) { 2278 pr_warn("map '%s': conflicting value size %u != %u.\n", 2279 map_name, map_def->value_size, sz); 2280 return -EINVAL; 2281 } 2282 map_def->value_size = sz; 2283 map_def->parts |= MAP_DEF_VALUE_SIZE; 2284 } else if (strcmp(name, "value") == 0) { 2285 __s64 sz; 2286 2287 t = btf__type_by_id(btf, m->type); 2288 if (!t) { 2289 pr_warn("map '%s': value type [%d] not found.\n", 2290 map_name, m->type); 2291 return -EINVAL; 2292 } 2293 if (!btf_is_ptr(t)) { 2294 pr_warn("map '%s': value spec is not PTR: %s.\n", 2295 map_name, btf_kind_str(t)); 2296 return -EINVAL; 2297 } 2298 sz = btf__resolve_size(btf, t->type); 2299 if (sz < 0) { 2300 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", 2301 map_name, t->type, (ssize_t)sz); 2302 return sz; 2303 } 2304 if (map_def->value_size && map_def->value_size != sz) { 2305 pr_warn("map '%s': conflicting value size %u != %zd.\n", 2306 map_name, map_def->value_size, (ssize_t)sz); 2307 return -EINVAL; 2308 } 2309 map_def->value_size = sz; 2310 map_def->value_type_id = t->type; 2311 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE; 2312 } 2313 else if (strcmp(name, "values") == 0) { 2314 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type); 2315 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY; 2316 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value"; 2317 char inner_map_name[128]; 2318 int err; 2319 2320 if (is_inner) { 2321 pr_warn("map '%s': multi-level inner maps not supported.\n", 2322 map_name); 2323 return -ENOTSUP; 2324 } 2325 if (i != vlen - 1) { 2326 pr_warn("map '%s': '%s' member should be last.\n", 2327 map_name, name); 2328 return -EINVAL; 2329 } 2330 if (!is_map_in_map && !is_prog_array) { 2331 pr_warn("map '%s': should be map-in-map or prog-array.\n", 2332 map_name); 2333 return -ENOTSUP; 2334 } 2335 if (map_def->value_size && map_def->value_size != 4) { 2336 pr_warn("map '%s': conflicting value size %u != 4.\n", 2337 map_name, map_def->value_size); 2338 return -EINVAL; 2339 } 2340 map_def->value_size = 4; 2341 t = btf__type_by_id(btf, m->type); 2342 if (!t) { 2343 pr_warn("map '%s': %s type [%d] not found.\n", 2344 map_name, desc, m->type); 2345 return -EINVAL; 2346 } 2347 if (!btf_is_array(t) || btf_array(t)->nelems) { 2348 pr_warn("map '%s': %s spec is not a zero-sized array.\n", 2349 map_name, desc); 2350 return -EINVAL; 2351 } 2352 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL); 2353 if (!btf_is_ptr(t)) { 2354 pr_warn("map '%s': %s def is of unexpected kind %s.\n", 2355 map_name, desc, btf_kind_str(t)); 2356 return -EINVAL; 2357 } 2358 t = skip_mods_and_typedefs(btf, t->type, NULL); 2359 if (is_prog_array) { 2360 if (!btf_is_func_proto(t)) { 2361 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n", 2362 map_name, btf_kind_str(t)); 2363 return -EINVAL; 2364 } 2365 continue; 2366 } 2367 if (!btf_is_struct(t)) { 2368 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2369 map_name, btf_kind_str(t)); 2370 return -EINVAL; 2371 } 2372 2373 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name); 2374 err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL); 2375 if (err) 2376 return err; 2377 2378 map_def->parts |= MAP_DEF_INNER_MAP; 2379 } else if (strcmp(name, "pinning") == 0) { 2380 __u32 val; 2381 2382 if (is_inner) { 2383 pr_warn("map '%s': inner def can't be pinned.\n", map_name); 2384 return -EINVAL; 2385 } 2386 if (!get_map_field_int(map_name, btf, m, &val)) 2387 return -EINVAL; 2388 if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) { 2389 pr_warn("map '%s': invalid pinning value %u.\n", 2390 map_name, val); 2391 return -EINVAL; 2392 } 2393 map_def->pinning = val; 2394 map_def->parts |= MAP_DEF_PINNING; 2395 } else if (strcmp(name, "map_extra") == 0) { 2396 __u32 map_extra; 2397 2398 if (!get_map_field_int(map_name, btf, m, &map_extra)) 2399 return -EINVAL; 2400 map_def->map_extra = map_extra; 2401 map_def->parts |= MAP_DEF_MAP_EXTRA; 2402 } else { 2403 if (strict) { 2404 pr_warn("map '%s': unknown field '%s'.\n", map_name, name); 2405 return -ENOTSUP; 2406 } 2407 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name); 2408 } 2409 } 2410 2411 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) { 2412 pr_warn("map '%s': map type isn't specified.\n", map_name); 2413 return -EINVAL; 2414 } 2415 2416 return 0; 2417 } 2418 2419 static size_t adjust_ringbuf_sz(size_t sz) 2420 { 2421 __u32 page_sz = sysconf(_SC_PAGE_SIZE); 2422 __u32 mul; 2423 2424 /* if user forgot to set any size, make sure they see error */ 2425 if (sz == 0) 2426 return 0; 2427 /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be 2428 * a power-of-2 multiple of kernel's page size. If user diligently 2429 * satisified these conditions, pass the size through. 2430 */ 2431 if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz)) 2432 return sz; 2433 2434 /* Otherwise find closest (page_sz * power_of_2) product bigger than 2435 * user-set size to satisfy both user size request and kernel 2436 * requirements and substitute correct max_entries for map creation. 2437 */ 2438 for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) { 2439 if (mul * page_sz > sz) 2440 return mul * page_sz; 2441 } 2442 2443 /* if it's impossible to satisfy the conditions (i.e., user size is 2444 * very close to UINT_MAX but is not a power-of-2 multiple of 2445 * page_size) then just return original size and let kernel reject it 2446 */ 2447 return sz; 2448 } 2449 2450 static bool map_is_ringbuf(const struct bpf_map *map) 2451 { 2452 return map->def.type == BPF_MAP_TYPE_RINGBUF || 2453 map->def.type == BPF_MAP_TYPE_USER_RINGBUF; 2454 } 2455 2456 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) 2457 { 2458 map->def.type = def->map_type; 2459 map->def.key_size = def->key_size; 2460 map->def.value_size = def->value_size; 2461 map->def.max_entries = def->max_entries; 2462 map->def.map_flags = def->map_flags; 2463 map->map_extra = def->map_extra; 2464 2465 map->numa_node = def->numa_node; 2466 map->btf_key_type_id = def->key_type_id; 2467 map->btf_value_type_id = def->value_type_id; 2468 2469 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ 2470 if (map_is_ringbuf(map)) 2471 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); 2472 2473 if (def->parts & MAP_DEF_MAP_TYPE) 2474 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); 2475 2476 if (def->parts & MAP_DEF_KEY_TYPE) 2477 pr_debug("map '%s': found key [%u], sz = %u.\n", 2478 map->name, def->key_type_id, def->key_size); 2479 else if (def->parts & MAP_DEF_KEY_SIZE) 2480 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size); 2481 2482 if (def->parts & MAP_DEF_VALUE_TYPE) 2483 pr_debug("map '%s': found value [%u], sz = %u.\n", 2484 map->name, def->value_type_id, def->value_size); 2485 else if (def->parts & MAP_DEF_VALUE_SIZE) 2486 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size); 2487 2488 if (def->parts & MAP_DEF_MAX_ENTRIES) 2489 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries); 2490 if (def->parts & MAP_DEF_MAP_FLAGS) 2491 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags); 2492 if (def->parts & MAP_DEF_MAP_EXTRA) 2493 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name, 2494 (unsigned long long)def->map_extra); 2495 if (def->parts & MAP_DEF_PINNING) 2496 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning); 2497 if (def->parts & MAP_DEF_NUMA_NODE) 2498 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node); 2499 2500 if (def->parts & MAP_DEF_INNER_MAP) 2501 pr_debug("map '%s': found inner map definition.\n", map->name); 2502 } 2503 2504 static const char *btf_var_linkage_str(__u32 linkage) 2505 { 2506 switch (linkage) { 2507 case BTF_VAR_STATIC: return "static"; 2508 case BTF_VAR_GLOBAL_ALLOCATED: return "global"; 2509 case BTF_VAR_GLOBAL_EXTERN: return "extern"; 2510 default: return "unknown"; 2511 } 2512 } 2513 2514 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 2515 const struct btf_type *sec, 2516 int var_idx, int sec_idx, 2517 const Elf_Data *data, bool strict, 2518 const char *pin_root_path) 2519 { 2520 struct btf_map_def map_def = {}, inner_def = {}; 2521 const struct btf_type *var, *def; 2522 const struct btf_var_secinfo *vi; 2523 const struct btf_var *var_extra; 2524 const char *map_name; 2525 struct bpf_map *map; 2526 int err; 2527 2528 vi = btf_var_secinfos(sec) + var_idx; 2529 var = btf__type_by_id(obj->btf, vi->type); 2530 var_extra = btf_var(var); 2531 map_name = btf__name_by_offset(obj->btf, var->name_off); 2532 2533 if (map_name == NULL || map_name[0] == '\0') { 2534 pr_warn("map #%d: empty name.\n", var_idx); 2535 return -EINVAL; 2536 } 2537 if ((__u64)vi->offset + vi->size > data->d_size) { 2538 pr_warn("map '%s' BTF data is corrupted.\n", map_name); 2539 return -EINVAL; 2540 } 2541 if (!btf_is_var(var)) { 2542 pr_warn("map '%s': unexpected var kind %s.\n", 2543 map_name, btf_kind_str(var)); 2544 return -EINVAL; 2545 } 2546 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) { 2547 pr_warn("map '%s': unsupported map linkage %s.\n", 2548 map_name, btf_var_linkage_str(var_extra->linkage)); 2549 return -EOPNOTSUPP; 2550 } 2551 2552 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 2553 if (!btf_is_struct(def)) { 2554 pr_warn("map '%s': unexpected def kind %s.\n", 2555 map_name, btf_kind_str(var)); 2556 return -EINVAL; 2557 } 2558 if (def->size > vi->size) { 2559 pr_warn("map '%s': invalid def size.\n", map_name); 2560 return -EINVAL; 2561 } 2562 2563 map = bpf_object__add_map(obj); 2564 if (IS_ERR(map)) 2565 return PTR_ERR(map); 2566 map->name = strdup(map_name); 2567 if (!map->name) { 2568 pr_warn("map '%s': failed to alloc map name.\n", map_name); 2569 return -ENOMEM; 2570 } 2571 map->libbpf_type = LIBBPF_MAP_UNSPEC; 2572 map->def.type = BPF_MAP_TYPE_UNSPEC; 2573 map->sec_idx = sec_idx; 2574 map->sec_offset = vi->offset; 2575 map->btf_var_idx = var_idx; 2576 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 2577 map_name, map->sec_idx, map->sec_offset); 2578 2579 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); 2580 if (err) 2581 return err; 2582 2583 fill_map_from_def(map, &map_def); 2584 2585 if (map_def.pinning == LIBBPF_PIN_BY_NAME) { 2586 err = build_map_pin_path(map, pin_root_path); 2587 if (err) { 2588 pr_warn("map '%s': couldn't build pin path.\n", map->name); 2589 return err; 2590 } 2591 } 2592 2593 if (map_def.parts & MAP_DEF_INNER_MAP) { 2594 map->inner_map = calloc(1, sizeof(*map->inner_map)); 2595 if (!map->inner_map) 2596 return -ENOMEM; 2597 map->inner_map->fd = -1; 2598 map->inner_map->sec_idx = sec_idx; 2599 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1); 2600 if (!map->inner_map->name) 2601 return -ENOMEM; 2602 sprintf(map->inner_map->name, "%s.inner", map_name); 2603 2604 fill_map_from_def(map->inner_map, &inner_def); 2605 } 2606 2607 err = map_fill_btf_type_info(obj, map); 2608 if (err) 2609 return err; 2610 2611 return 0; 2612 } 2613 2614 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, 2615 const char *pin_root_path) 2616 { 2617 const struct btf_type *sec = NULL; 2618 int nr_types, i, vlen, err; 2619 const struct btf_type *t; 2620 const char *name; 2621 Elf_Data *data; 2622 Elf_Scn *scn; 2623 2624 if (obj->efile.btf_maps_shndx < 0) 2625 return 0; 2626 2627 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); 2628 data = elf_sec_data(obj, scn); 2629 if (!scn || !data) { 2630 pr_warn("elf: failed to get %s map definitions for %s\n", 2631 MAPS_ELF_SEC, obj->path); 2632 return -EINVAL; 2633 } 2634 2635 nr_types = btf__type_cnt(obj->btf); 2636 for (i = 1; i < nr_types; i++) { 2637 t = btf__type_by_id(obj->btf, i); 2638 if (!btf_is_datasec(t)) 2639 continue; 2640 name = btf__name_by_offset(obj->btf, t->name_off); 2641 if (strcmp(name, MAPS_ELF_SEC) == 0) { 2642 sec = t; 2643 obj->efile.btf_maps_sec_btf_id = i; 2644 break; 2645 } 2646 } 2647 2648 if (!sec) { 2649 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 2650 return -ENOENT; 2651 } 2652 2653 vlen = btf_vlen(sec); 2654 for (i = 0; i < vlen; i++) { 2655 err = bpf_object__init_user_btf_map(obj, sec, i, 2656 obj->efile.btf_maps_shndx, 2657 data, strict, 2658 pin_root_path); 2659 if (err) 2660 return err; 2661 } 2662 2663 return 0; 2664 } 2665 2666 static int bpf_object__init_maps(struct bpf_object *obj, 2667 const struct bpf_object_open_opts *opts) 2668 { 2669 const char *pin_root_path; 2670 bool strict; 2671 int err = 0; 2672 2673 strict = !OPTS_GET(opts, relaxed_maps, false); 2674 pin_root_path = OPTS_GET(opts, pin_root_path, NULL); 2675 2676 err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path); 2677 err = err ?: bpf_object__init_global_data_maps(obj); 2678 err = err ?: bpf_object__init_kconfig_map(obj); 2679 err = err ?: bpf_object_init_struct_ops(obj); 2680 2681 return err; 2682 } 2683 2684 static bool section_have_execinstr(struct bpf_object *obj, int idx) 2685 { 2686 Elf64_Shdr *sh; 2687 2688 sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx)); 2689 if (!sh) 2690 return false; 2691 2692 return sh->sh_flags & SHF_EXECINSTR; 2693 } 2694 2695 static bool btf_needs_sanitization(struct bpf_object *obj) 2696 { 2697 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2698 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2699 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2700 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2701 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); 2702 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); 2703 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); 2704 2705 return !has_func || !has_datasec || !has_func_global || !has_float || 2706 !has_decl_tag || !has_type_tag || !has_enum64; 2707 } 2708 2709 static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) 2710 { 2711 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2712 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2713 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2714 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2715 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); 2716 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); 2717 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); 2718 int enum64_placeholder_id = 0; 2719 struct btf_type *t; 2720 int i, j, vlen; 2721 2722 for (i = 1; i < btf__type_cnt(btf); i++) { 2723 t = (struct btf_type *)btf__type_by_id(btf, i); 2724 2725 if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) { 2726 /* replace VAR/DECL_TAG with INT */ 2727 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 2728 /* 2729 * using size = 1 is the safest choice, 4 will be too 2730 * big and cause kernel BTF validation failure if 2731 * original variable took less than 4 bytes 2732 */ 2733 t->size = 1; 2734 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); 2735 } else if (!has_datasec && btf_is_datasec(t)) { 2736 /* replace DATASEC with STRUCT */ 2737 const struct btf_var_secinfo *v = btf_var_secinfos(t); 2738 struct btf_member *m = btf_members(t); 2739 struct btf_type *vt; 2740 char *name; 2741 2742 name = (char *)btf__name_by_offset(btf, t->name_off); 2743 while (*name) { 2744 if (*name == '.') 2745 *name = '_'; 2746 name++; 2747 } 2748 2749 vlen = btf_vlen(t); 2750 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 2751 for (j = 0; j < vlen; j++, v++, m++) { 2752 /* order of field assignments is important */ 2753 m->offset = v->offset * 8; 2754 m->type = v->type; 2755 /* preserve variable name as member name */ 2756 vt = (void *)btf__type_by_id(btf, v->type); 2757 m->name_off = vt->name_off; 2758 } 2759 } else if (!has_func && btf_is_func_proto(t)) { 2760 /* replace FUNC_PROTO with ENUM */ 2761 vlen = btf_vlen(t); 2762 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 2763 t->size = sizeof(__u32); /* kernel enforced */ 2764 } else if (!has_func && btf_is_func(t)) { 2765 /* replace FUNC with TYPEDEF */ 2766 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 2767 } else if (!has_func_global && btf_is_func(t)) { 2768 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ 2769 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); 2770 } else if (!has_float && btf_is_float(t)) { 2771 /* replace FLOAT with an equally-sized empty STRUCT; 2772 * since C compilers do not accept e.g. "float" as a 2773 * valid struct name, make it anonymous 2774 */ 2775 t->name_off = 0; 2776 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); 2777 } else if (!has_type_tag && btf_is_type_tag(t)) { 2778 /* replace TYPE_TAG with a CONST */ 2779 t->name_off = 0; 2780 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0); 2781 } else if (!has_enum64 && btf_is_enum(t)) { 2782 /* clear the kflag */ 2783 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false); 2784 } else if (!has_enum64 && btf_is_enum64(t)) { 2785 /* replace ENUM64 with a union */ 2786 struct btf_member *m; 2787 2788 if (enum64_placeholder_id == 0) { 2789 enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0); 2790 if (enum64_placeholder_id < 0) 2791 return enum64_placeholder_id; 2792 2793 t = (struct btf_type *)btf__type_by_id(btf, i); 2794 } 2795 2796 m = btf_members(t); 2797 vlen = btf_vlen(t); 2798 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen); 2799 for (j = 0; j < vlen; j++, m++) { 2800 m->type = enum64_placeholder_id; 2801 m->offset = 0; 2802 } 2803 } 2804 } 2805 2806 return 0; 2807 } 2808 2809 static bool libbpf_needs_btf(const struct bpf_object *obj) 2810 { 2811 return obj->efile.btf_maps_shndx >= 0 || 2812 obj->efile.st_ops_shndx >= 0 || 2813 obj->efile.st_ops_link_shndx >= 0 || 2814 obj->nr_extern > 0; 2815 } 2816 2817 static bool kernel_needs_btf(const struct bpf_object *obj) 2818 { 2819 return obj->efile.st_ops_shndx >= 0 || obj->efile.st_ops_link_shndx >= 0; 2820 } 2821 2822 static int bpf_object__init_btf(struct bpf_object *obj, 2823 Elf_Data *btf_data, 2824 Elf_Data *btf_ext_data) 2825 { 2826 int err = -ENOENT; 2827 2828 if (btf_data) { 2829 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 2830 err = libbpf_get_error(obj->btf); 2831 if (err) { 2832 obj->btf = NULL; 2833 pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err); 2834 goto out; 2835 } 2836 /* enforce 8-byte pointers for BPF-targeted BTFs */ 2837 btf__set_pointer_size(obj->btf, 8); 2838 } 2839 if (btf_ext_data) { 2840 struct btf_ext_info *ext_segs[3]; 2841 int seg_num, sec_num; 2842 2843 if (!obj->btf) { 2844 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 2845 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 2846 goto out; 2847 } 2848 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); 2849 err = libbpf_get_error(obj->btf_ext); 2850 if (err) { 2851 pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n", 2852 BTF_EXT_ELF_SEC, err); 2853 obj->btf_ext = NULL; 2854 goto out; 2855 } 2856 2857 /* setup .BTF.ext to ELF section mapping */ 2858 ext_segs[0] = &obj->btf_ext->func_info; 2859 ext_segs[1] = &obj->btf_ext->line_info; 2860 ext_segs[2] = &obj->btf_ext->core_relo_info; 2861 for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) { 2862 struct btf_ext_info *seg = ext_segs[seg_num]; 2863 const struct btf_ext_info_sec *sec; 2864 const char *sec_name; 2865 Elf_Scn *scn; 2866 2867 if (seg->sec_cnt == 0) 2868 continue; 2869 2870 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs)); 2871 if (!seg->sec_idxs) { 2872 err = -ENOMEM; 2873 goto out; 2874 } 2875 2876 sec_num = 0; 2877 for_each_btf_ext_sec(seg, sec) { 2878 /* preventively increment index to avoid doing 2879 * this before every continue below 2880 */ 2881 sec_num++; 2882 2883 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 2884 if (str_is_empty(sec_name)) 2885 continue; 2886 scn = elf_sec_by_name(obj, sec_name); 2887 if (!scn) 2888 continue; 2889 2890 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn); 2891 } 2892 } 2893 } 2894 out: 2895 if (err && libbpf_needs_btf(obj)) { 2896 pr_warn("BTF is required, but is missing or corrupted.\n"); 2897 return err; 2898 } 2899 return 0; 2900 } 2901 2902 static int compare_vsi_off(const void *_a, const void *_b) 2903 { 2904 const struct btf_var_secinfo *a = _a; 2905 const struct btf_var_secinfo *b = _b; 2906 2907 return a->offset - b->offset; 2908 } 2909 2910 static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, 2911 struct btf_type *t) 2912 { 2913 __u32 size = 0, i, vars = btf_vlen(t); 2914 const char *sec_name = btf__name_by_offset(btf, t->name_off); 2915 struct btf_var_secinfo *vsi; 2916 bool fixup_offsets = false; 2917 int err; 2918 2919 if (!sec_name) { 2920 pr_debug("No name found in string section for DATASEC kind.\n"); 2921 return -ENOENT; 2922 } 2923 2924 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and 2925 * variable offsets set at the previous step. Further, not every 2926 * extern BTF VAR has corresponding ELF symbol preserved, so we skip 2927 * all fixups altogether for such sections and go straight to sorting 2928 * VARs within their DATASEC. 2929 */ 2930 if (strcmp(sec_name, KCONFIG_SEC) == 0 || strcmp(sec_name, KSYMS_SEC) == 0) 2931 goto sort_vars; 2932 2933 /* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to 2934 * fix this up. But BPF static linker already fixes this up and fills 2935 * all the sizes and offsets during static linking. So this step has 2936 * to be optional. But the STV_HIDDEN handling is non-optional for any 2937 * non-extern DATASEC, so the variable fixup loop below handles both 2938 * functions at the same time, paying the cost of BTF VAR <-> ELF 2939 * symbol matching just once. 2940 */ 2941 if (t->size == 0) { 2942 err = find_elf_sec_sz(obj, sec_name, &size); 2943 if (err || !size) { 2944 pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n", 2945 sec_name, size, err); 2946 return -ENOENT; 2947 } 2948 2949 t->size = size; 2950 fixup_offsets = true; 2951 } 2952 2953 for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) { 2954 const struct btf_type *t_var; 2955 struct btf_var *var; 2956 const char *var_name; 2957 Elf64_Sym *sym; 2958 2959 t_var = btf__type_by_id(btf, vsi->type); 2960 if (!t_var || !btf_is_var(t_var)) { 2961 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name); 2962 return -EINVAL; 2963 } 2964 2965 var = btf_var(t_var); 2966 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN) 2967 continue; 2968 2969 var_name = btf__name_by_offset(btf, t_var->name_off); 2970 if (!var_name) { 2971 pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n", 2972 sec_name, i); 2973 return -ENOENT; 2974 } 2975 2976 sym = find_elf_var_sym(obj, var_name); 2977 if (IS_ERR(sym)) { 2978 pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n", 2979 sec_name, var_name); 2980 return -ENOENT; 2981 } 2982 2983 if (fixup_offsets) 2984 vsi->offset = sym->st_value; 2985 2986 /* if variable is a global/weak symbol, but has restricted 2987 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR 2988 * as static. This follows similar logic for functions (BPF 2989 * subprogs) and influences libbpf's further decisions about 2990 * whether to make global data BPF array maps as 2991 * BPF_F_MMAPABLE. 2992 */ 2993 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN 2994 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL) 2995 var->linkage = BTF_VAR_STATIC; 2996 } 2997 2998 sort_vars: 2999 qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off); 3000 return 0; 3001 } 3002 3003 static int bpf_object_fixup_btf(struct bpf_object *obj) 3004 { 3005 int i, n, err = 0; 3006 3007 if (!obj->btf) 3008 return 0; 3009 3010 n = btf__type_cnt(obj->btf); 3011 for (i = 1; i < n; i++) { 3012 struct btf_type *t = btf_type_by_id(obj->btf, i); 3013 3014 /* Loader needs to fix up some of the things compiler 3015 * couldn't get its hands on while emitting BTF. This 3016 * is section size and global variable offset. We use 3017 * the info from the ELF itself for this purpose. 3018 */ 3019 if (btf_is_datasec(t)) { 3020 err = btf_fixup_datasec(obj, obj->btf, t); 3021 if (err) 3022 return err; 3023 } 3024 } 3025 3026 return 0; 3027 } 3028 3029 static bool prog_needs_vmlinux_btf(struct bpf_program *prog) 3030 { 3031 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 3032 prog->type == BPF_PROG_TYPE_LSM) 3033 return true; 3034 3035 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs 3036 * also need vmlinux BTF 3037 */ 3038 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) 3039 return true; 3040 3041 return false; 3042 } 3043 3044 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) 3045 { 3046 struct bpf_program *prog; 3047 int i; 3048 3049 /* CO-RE relocations need kernel BTF, only when btf_custom_path 3050 * is not specified 3051 */ 3052 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path) 3053 return true; 3054 3055 /* Support for typed ksyms needs kernel BTF */ 3056 for (i = 0; i < obj->nr_extern; i++) { 3057 const struct extern_desc *ext; 3058 3059 ext = &obj->externs[i]; 3060 if (ext->type == EXT_KSYM && ext->ksym.type_id) 3061 return true; 3062 } 3063 3064 bpf_object__for_each_program(prog, obj) { 3065 if (!prog->autoload) 3066 continue; 3067 if (prog_needs_vmlinux_btf(prog)) 3068 return true; 3069 } 3070 3071 return false; 3072 } 3073 3074 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) 3075 { 3076 int err; 3077 3078 /* btf_vmlinux could be loaded earlier */ 3079 if (obj->btf_vmlinux || obj->gen_loader) 3080 return 0; 3081 3082 if (!force && !obj_needs_vmlinux_btf(obj)) 3083 return 0; 3084 3085 obj->btf_vmlinux = btf__load_vmlinux_btf(); 3086 err = libbpf_get_error(obj->btf_vmlinux); 3087 if (err) { 3088 pr_warn("Error loading vmlinux BTF: %d\n", err); 3089 obj->btf_vmlinux = NULL; 3090 return err; 3091 } 3092 return 0; 3093 } 3094 3095 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 3096 { 3097 struct btf *kern_btf = obj->btf; 3098 bool btf_mandatory, sanitize; 3099 int i, err = 0; 3100 3101 if (!obj->btf) 3102 return 0; 3103 3104 if (!kernel_supports(obj, FEAT_BTF)) { 3105 if (kernel_needs_btf(obj)) { 3106 err = -EOPNOTSUPP; 3107 goto report; 3108 } 3109 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n"); 3110 return 0; 3111 } 3112 3113 /* Even though some subprogs are global/weak, user might prefer more 3114 * permissive BPF verification process that BPF verifier performs for 3115 * static functions, taking into account more context from the caller 3116 * functions. In such case, they need to mark such subprogs with 3117 * __attribute__((visibility("hidden"))) and libbpf will adjust 3118 * corresponding FUNC BTF type to be marked as static and trigger more 3119 * involved BPF verification process. 3120 */ 3121 for (i = 0; i < obj->nr_programs; i++) { 3122 struct bpf_program *prog = &obj->programs[i]; 3123 struct btf_type *t; 3124 const char *name; 3125 int j, n; 3126 3127 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) 3128 continue; 3129 3130 n = btf__type_cnt(obj->btf); 3131 for (j = 1; j < n; j++) { 3132 t = btf_type_by_id(obj->btf, j); 3133 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) 3134 continue; 3135 3136 name = btf__str_by_offset(obj->btf, t->name_off); 3137 if (strcmp(name, prog->name) != 0) 3138 continue; 3139 3140 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0); 3141 break; 3142 } 3143 } 3144 3145 sanitize = btf_needs_sanitization(obj); 3146 if (sanitize) { 3147 const void *raw_data; 3148 __u32 sz; 3149 3150 /* clone BTF to sanitize a copy and leave the original intact */ 3151 raw_data = btf__raw_data(obj->btf, &sz); 3152 kern_btf = btf__new(raw_data, sz); 3153 err = libbpf_get_error(kern_btf); 3154 if (err) 3155 return err; 3156 3157 /* enforce 8-byte pointers for BPF-targeted BTFs */ 3158 btf__set_pointer_size(obj->btf, 8); 3159 err = bpf_object__sanitize_btf(obj, kern_btf); 3160 if (err) 3161 return err; 3162 } 3163 3164 if (obj->gen_loader) { 3165 __u32 raw_size = 0; 3166 const void *raw_data = btf__raw_data(kern_btf, &raw_size); 3167 3168 if (!raw_data) 3169 return -ENOMEM; 3170 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size); 3171 /* Pretend to have valid FD to pass various fd >= 0 checks. 3172 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 3173 */ 3174 btf__set_fd(kern_btf, 0); 3175 } else { 3176 /* currently BPF_BTF_LOAD only supports log_level 1 */ 3177 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size, 3178 obj->log_level ? 1 : 0); 3179 } 3180 if (sanitize) { 3181 if (!err) { 3182 /* move fd to libbpf's BTF */ 3183 btf__set_fd(obj->btf, btf__fd(kern_btf)); 3184 btf__set_fd(kern_btf, -1); 3185 } 3186 btf__free(kern_btf); 3187 } 3188 report: 3189 if (err) { 3190 btf_mandatory = kernel_needs_btf(obj); 3191 pr_warn("Error loading .BTF into kernel: %d. %s\n", err, 3192 btf_mandatory ? "BTF is mandatory, can't proceed." 3193 : "BTF is optional, ignoring."); 3194 if (!btf_mandatory) 3195 err = 0; 3196 } 3197 return err; 3198 } 3199 3200 static const char *elf_sym_str(const struct bpf_object *obj, size_t off) 3201 { 3202 const char *name; 3203 3204 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); 3205 if (!name) { 3206 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 3207 off, obj->path, elf_errmsg(-1)); 3208 return NULL; 3209 } 3210 3211 return name; 3212 } 3213 3214 static const char *elf_sec_str(const struct bpf_object *obj, size_t off) 3215 { 3216 const char *name; 3217 3218 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); 3219 if (!name) { 3220 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 3221 off, obj->path, elf_errmsg(-1)); 3222 return NULL; 3223 } 3224 3225 return name; 3226 } 3227 3228 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx) 3229 { 3230 Elf_Scn *scn; 3231 3232 scn = elf_getscn(obj->efile.elf, idx); 3233 if (!scn) { 3234 pr_warn("elf: failed to get section(%zu) from %s: %s\n", 3235 idx, obj->path, elf_errmsg(-1)); 3236 return NULL; 3237 } 3238 return scn; 3239 } 3240 3241 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name) 3242 { 3243 Elf_Scn *scn = NULL; 3244 Elf *elf = obj->efile.elf; 3245 const char *sec_name; 3246 3247 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3248 sec_name = elf_sec_name(obj, scn); 3249 if (!sec_name) 3250 return NULL; 3251 3252 if (strcmp(sec_name, name) != 0) 3253 continue; 3254 3255 return scn; 3256 } 3257 return NULL; 3258 } 3259 3260 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn) 3261 { 3262 Elf64_Shdr *shdr; 3263 3264 if (!scn) 3265 return NULL; 3266 3267 shdr = elf64_getshdr(scn); 3268 if (!shdr) { 3269 pr_warn("elf: failed to get section(%zu) header from %s: %s\n", 3270 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 3271 return NULL; 3272 } 3273 3274 return shdr; 3275 } 3276 3277 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn) 3278 { 3279 const char *name; 3280 Elf64_Shdr *sh; 3281 3282 if (!scn) 3283 return NULL; 3284 3285 sh = elf_sec_hdr(obj, scn); 3286 if (!sh) 3287 return NULL; 3288 3289 name = elf_sec_str(obj, sh->sh_name); 3290 if (!name) { 3291 pr_warn("elf: failed to get section(%zu) name from %s: %s\n", 3292 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 3293 return NULL; 3294 } 3295 3296 return name; 3297 } 3298 3299 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn) 3300 { 3301 Elf_Data *data; 3302 3303 if (!scn) 3304 return NULL; 3305 3306 data = elf_getdata(scn, 0); 3307 if (!data) { 3308 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n", 3309 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>", 3310 obj->path, elf_errmsg(-1)); 3311 return NULL; 3312 } 3313 3314 return data; 3315 } 3316 3317 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx) 3318 { 3319 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym)) 3320 return NULL; 3321 3322 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx; 3323 } 3324 3325 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx) 3326 { 3327 if (idx >= data->d_size / sizeof(Elf64_Rel)) 3328 return NULL; 3329 3330 return (Elf64_Rel *)data->d_buf + idx; 3331 } 3332 3333 static bool is_sec_name_dwarf(const char *name) 3334 { 3335 /* approximation, but the actual list is too long */ 3336 return str_has_pfx(name, ".debug_"); 3337 } 3338 3339 static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name) 3340 { 3341 /* no special handling of .strtab */ 3342 if (hdr->sh_type == SHT_STRTAB) 3343 return true; 3344 3345 /* ignore .llvm_addrsig section as well */ 3346 if (hdr->sh_type == SHT_LLVM_ADDRSIG) 3347 return true; 3348 3349 /* no subprograms will lead to an empty .text section, ignore it */ 3350 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 && 3351 strcmp(name, ".text") == 0) 3352 return true; 3353 3354 /* DWARF sections */ 3355 if (is_sec_name_dwarf(name)) 3356 return true; 3357 3358 if (str_has_pfx(name, ".rel")) { 3359 name += sizeof(".rel") - 1; 3360 /* DWARF section relocations */ 3361 if (is_sec_name_dwarf(name)) 3362 return true; 3363 3364 /* .BTF and .BTF.ext don't need relocations */ 3365 if (strcmp(name, BTF_ELF_SEC) == 0 || 3366 strcmp(name, BTF_EXT_ELF_SEC) == 0) 3367 return true; 3368 } 3369 3370 return false; 3371 } 3372 3373 static int cmp_progs(const void *_a, const void *_b) 3374 { 3375 const struct bpf_program *a = _a; 3376 const struct bpf_program *b = _b; 3377 3378 if (a->sec_idx != b->sec_idx) 3379 return a->sec_idx < b->sec_idx ? -1 : 1; 3380 3381 /* sec_insn_off can't be the same within the section */ 3382 return a->sec_insn_off < b->sec_insn_off ? -1 : 1; 3383 } 3384 3385 static int bpf_object__elf_collect(struct bpf_object *obj) 3386 { 3387 struct elf_sec_desc *sec_desc; 3388 Elf *elf = obj->efile.elf; 3389 Elf_Data *btf_ext_data = NULL; 3390 Elf_Data *btf_data = NULL; 3391 int idx = 0, err = 0; 3392 const char *name; 3393 Elf_Data *data; 3394 Elf_Scn *scn; 3395 Elf64_Shdr *sh; 3396 3397 /* ELF section indices are 0-based, but sec #0 is special "invalid" 3398 * section. Since section count retrieved by elf_getshdrnum() does 3399 * include sec #0, it is already the necessary size of an array to keep 3400 * all the sections. 3401 */ 3402 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) { 3403 pr_warn("elf: failed to get the number of sections for %s: %s\n", 3404 obj->path, elf_errmsg(-1)); 3405 return -LIBBPF_ERRNO__FORMAT; 3406 } 3407 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs)); 3408 if (!obj->efile.secs) 3409 return -ENOMEM; 3410 3411 /* a bunch of ELF parsing functionality depends on processing symbols, 3412 * so do the first pass and find the symbol table 3413 */ 3414 scn = NULL; 3415 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3416 sh = elf_sec_hdr(obj, scn); 3417 if (!sh) 3418 return -LIBBPF_ERRNO__FORMAT; 3419 3420 if (sh->sh_type == SHT_SYMTAB) { 3421 if (obj->efile.symbols) { 3422 pr_warn("elf: multiple symbol tables in %s\n", obj->path); 3423 return -LIBBPF_ERRNO__FORMAT; 3424 } 3425 3426 data = elf_sec_data(obj, scn); 3427 if (!data) 3428 return -LIBBPF_ERRNO__FORMAT; 3429 3430 idx = elf_ndxscn(scn); 3431 3432 obj->efile.symbols = data; 3433 obj->efile.symbols_shndx = idx; 3434 obj->efile.strtabidx = sh->sh_link; 3435 } 3436 } 3437 3438 if (!obj->efile.symbols) { 3439 pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n", 3440 obj->path); 3441 return -ENOENT; 3442 } 3443 3444 scn = NULL; 3445 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3446 idx = elf_ndxscn(scn); 3447 sec_desc = &obj->efile.secs[idx]; 3448 3449 sh = elf_sec_hdr(obj, scn); 3450 if (!sh) 3451 return -LIBBPF_ERRNO__FORMAT; 3452 3453 name = elf_sec_str(obj, sh->sh_name); 3454 if (!name) 3455 return -LIBBPF_ERRNO__FORMAT; 3456 3457 if (ignore_elf_section(sh, name)) 3458 continue; 3459 3460 data = elf_sec_data(obj, scn); 3461 if (!data) 3462 return -LIBBPF_ERRNO__FORMAT; 3463 3464 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 3465 idx, name, (unsigned long)data->d_size, 3466 (int)sh->sh_link, (unsigned long)sh->sh_flags, 3467 (int)sh->sh_type); 3468 3469 if (strcmp(name, "license") == 0) { 3470 err = bpf_object__init_license(obj, data->d_buf, data->d_size); 3471 if (err) 3472 return err; 3473 } else if (strcmp(name, "version") == 0) { 3474 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); 3475 if (err) 3476 return err; 3477 } else if (strcmp(name, "maps") == 0) { 3478 pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n"); 3479 return -ENOTSUP; 3480 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 3481 obj->efile.btf_maps_shndx = idx; 3482 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 3483 if (sh->sh_type != SHT_PROGBITS) 3484 return -LIBBPF_ERRNO__FORMAT; 3485 btf_data = data; 3486 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 3487 if (sh->sh_type != SHT_PROGBITS) 3488 return -LIBBPF_ERRNO__FORMAT; 3489 btf_ext_data = data; 3490 } else if (sh->sh_type == SHT_SYMTAB) { 3491 /* already processed during the first pass above */ 3492 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) { 3493 if (sh->sh_flags & SHF_EXECINSTR) { 3494 if (strcmp(name, ".text") == 0) 3495 obj->efile.text_shndx = idx; 3496 err = bpf_object__add_programs(obj, data, name, idx); 3497 if (err) 3498 return err; 3499 } else if (strcmp(name, DATA_SEC) == 0 || 3500 str_has_pfx(name, DATA_SEC ".")) { 3501 sec_desc->sec_type = SEC_DATA; 3502 sec_desc->shdr = sh; 3503 sec_desc->data = data; 3504 } else if (strcmp(name, RODATA_SEC) == 0 || 3505 str_has_pfx(name, RODATA_SEC ".")) { 3506 sec_desc->sec_type = SEC_RODATA; 3507 sec_desc->shdr = sh; 3508 sec_desc->data = data; 3509 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) { 3510 obj->efile.st_ops_data = data; 3511 obj->efile.st_ops_shndx = idx; 3512 } else if (strcmp(name, STRUCT_OPS_LINK_SEC) == 0) { 3513 obj->efile.st_ops_link_data = data; 3514 obj->efile.st_ops_link_shndx = idx; 3515 } else { 3516 pr_info("elf: skipping unrecognized data section(%d) %s\n", 3517 idx, name); 3518 } 3519 } else if (sh->sh_type == SHT_REL) { 3520 int targ_sec_idx = sh->sh_info; /* points to other section */ 3521 3522 if (sh->sh_entsize != sizeof(Elf64_Rel) || 3523 targ_sec_idx >= obj->efile.sec_cnt) 3524 return -LIBBPF_ERRNO__FORMAT; 3525 3526 /* Only do relo for section with exec instructions */ 3527 if (!section_have_execinstr(obj, targ_sec_idx) && 3528 strcmp(name, ".rel" STRUCT_OPS_SEC) && 3529 strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) && 3530 strcmp(name, ".rel" MAPS_ELF_SEC)) { 3531 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", 3532 idx, name, targ_sec_idx, 3533 elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>"); 3534 continue; 3535 } 3536 3537 sec_desc->sec_type = SEC_RELO; 3538 sec_desc->shdr = sh; 3539 sec_desc->data = data; 3540 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 || 3541 str_has_pfx(name, BSS_SEC "."))) { 3542 sec_desc->sec_type = SEC_BSS; 3543 sec_desc->shdr = sh; 3544 sec_desc->data = data; 3545 } else { 3546 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name, 3547 (size_t)sh->sh_size); 3548 } 3549 } 3550 3551 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { 3552 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); 3553 return -LIBBPF_ERRNO__FORMAT; 3554 } 3555 3556 /* sort BPF programs by section name and in-section instruction offset 3557 * for faster search 3558 */ 3559 if (obj->nr_programs) 3560 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); 3561 3562 return bpf_object__init_btf(obj, btf_data, btf_ext_data); 3563 } 3564 3565 static bool sym_is_extern(const Elf64_Sym *sym) 3566 { 3567 int bind = ELF64_ST_BIND(sym->st_info); 3568 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ 3569 return sym->st_shndx == SHN_UNDEF && 3570 (bind == STB_GLOBAL || bind == STB_WEAK) && 3571 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE; 3572 } 3573 3574 static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx) 3575 { 3576 int bind = ELF64_ST_BIND(sym->st_info); 3577 int type = ELF64_ST_TYPE(sym->st_info); 3578 3579 /* in .text section */ 3580 if (sym->st_shndx != text_shndx) 3581 return false; 3582 3583 /* local function */ 3584 if (bind == STB_LOCAL && type == STT_SECTION) 3585 return true; 3586 3587 /* global function */ 3588 return bind == STB_GLOBAL && type == STT_FUNC; 3589 } 3590 3591 static int find_extern_btf_id(const struct btf *btf, const char *ext_name) 3592 { 3593 const struct btf_type *t; 3594 const char *tname; 3595 int i, n; 3596 3597 if (!btf) 3598 return -ESRCH; 3599 3600 n = btf__type_cnt(btf); 3601 for (i = 1; i < n; i++) { 3602 t = btf__type_by_id(btf, i); 3603 3604 if (!btf_is_var(t) && !btf_is_func(t)) 3605 continue; 3606 3607 tname = btf__name_by_offset(btf, t->name_off); 3608 if (strcmp(tname, ext_name)) 3609 continue; 3610 3611 if (btf_is_var(t) && 3612 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) 3613 return -EINVAL; 3614 3615 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN) 3616 return -EINVAL; 3617 3618 return i; 3619 } 3620 3621 return -ENOENT; 3622 } 3623 3624 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) { 3625 const struct btf_var_secinfo *vs; 3626 const struct btf_type *t; 3627 int i, j, n; 3628 3629 if (!btf) 3630 return -ESRCH; 3631 3632 n = btf__type_cnt(btf); 3633 for (i = 1; i < n; i++) { 3634 t = btf__type_by_id(btf, i); 3635 3636 if (!btf_is_datasec(t)) 3637 continue; 3638 3639 vs = btf_var_secinfos(t); 3640 for (j = 0; j < btf_vlen(t); j++, vs++) { 3641 if (vs->type == ext_btf_id) 3642 return i; 3643 } 3644 } 3645 3646 return -ENOENT; 3647 } 3648 3649 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, 3650 bool *is_signed) 3651 { 3652 const struct btf_type *t; 3653 const char *name; 3654 3655 t = skip_mods_and_typedefs(btf, id, NULL); 3656 name = btf__name_by_offset(btf, t->name_off); 3657 3658 if (is_signed) 3659 *is_signed = false; 3660 switch (btf_kind(t)) { 3661 case BTF_KIND_INT: { 3662 int enc = btf_int_encoding(t); 3663 3664 if (enc & BTF_INT_BOOL) 3665 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; 3666 if (is_signed) 3667 *is_signed = enc & BTF_INT_SIGNED; 3668 if (t->size == 1) 3669 return KCFG_CHAR; 3670 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) 3671 return KCFG_UNKNOWN; 3672 return KCFG_INT; 3673 } 3674 case BTF_KIND_ENUM: 3675 if (t->size != 4) 3676 return KCFG_UNKNOWN; 3677 if (strcmp(name, "libbpf_tristate")) 3678 return KCFG_UNKNOWN; 3679 return KCFG_TRISTATE; 3680 case BTF_KIND_ENUM64: 3681 if (strcmp(name, "libbpf_tristate")) 3682 return KCFG_UNKNOWN; 3683 return KCFG_TRISTATE; 3684 case BTF_KIND_ARRAY: 3685 if (btf_array(t)->nelems == 0) 3686 return KCFG_UNKNOWN; 3687 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) 3688 return KCFG_UNKNOWN; 3689 return KCFG_CHAR_ARR; 3690 default: 3691 return KCFG_UNKNOWN; 3692 } 3693 } 3694 3695 static int cmp_externs(const void *_a, const void *_b) 3696 { 3697 const struct extern_desc *a = _a; 3698 const struct extern_desc *b = _b; 3699 3700 if (a->type != b->type) 3701 return a->type < b->type ? -1 : 1; 3702 3703 if (a->type == EXT_KCFG) { 3704 /* descending order by alignment requirements */ 3705 if (a->kcfg.align != b->kcfg.align) 3706 return a->kcfg.align > b->kcfg.align ? -1 : 1; 3707 /* ascending order by size, within same alignment class */ 3708 if (a->kcfg.sz != b->kcfg.sz) 3709 return a->kcfg.sz < b->kcfg.sz ? -1 : 1; 3710 } 3711 3712 /* resolve ties by name */ 3713 return strcmp(a->name, b->name); 3714 } 3715 3716 static int find_int_btf_id(const struct btf *btf) 3717 { 3718 const struct btf_type *t; 3719 int i, n; 3720 3721 n = btf__type_cnt(btf); 3722 for (i = 1; i < n; i++) { 3723 t = btf__type_by_id(btf, i); 3724 3725 if (btf_is_int(t) && btf_int_bits(t) == 32) 3726 return i; 3727 } 3728 3729 return 0; 3730 } 3731 3732 static int add_dummy_ksym_var(struct btf *btf) 3733 { 3734 int i, int_btf_id, sec_btf_id, dummy_var_btf_id; 3735 const struct btf_var_secinfo *vs; 3736 const struct btf_type *sec; 3737 3738 if (!btf) 3739 return 0; 3740 3741 sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC, 3742 BTF_KIND_DATASEC); 3743 if (sec_btf_id < 0) 3744 return 0; 3745 3746 sec = btf__type_by_id(btf, sec_btf_id); 3747 vs = btf_var_secinfos(sec); 3748 for (i = 0; i < btf_vlen(sec); i++, vs++) { 3749 const struct btf_type *vt; 3750 3751 vt = btf__type_by_id(btf, vs->type); 3752 if (btf_is_func(vt)) 3753 break; 3754 } 3755 3756 /* No func in ksyms sec. No need to add dummy var. */ 3757 if (i == btf_vlen(sec)) 3758 return 0; 3759 3760 int_btf_id = find_int_btf_id(btf); 3761 dummy_var_btf_id = btf__add_var(btf, 3762 "dummy_ksym", 3763 BTF_VAR_GLOBAL_ALLOCATED, 3764 int_btf_id); 3765 if (dummy_var_btf_id < 0) 3766 pr_warn("cannot create a dummy_ksym var\n"); 3767 3768 return dummy_var_btf_id; 3769 } 3770 3771 static int bpf_object__collect_externs(struct bpf_object *obj) 3772 { 3773 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL; 3774 const struct btf_type *t; 3775 struct extern_desc *ext; 3776 int i, n, off, dummy_var_btf_id; 3777 const char *ext_name, *sec_name; 3778 size_t ext_essent_len; 3779 Elf_Scn *scn; 3780 Elf64_Shdr *sh; 3781 3782 if (!obj->efile.symbols) 3783 return 0; 3784 3785 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); 3786 sh = elf_sec_hdr(obj, scn); 3787 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym)) 3788 return -LIBBPF_ERRNO__FORMAT; 3789 3790 dummy_var_btf_id = add_dummy_ksym_var(obj->btf); 3791 if (dummy_var_btf_id < 0) 3792 return dummy_var_btf_id; 3793 3794 n = sh->sh_size / sh->sh_entsize; 3795 pr_debug("looking for externs among %d symbols...\n", n); 3796 3797 for (i = 0; i < n; i++) { 3798 Elf64_Sym *sym = elf_sym_by_idx(obj, i); 3799 3800 if (!sym) 3801 return -LIBBPF_ERRNO__FORMAT; 3802 if (!sym_is_extern(sym)) 3803 continue; 3804 ext_name = elf_sym_str(obj, sym->st_name); 3805 if (!ext_name || !ext_name[0]) 3806 continue; 3807 3808 ext = obj->externs; 3809 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); 3810 if (!ext) 3811 return -ENOMEM; 3812 obj->externs = ext; 3813 ext = &ext[obj->nr_extern]; 3814 memset(ext, 0, sizeof(*ext)); 3815 obj->nr_extern++; 3816 3817 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); 3818 if (ext->btf_id <= 0) { 3819 pr_warn("failed to find BTF for extern '%s': %d\n", 3820 ext_name, ext->btf_id); 3821 return ext->btf_id; 3822 } 3823 t = btf__type_by_id(obj->btf, ext->btf_id); 3824 ext->name = btf__name_by_offset(obj->btf, t->name_off); 3825 ext->sym_idx = i; 3826 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK; 3827 3828 ext_essent_len = bpf_core_essential_name_len(ext->name); 3829 ext->essent_name = NULL; 3830 if (ext_essent_len != strlen(ext->name)) { 3831 ext->essent_name = strndup(ext->name, ext_essent_len); 3832 if (!ext->essent_name) 3833 return -ENOMEM; 3834 } 3835 3836 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); 3837 if (ext->sec_btf_id <= 0) { 3838 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n", 3839 ext_name, ext->btf_id, ext->sec_btf_id); 3840 return ext->sec_btf_id; 3841 } 3842 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); 3843 sec_name = btf__name_by_offset(obj->btf, sec->name_off); 3844 3845 if (strcmp(sec_name, KCONFIG_SEC) == 0) { 3846 if (btf_is_func(t)) { 3847 pr_warn("extern function %s is unsupported under %s section\n", 3848 ext->name, KCONFIG_SEC); 3849 return -ENOTSUP; 3850 } 3851 kcfg_sec = sec; 3852 ext->type = EXT_KCFG; 3853 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); 3854 if (ext->kcfg.sz <= 0) { 3855 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n", 3856 ext_name, ext->kcfg.sz); 3857 return ext->kcfg.sz; 3858 } 3859 ext->kcfg.align = btf__align_of(obj->btf, t->type); 3860 if (ext->kcfg.align <= 0) { 3861 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n", 3862 ext_name, ext->kcfg.align); 3863 return -EINVAL; 3864 } 3865 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, 3866 &ext->kcfg.is_signed); 3867 if (ext->kcfg.type == KCFG_UNKNOWN) { 3868 pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name); 3869 return -ENOTSUP; 3870 } 3871 } else if (strcmp(sec_name, KSYMS_SEC) == 0) { 3872 ksym_sec = sec; 3873 ext->type = EXT_KSYM; 3874 skip_mods_and_typedefs(obj->btf, t->type, 3875 &ext->ksym.type_id); 3876 } else { 3877 pr_warn("unrecognized extern section '%s'\n", sec_name); 3878 return -ENOTSUP; 3879 } 3880 } 3881 pr_debug("collected %d externs total\n", obj->nr_extern); 3882 3883 if (!obj->nr_extern) 3884 return 0; 3885 3886 /* sort externs by type, for kcfg ones also by (align, size, name) */ 3887 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); 3888 3889 /* for .ksyms section, we need to turn all externs into allocated 3890 * variables in BTF to pass kernel verification; we do this by 3891 * pretending that each extern is a 8-byte variable 3892 */ 3893 if (ksym_sec) { 3894 /* find existing 4-byte integer type in BTF to use for fake 3895 * extern variables in DATASEC 3896 */ 3897 int int_btf_id = find_int_btf_id(obj->btf); 3898 /* For extern function, a dummy_var added earlier 3899 * will be used to replace the vs->type and 3900 * its name string will be used to refill 3901 * the missing param's name. 3902 */ 3903 const struct btf_type *dummy_var; 3904 3905 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); 3906 for (i = 0; i < obj->nr_extern; i++) { 3907 ext = &obj->externs[i]; 3908 if (ext->type != EXT_KSYM) 3909 continue; 3910 pr_debug("extern (ksym) #%d: symbol %d, name %s\n", 3911 i, ext->sym_idx, ext->name); 3912 } 3913 3914 sec = ksym_sec; 3915 n = btf_vlen(sec); 3916 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) { 3917 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 3918 struct btf_type *vt; 3919 3920 vt = (void *)btf__type_by_id(obj->btf, vs->type); 3921 ext_name = btf__name_by_offset(obj->btf, vt->name_off); 3922 ext = find_extern_by_name(obj, ext_name); 3923 if (!ext) { 3924 pr_warn("failed to find extern definition for BTF %s '%s'\n", 3925 btf_kind_str(vt), ext_name); 3926 return -ESRCH; 3927 } 3928 if (btf_is_func(vt)) { 3929 const struct btf_type *func_proto; 3930 struct btf_param *param; 3931 int j; 3932 3933 func_proto = btf__type_by_id(obj->btf, 3934 vt->type); 3935 param = btf_params(func_proto); 3936 /* Reuse the dummy_var string if the 3937 * func proto does not have param name. 3938 */ 3939 for (j = 0; j < btf_vlen(func_proto); j++) 3940 if (param[j].type && !param[j].name_off) 3941 param[j].name_off = 3942 dummy_var->name_off; 3943 vs->type = dummy_var_btf_id; 3944 vt->info &= ~0xffff; 3945 vt->info |= BTF_FUNC_GLOBAL; 3946 } else { 3947 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 3948 vt->type = int_btf_id; 3949 } 3950 vs->offset = off; 3951 vs->size = sizeof(int); 3952 } 3953 sec->size = off; 3954 } 3955 3956 if (kcfg_sec) { 3957 sec = kcfg_sec; 3958 /* for kcfg externs calculate their offsets within a .kconfig map */ 3959 off = 0; 3960 for (i = 0; i < obj->nr_extern; i++) { 3961 ext = &obj->externs[i]; 3962 if (ext->type != EXT_KCFG) 3963 continue; 3964 3965 ext->kcfg.data_off = roundup(off, ext->kcfg.align); 3966 off = ext->kcfg.data_off + ext->kcfg.sz; 3967 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", 3968 i, ext->sym_idx, ext->kcfg.data_off, ext->name); 3969 } 3970 sec->size = off; 3971 n = btf_vlen(sec); 3972 for (i = 0; i < n; i++) { 3973 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 3974 3975 t = btf__type_by_id(obj->btf, vs->type); 3976 ext_name = btf__name_by_offset(obj->btf, t->name_off); 3977 ext = find_extern_by_name(obj, ext_name); 3978 if (!ext) { 3979 pr_warn("failed to find extern definition for BTF var '%s'\n", 3980 ext_name); 3981 return -ESRCH; 3982 } 3983 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 3984 vs->offset = ext->kcfg.data_off; 3985 } 3986 } 3987 return 0; 3988 } 3989 3990 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog) 3991 { 3992 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1; 3993 } 3994 3995 struct bpf_program * 3996 bpf_object__find_program_by_name(const struct bpf_object *obj, 3997 const char *name) 3998 { 3999 struct bpf_program *prog; 4000 4001 bpf_object__for_each_program(prog, obj) { 4002 if (prog_is_subprog(obj, prog)) 4003 continue; 4004 if (!strcmp(prog->name, name)) 4005 return prog; 4006 } 4007 return errno = ENOENT, NULL; 4008 } 4009 4010 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 4011 int shndx) 4012 { 4013 switch (obj->efile.secs[shndx].sec_type) { 4014 case SEC_BSS: 4015 case SEC_DATA: 4016 case SEC_RODATA: 4017 return true; 4018 default: 4019 return false; 4020 } 4021 } 4022 4023 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 4024 int shndx) 4025 { 4026 return shndx == obj->efile.btf_maps_shndx; 4027 } 4028 4029 static enum libbpf_map_type 4030 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 4031 { 4032 if (shndx == obj->efile.symbols_shndx) 4033 return LIBBPF_MAP_KCONFIG; 4034 4035 switch (obj->efile.secs[shndx].sec_type) { 4036 case SEC_BSS: 4037 return LIBBPF_MAP_BSS; 4038 case SEC_DATA: 4039 return LIBBPF_MAP_DATA; 4040 case SEC_RODATA: 4041 return LIBBPF_MAP_RODATA; 4042 default: 4043 return LIBBPF_MAP_UNSPEC; 4044 } 4045 } 4046 4047 static int bpf_program__record_reloc(struct bpf_program *prog, 4048 struct reloc_desc *reloc_desc, 4049 __u32 insn_idx, const char *sym_name, 4050 const Elf64_Sym *sym, const Elf64_Rel *rel) 4051 { 4052 struct bpf_insn *insn = &prog->insns[insn_idx]; 4053 size_t map_idx, nr_maps = prog->obj->nr_maps; 4054 struct bpf_object *obj = prog->obj; 4055 __u32 shdr_idx = sym->st_shndx; 4056 enum libbpf_map_type type; 4057 const char *sym_sec_name; 4058 struct bpf_map *map; 4059 4060 if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { 4061 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", 4062 prog->name, sym_name, insn_idx, insn->code); 4063 return -LIBBPF_ERRNO__RELOC; 4064 } 4065 4066 if (sym_is_extern(sym)) { 4067 int sym_idx = ELF64_R_SYM(rel->r_info); 4068 int i, n = obj->nr_extern; 4069 struct extern_desc *ext; 4070 4071 for (i = 0; i < n; i++) { 4072 ext = &obj->externs[i]; 4073 if (ext->sym_idx == sym_idx) 4074 break; 4075 } 4076 if (i >= n) { 4077 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n", 4078 prog->name, sym_name, sym_idx); 4079 return -LIBBPF_ERRNO__RELOC; 4080 } 4081 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n", 4082 prog->name, i, ext->name, ext->sym_idx, insn_idx); 4083 if (insn->code == (BPF_JMP | BPF_CALL)) 4084 reloc_desc->type = RELO_EXTERN_CALL; 4085 else 4086 reloc_desc->type = RELO_EXTERN_LD64; 4087 reloc_desc->insn_idx = insn_idx; 4088 reloc_desc->ext_idx = i; 4089 return 0; 4090 } 4091 4092 /* sub-program call relocation */ 4093 if (is_call_insn(insn)) { 4094 if (insn->src_reg != BPF_PSEUDO_CALL) { 4095 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); 4096 return -LIBBPF_ERRNO__RELOC; 4097 } 4098 /* text_shndx can be 0, if no default "main" program exists */ 4099 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { 4100 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 4101 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n", 4102 prog->name, sym_name, sym_sec_name); 4103 return -LIBBPF_ERRNO__RELOC; 4104 } 4105 if (sym->st_value % BPF_INSN_SZ) { 4106 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n", 4107 prog->name, sym_name, (size_t)sym->st_value); 4108 return -LIBBPF_ERRNO__RELOC; 4109 } 4110 reloc_desc->type = RELO_CALL; 4111 reloc_desc->insn_idx = insn_idx; 4112 reloc_desc->sym_off = sym->st_value; 4113 return 0; 4114 } 4115 4116 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { 4117 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n", 4118 prog->name, sym_name, shdr_idx); 4119 return -LIBBPF_ERRNO__RELOC; 4120 } 4121 4122 /* loading subprog addresses */ 4123 if (sym_is_subprog(sym, obj->efile.text_shndx)) { 4124 /* global_func: sym->st_value = offset in the section, insn->imm = 0. 4125 * local_func: sym->st_value = 0, insn->imm = offset in the section. 4126 */ 4127 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { 4128 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n", 4129 prog->name, sym_name, (size_t)sym->st_value, insn->imm); 4130 return -LIBBPF_ERRNO__RELOC; 4131 } 4132 4133 reloc_desc->type = RELO_SUBPROG_ADDR; 4134 reloc_desc->insn_idx = insn_idx; 4135 reloc_desc->sym_off = sym->st_value; 4136 return 0; 4137 } 4138 4139 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 4140 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 4141 4142 /* generic map reference relocation */ 4143 if (type == LIBBPF_MAP_UNSPEC) { 4144 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { 4145 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n", 4146 prog->name, sym_name, sym_sec_name); 4147 return -LIBBPF_ERRNO__RELOC; 4148 } 4149 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 4150 map = &obj->maps[map_idx]; 4151 if (map->libbpf_type != type || 4152 map->sec_idx != sym->st_shndx || 4153 map->sec_offset != sym->st_value) 4154 continue; 4155 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", 4156 prog->name, map_idx, map->name, map->sec_idx, 4157 map->sec_offset, insn_idx); 4158 break; 4159 } 4160 if (map_idx >= nr_maps) { 4161 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n", 4162 prog->name, sym_sec_name, (size_t)sym->st_value); 4163 return -LIBBPF_ERRNO__RELOC; 4164 } 4165 reloc_desc->type = RELO_LD64; 4166 reloc_desc->insn_idx = insn_idx; 4167 reloc_desc->map_idx = map_idx; 4168 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ 4169 return 0; 4170 } 4171 4172 /* global data map relocation */ 4173 if (!bpf_object__shndx_is_data(obj, shdr_idx)) { 4174 pr_warn("prog '%s': bad data relo against section '%s'\n", 4175 prog->name, sym_sec_name); 4176 return -LIBBPF_ERRNO__RELOC; 4177 } 4178 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 4179 map = &obj->maps[map_idx]; 4180 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx) 4181 continue; 4182 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", 4183 prog->name, map_idx, map->name, map->sec_idx, 4184 map->sec_offset, insn_idx); 4185 break; 4186 } 4187 if (map_idx >= nr_maps) { 4188 pr_warn("prog '%s': data relo failed to find map for section '%s'\n", 4189 prog->name, sym_sec_name); 4190 return -LIBBPF_ERRNO__RELOC; 4191 } 4192 4193 reloc_desc->type = RELO_DATA; 4194 reloc_desc->insn_idx = insn_idx; 4195 reloc_desc->map_idx = map_idx; 4196 reloc_desc->sym_off = sym->st_value; 4197 return 0; 4198 } 4199 4200 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx) 4201 { 4202 return insn_idx >= prog->sec_insn_off && 4203 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; 4204 } 4205 4206 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, 4207 size_t sec_idx, size_t insn_idx) 4208 { 4209 int l = 0, r = obj->nr_programs - 1, m; 4210 struct bpf_program *prog; 4211 4212 if (!obj->nr_programs) 4213 return NULL; 4214 4215 while (l < r) { 4216 m = l + (r - l + 1) / 2; 4217 prog = &obj->programs[m]; 4218 4219 if (prog->sec_idx < sec_idx || 4220 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) 4221 l = m; 4222 else 4223 r = m - 1; 4224 } 4225 /* matching program could be at index l, but it still might be the 4226 * wrong one, so we need to double check conditions for the last time 4227 */ 4228 prog = &obj->programs[l]; 4229 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) 4230 return prog; 4231 return NULL; 4232 } 4233 4234 static int 4235 bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) 4236 { 4237 const char *relo_sec_name, *sec_name; 4238 size_t sec_idx = shdr->sh_info, sym_idx; 4239 struct bpf_program *prog; 4240 struct reloc_desc *relos; 4241 int err, i, nrels; 4242 const char *sym_name; 4243 __u32 insn_idx; 4244 Elf_Scn *scn; 4245 Elf_Data *scn_data; 4246 Elf64_Sym *sym; 4247 Elf64_Rel *rel; 4248 4249 if (sec_idx >= obj->efile.sec_cnt) 4250 return -EINVAL; 4251 4252 scn = elf_sec_by_idx(obj, sec_idx); 4253 scn_data = elf_sec_data(obj, scn); 4254 4255 relo_sec_name = elf_sec_str(obj, shdr->sh_name); 4256 sec_name = elf_sec_name(obj, scn); 4257 if (!relo_sec_name || !sec_name) 4258 return -EINVAL; 4259 4260 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n", 4261 relo_sec_name, sec_idx, sec_name); 4262 nrels = shdr->sh_size / shdr->sh_entsize; 4263 4264 for (i = 0; i < nrels; i++) { 4265 rel = elf_rel_by_idx(data, i); 4266 if (!rel) { 4267 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); 4268 return -LIBBPF_ERRNO__FORMAT; 4269 } 4270 4271 sym_idx = ELF64_R_SYM(rel->r_info); 4272 sym = elf_sym_by_idx(obj, sym_idx); 4273 if (!sym) { 4274 pr_warn("sec '%s': symbol #%zu not found for relo #%d\n", 4275 relo_sec_name, sym_idx, i); 4276 return -LIBBPF_ERRNO__FORMAT; 4277 } 4278 4279 if (sym->st_shndx >= obj->efile.sec_cnt) { 4280 pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n", 4281 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i); 4282 return -LIBBPF_ERRNO__FORMAT; 4283 } 4284 4285 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) { 4286 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n", 4287 relo_sec_name, (size_t)rel->r_offset, i); 4288 return -LIBBPF_ERRNO__FORMAT; 4289 } 4290 4291 insn_idx = rel->r_offset / BPF_INSN_SZ; 4292 /* relocations against static functions are recorded as 4293 * relocations against the section that contains a function; 4294 * in such case, symbol will be STT_SECTION and sym.st_name 4295 * will point to empty string (0), so fetch section name 4296 * instead 4297 */ 4298 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0) 4299 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)); 4300 else 4301 sym_name = elf_sym_str(obj, sym->st_name); 4302 sym_name = sym_name ?: "<?"; 4303 4304 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n", 4305 relo_sec_name, i, insn_idx, sym_name); 4306 4307 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 4308 if (!prog) { 4309 pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n", 4310 relo_sec_name, i, sec_name, insn_idx); 4311 continue; 4312 } 4313 4314 relos = libbpf_reallocarray(prog->reloc_desc, 4315 prog->nr_reloc + 1, sizeof(*relos)); 4316 if (!relos) 4317 return -ENOMEM; 4318 prog->reloc_desc = relos; 4319 4320 /* adjust insn_idx to local BPF program frame of reference */ 4321 insn_idx -= prog->sec_insn_off; 4322 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], 4323 insn_idx, sym_name, sym, rel); 4324 if (err) 4325 return err; 4326 4327 prog->nr_reloc++; 4328 } 4329 return 0; 4330 } 4331 4332 static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map) 4333 { 4334 int id; 4335 4336 if (!obj->btf) 4337 return -ENOENT; 4338 4339 /* if it's BTF-defined map, we don't need to search for type IDs. 4340 * For struct_ops map, it does not need btf_key_type_id and 4341 * btf_value_type_id. 4342 */ 4343 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map)) 4344 return 0; 4345 4346 /* 4347 * LLVM annotates global data differently in BTF, that is, 4348 * only as '.data', '.bss' or '.rodata'. 4349 */ 4350 if (!bpf_map__is_internal(map)) 4351 return -ENOENT; 4352 4353 id = btf__find_by_name(obj->btf, map->real_name); 4354 if (id < 0) 4355 return id; 4356 4357 map->btf_key_type_id = 0; 4358 map->btf_value_type_id = id; 4359 return 0; 4360 } 4361 4362 static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) 4363 { 4364 char file[PATH_MAX], buff[4096]; 4365 FILE *fp; 4366 __u32 val; 4367 int err; 4368 4369 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd); 4370 memset(info, 0, sizeof(*info)); 4371 4372 fp = fopen(file, "re"); 4373 if (!fp) { 4374 err = -errno; 4375 pr_warn("failed to open %s: %d. No procfs support?\n", file, 4376 err); 4377 return err; 4378 } 4379 4380 while (fgets(buff, sizeof(buff), fp)) { 4381 if (sscanf(buff, "map_type:\t%u", &val) == 1) 4382 info->type = val; 4383 else if (sscanf(buff, "key_size:\t%u", &val) == 1) 4384 info->key_size = val; 4385 else if (sscanf(buff, "value_size:\t%u", &val) == 1) 4386 info->value_size = val; 4387 else if (sscanf(buff, "max_entries:\t%u", &val) == 1) 4388 info->max_entries = val; 4389 else if (sscanf(buff, "map_flags:\t%i", &val) == 1) 4390 info->map_flags = val; 4391 } 4392 4393 fclose(fp); 4394 4395 return 0; 4396 } 4397 4398 bool bpf_map__autocreate(const struct bpf_map *map) 4399 { 4400 return map->autocreate; 4401 } 4402 4403 int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) 4404 { 4405 if (map->obj->loaded) 4406 return libbpf_err(-EBUSY); 4407 4408 map->autocreate = autocreate; 4409 return 0; 4410 } 4411 4412 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 4413 { 4414 struct bpf_map_info info; 4415 __u32 len = sizeof(info), name_len; 4416 int new_fd, err; 4417 char *new_name; 4418 4419 memset(&info, 0, len); 4420 err = bpf_map_get_info_by_fd(fd, &info, &len); 4421 if (err && errno == EINVAL) 4422 err = bpf_get_map_info_from_fdinfo(fd, &info); 4423 if (err) 4424 return libbpf_err(err); 4425 4426 name_len = strlen(info.name); 4427 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0) 4428 new_name = strdup(map->name); 4429 else 4430 new_name = strdup(info.name); 4431 4432 if (!new_name) 4433 return libbpf_err(-errno); 4434 4435 /* 4436 * Like dup(), but make sure new FD is >= 3 and has O_CLOEXEC set. 4437 * This is similar to what we do in ensure_good_fd(), but without 4438 * closing original FD. 4439 */ 4440 new_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); 4441 if (new_fd < 0) { 4442 err = -errno; 4443 goto err_free_new_name; 4444 } 4445 4446 err = zclose(map->fd); 4447 if (err) { 4448 err = -errno; 4449 goto err_close_new_fd; 4450 } 4451 free(map->name); 4452 4453 map->fd = new_fd; 4454 map->name = new_name; 4455 map->def.type = info.type; 4456 map->def.key_size = info.key_size; 4457 map->def.value_size = info.value_size; 4458 map->def.max_entries = info.max_entries; 4459 map->def.map_flags = info.map_flags; 4460 map->btf_key_type_id = info.btf_key_type_id; 4461 map->btf_value_type_id = info.btf_value_type_id; 4462 map->reused = true; 4463 map->map_extra = info.map_extra; 4464 4465 return 0; 4466 4467 err_close_new_fd: 4468 close(new_fd); 4469 err_free_new_name: 4470 free(new_name); 4471 return libbpf_err(err); 4472 } 4473 4474 __u32 bpf_map__max_entries(const struct bpf_map *map) 4475 { 4476 return map->def.max_entries; 4477 } 4478 4479 struct bpf_map *bpf_map__inner_map(struct bpf_map *map) 4480 { 4481 if (!bpf_map_type__is_map_in_map(map->def.type)) 4482 return errno = EINVAL, NULL; 4483 4484 return map->inner_map; 4485 } 4486 4487 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) 4488 { 4489 if (map->obj->loaded) 4490 return libbpf_err(-EBUSY); 4491 4492 map->def.max_entries = max_entries; 4493 4494 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ 4495 if (map_is_ringbuf(map)) 4496 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); 4497 4498 return 0; 4499 } 4500 4501 static int 4502 bpf_object__probe_loading(struct bpf_object *obj) 4503 { 4504 char *cp, errmsg[STRERR_BUFSIZE]; 4505 struct bpf_insn insns[] = { 4506 BPF_MOV64_IMM(BPF_REG_0, 0), 4507 BPF_EXIT_INSN(), 4508 }; 4509 int ret, insn_cnt = ARRAY_SIZE(insns); 4510 4511 if (obj->gen_loader) 4512 return 0; 4513 4514 ret = bump_rlimit_memlock(); 4515 if (ret) 4516 pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret); 4517 4518 /* make sure basic loading works */ 4519 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); 4520 if (ret < 0) 4521 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); 4522 if (ret < 0) { 4523 ret = errno; 4524 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4525 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF " 4526 "program. Make sure your kernel supports BPF " 4527 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is " 4528 "set to big enough value.\n", __func__, cp, ret); 4529 return -ret; 4530 } 4531 close(ret); 4532 4533 return 0; 4534 } 4535 4536 static int probe_fd(int fd) 4537 { 4538 if (fd >= 0) 4539 close(fd); 4540 return fd >= 0; 4541 } 4542 4543 static int probe_kern_prog_name(void) 4544 { 4545 const size_t attr_sz = offsetofend(union bpf_attr, prog_name); 4546 struct bpf_insn insns[] = { 4547 BPF_MOV64_IMM(BPF_REG_0, 0), 4548 BPF_EXIT_INSN(), 4549 }; 4550 union bpf_attr attr; 4551 int ret; 4552 4553 memset(&attr, 0, attr_sz); 4554 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4555 attr.license = ptr_to_u64("GPL"); 4556 attr.insns = ptr_to_u64(insns); 4557 attr.insn_cnt = (__u32)ARRAY_SIZE(insns); 4558 libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name)); 4559 4560 /* make sure loading with name works */ 4561 ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS); 4562 return probe_fd(ret); 4563 } 4564 4565 static int probe_kern_global_data(void) 4566 { 4567 char *cp, errmsg[STRERR_BUFSIZE]; 4568 struct bpf_insn insns[] = { 4569 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 4570 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 4571 BPF_MOV64_IMM(BPF_REG_0, 0), 4572 BPF_EXIT_INSN(), 4573 }; 4574 int ret, map, insn_cnt = ARRAY_SIZE(insns); 4575 4576 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL); 4577 if (map < 0) { 4578 ret = -errno; 4579 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4580 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 4581 __func__, cp, -ret); 4582 return ret; 4583 } 4584 4585 insns[0].imm = map; 4586 4587 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); 4588 close(map); 4589 return probe_fd(ret); 4590 } 4591 4592 static int probe_kern_btf(void) 4593 { 4594 static const char strs[] = "\0int"; 4595 __u32 types[] = { 4596 /* int */ 4597 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 4598 }; 4599 4600 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4601 strs, sizeof(strs))); 4602 } 4603 4604 static int probe_kern_btf_func(void) 4605 { 4606 static const char strs[] = "\0int\0x\0a"; 4607 /* void x(int a) {} */ 4608 __u32 types[] = { 4609 /* int */ 4610 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4611 /* FUNC_PROTO */ /* [2] */ 4612 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 4613 BTF_PARAM_ENC(7, 1), 4614 /* FUNC x */ /* [3] */ 4615 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 4616 }; 4617 4618 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4619 strs, sizeof(strs))); 4620 } 4621 4622 static int probe_kern_btf_func_global(void) 4623 { 4624 static const char strs[] = "\0int\0x\0a"; 4625 /* static void x(int a) {} */ 4626 __u32 types[] = { 4627 /* int */ 4628 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4629 /* FUNC_PROTO */ /* [2] */ 4630 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 4631 BTF_PARAM_ENC(7, 1), 4632 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ 4633 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), 4634 }; 4635 4636 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4637 strs, sizeof(strs))); 4638 } 4639 4640 static int probe_kern_btf_datasec(void) 4641 { 4642 static const char strs[] = "\0x\0.data"; 4643 /* static int a; */ 4644 __u32 types[] = { 4645 /* int */ 4646 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4647 /* VAR x */ /* [2] */ 4648 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 4649 BTF_VAR_STATIC, 4650 /* DATASEC val */ /* [3] */ 4651 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 4652 BTF_VAR_SECINFO_ENC(2, 0, 4), 4653 }; 4654 4655 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4656 strs, sizeof(strs))); 4657 } 4658 4659 static int probe_kern_btf_float(void) 4660 { 4661 static const char strs[] = "\0float"; 4662 __u32 types[] = { 4663 /* float */ 4664 BTF_TYPE_FLOAT_ENC(1, 4), 4665 }; 4666 4667 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4668 strs, sizeof(strs))); 4669 } 4670 4671 static int probe_kern_btf_decl_tag(void) 4672 { 4673 static const char strs[] = "\0tag"; 4674 __u32 types[] = { 4675 /* int */ 4676 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4677 /* VAR x */ /* [2] */ 4678 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 4679 BTF_VAR_STATIC, 4680 /* attr */ 4681 BTF_TYPE_DECL_TAG_ENC(1, 2, -1), 4682 }; 4683 4684 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4685 strs, sizeof(strs))); 4686 } 4687 4688 static int probe_kern_btf_type_tag(void) 4689 { 4690 static const char strs[] = "\0tag"; 4691 __u32 types[] = { 4692 /* int */ 4693 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 4694 /* attr */ 4695 BTF_TYPE_TYPE_TAG_ENC(1, 1), /* [2] */ 4696 /* ptr */ 4697 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2), /* [3] */ 4698 }; 4699 4700 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4701 strs, sizeof(strs))); 4702 } 4703 4704 static int probe_kern_array_mmap(void) 4705 { 4706 LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); 4707 int fd; 4708 4709 fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts); 4710 return probe_fd(fd); 4711 } 4712 4713 static int probe_kern_exp_attach_type(void) 4714 { 4715 LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE); 4716 struct bpf_insn insns[] = { 4717 BPF_MOV64_IMM(BPF_REG_0, 0), 4718 BPF_EXIT_INSN(), 4719 }; 4720 int fd, insn_cnt = ARRAY_SIZE(insns); 4721 4722 /* use any valid combination of program type and (optional) 4723 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) 4724 * to see if kernel supports expected_attach_type field for 4725 * BPF_PROG_LOAD command 4726 */ 4727 fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts); 4728 return probe_fd(fd); 4729 } 4730 4731 static int probe_kern_probe_read_kernel(void) 4732 { 4733 struct bpf_insn insns[] = { 4734 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ 4735 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ 4736 BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ 4737 BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ 4738 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), 4739 BPF_EXIT_INSN(), 4740 }; 4741 int fd, insn_cnt = ARRAY_SIZE(insns); 4742 4743 fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); 4744 return probe_fd(fd); 4745 } 4746 4747 static int probe_prog_bind_map(void) 4748 { 4749 char *cp, errmsg[STRERR_BUFSIZE]; 4750 struct bpf_insn insns[] = { 4751 BPF_MOV64_IMM(BPF_REG_0, 0), 4752 BPF_EXIT_INSN(), 4753 }; 4754 int ret, map, prog, insn_cnt = ARRAY_SIZE(insns); 4755 4756 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL); 4757 if (map < 0) { 4758 ret = -errno; 4759 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4760 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 4761 __func__, cp, -ret); 4762 return ret; 4763 } 4764 4765 prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); 4766 if (prog < 0) { 4767 close(map); 4768 return 0; 4769 } 4770 4771 ret = bpf_prog_bind_map(prog, map, NULL); 4772 4773 close(map); 4774 close(prog); 4775 4776 return ret >= 0; 4777 } 4778 4779 static int probe_module_btf(void) 4780 { 4781 static const char strs[] = "\0int"; 4782 __u32 types[] = { 4783 /* int */ 4784 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 4785 }; 4786 struct bpf_btf_info info; 4787 __u32 len = sizeof(info); 4788 char name[16]; 4789 int fd, err; 4790 4791 fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs)); 4792 if (fd < 0) 4793 return 0; /* BTF not supported at all */ 4794 4795 memset(&info, 0, sizeof(info)); 4796 info.name = ptr_to_u64(name); 4797 info.name_len = sizeof(name); 4798 4799 /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; 4800 * kernel's module BTF support coincides with support for 4801 * name/name_len fields in struct bpf_btf_info. 4802 */ 4803 err = bpf_btf_get_info_by_fd(fd, &info, &len); 4804 close(fd); 4805 return !err; 4806 } 4807 4808 static int probe_perf_link(void) 4809 { 4810 struct bpf_insn insns[] = { 4811 BPF_MOV64_IMM(BPF_REG_0, 0), 4812 BPF_EXIT_INSN(), 4813 }; 4814 int prog_fd, link_fd, err; 4815 4816 prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", 4817 insns, ARRAY_SIZE(insns), NULL); 4818 if (prog_fd < 0) 4819 return -errno; 4820 4821 /* use invalid perf_event FD to get EBADF, if link is supported; 4822 * otherwise EINVAL should be returned 4823 */ 4824 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); 4825 err = -errno; /* close() can clobber errno */ 4826 4827 if (link_fd >= 0) 4828 close(link_fd); 4829 close(prog_fd); 4830 4831 return link_fd < 0 && err == -EBADF; 4832 } 4833 4834 static int probe_uprobe_multi_link(void) 4835 { 4836 LIBBPF_OPTS(bpf_prog_load_opts, load_opts, 4837 .expected_attach_type = BPF_TRACE_UPROBE_MULTI, 4838 ); 4839 LIBBPF_OPTS(bpf_link_create_opts, link_opts); 4840 struct bpf_insn insns[] = { 4841 BPF_MOV64_IMM(BPF_REG_0, 0), 4842 BPF_EXIT_INSN(), 4843 }; 4844 int prog_fd, link_fd, err; 4845 unsigned long offset = 0; 4846 4847 prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", 4848 insns, ARRAY_SIZE(insns), &load_opts); 4849 if (prog_fd < 0) 4850 return -errno; 4851 4852 /* Creating uprobe in '/' binary should fail with -EBADF. */ 4853 link_opts.uprobe_multi.path = "/"; 4854 link_opts.uprobe_multi.offsets = &offset; 4855 link_opts.uprobe_multi.cnt = 1; 4856 4857 link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts); 4858 err = -errno; /* close() can clobber errno */ 4859 4860 if (link_fd >= 0) 4861 close(link_fd); 4862 close(prog_fd); 4863 4864 return link_fd < 0 && err == -EBADF; 4865 } 4866 4867 static int probe_kern_bpf_cookie(void) 4868 { 4869 struct bpf_insn insns[] = { 4870 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie), 4871 BPF_EXIT_INSN(), 4872 }; 4873 int ret, insn_cnt = ARRAY_SIZE(insns); 4874 4875 ret = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL); 4876 return probe_fd(ret); 4877 } 4878 4879 static int probe_kern_btf_enum64(void) 4880 { 4881 static const char strs[] = "\0enum64"; 4882 __u32 types[] = { 4883 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8), 4884 }; 4885 4886 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 4887 strs, sizeof(strs))); 4888 } 4889 4890 static int probe_kern_syscall_wrapper(void); 4891 4892 enum kern_feature_result { 4893 FEAT_UNKNOWN = 0, 4894 FEAT_SUPPORTED = 1, 4895 FEAT_MISSING = 2, 4896 }; 4897 4898 typedef int (*feature_probe_fn)(void); 4899 4900 static struct kern_feature_desc { 4901 const char *desc; 4902 feature_probe_fn probe; 4903 enum kern_feature_result res; 4904 } feature_probes[__FEAT_CNT] = { 4905 [FEAT_PROG_NAME] = { 4906 "BPF program name", probe_kern_prog_name, 4907 }, 4908 [FEAT_GLOBAL_DATA] = { 4909 "global variables", probe_kern_global_data, 4910 }, 4911 [FEAT_BTF] = { 4912 "minimal BTF", probe_kern_btf, 4913 }, 4914 [FEAT_BTF_FUNC] = { 4915 "BTF functions", probe_kern_btf_func, 4916 }, 4917 [FEAT_BTF_GLOBAL_FUNC] = { 4918 "BTF global function", probe_kern_btf_func_global, 4919 }, 4920 [FEAT_BTF_DATASEC] = { 4921 "BTF data section and variable", probe_kern_btf_datasec, 4922 }, 4923 [FEAT_ARRAY_MMAP] = { 4924 "ARRAY map mmap()", probe_kern_array_mmap, 4925 }, 4926 [FEAT_EXP_ATTACH_TYPE] = { 4927 "BPF_PROG_LOAD expected_attach_type attribute", 4928 probe_kern_exp_attach_type, 4929 }, 4930 [FEAT_PROBE_READ_KERN] = { 4931 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, 4932 }, 4933 [FEAT_PROG_BIND_MAP] = { 4934 "BPF_PROG_BIND_MAP support", probe_prog_bind_map, 4935 }, 4936 [FEAT_MODULE_BTF] = { 4937 "module BTF support", probe_module_btf, 4938 }, 4939 [FEAT_BTF_FLOAT] = { 4940 "BTF_KIND_FLOAT support", probe_kern_btf_float, 4941 }, 4942 [FEAT_PERF_LINK] = { 4943 "BPF perf link support", probe_perf_link, 4944 }, 4945 [FEAT_BTF_DECL_TAG] = { 4946 "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag, 4947 }, 4948 [FEAT_BTF_TYPE_TAG] = { 4949 "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag, 4950 }, 4951 [FEAT_MEMCG_ACCOUNT] = { 4952 "memcg-based memory accounting", probe_memcg_account, 4953 }, 4954 [FEAT_BPF_COOKIE] = { 4955 "BPF cookie support", probe_kern_bpf_cookie, 4956 }, 4957 [FEAT_BTF_ENUM64] = { 4958 "BTF_KIND_ENUM64 support", probe_kern_btf_enum64, 4959 }, 4960 [FEAT_SYSCALL_WRAPPER] = { 4961 "Kernel using syscall wrapper", probe_kern_syscall_wrapper, 4962 }, 4963 [FEAT_UPROBE_MULTI_LINK] = { 4964 "BPF multi-uprobe link support", probe_uprobe_multi_link, 4965 }, 4966 }; 4967 4968 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) 4969 { 4970 struct kern_feature_desc *feat = &feature_probes[feat_id]; 4971 int ret; 4972 4973 if (obj && obj->gen_loader) 4974 /* To generate loader program assume the latest kernel 4975 * to avoid doing extra prog_load, map_create syscalls. 4976 */ 4977 return true; 4978 4979 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) { 4980 ret = feat->probe(); 4981 if (ret > 0) { 4982 WRITE_ONCE(feat->res, FEAT_SUPPORTED); 4983 } else if (ret == 0) { 4984 WRITE_ONCE(feat->res, FEAT_MISSING); 4985 } else { 4986 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); 4987 WRITE_ONCE(feat->res, FEAT_MISSING); 4988 } 4989 } 4990 4991 return READ_ONCE(feat->res) == FEAT_SUPPORTED; 4992 } 4993 4994 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) 4995 { 4996 struct bpf_map_info map_info; 4997 char msg[STRERR_BUFSIZE]; 4998 __u32 map_info_len = sizeof(map_info); 4999 int err; 5000 5001 memset(&map_info, 0, map_info_len); 5002 err = bpf_map_get_info_by_fd(map_fd, &map_info, &map_info_len); 5003 if (err && errno == EINVAL) 5004 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info); 5005 if (err) { 5006 pr_warn("failed to get map info for map FD %d: %s\n", map_fd, 5007 libbpf_strerror_r(errno, msg, sizeof(msg))); 5008 return false; 5009 } 5010 5011 return (map_info.type == map->def.type && 5012 map_info.key_size == map->def.key_size && 5013 map_info.value_size == map->def.value_size && 5014 map_info.max_entries == map->def.max_entries && 5015 map_info.map_flags == map->def.map_flags && 5016 map_info.map_extra == map->map_extra); 5017 } 5018 5019 static int 5020 bpf_object__reuse_map(struct bpf_map *map) 5021 { 5022 char *cp, errmsg[STRERR_BUFSIZE]; 5023 int err, pin_fd; 5024 5025 pin_fd = bpf_obj_get(map->pin_path); 5026 if (pin_fd < 0) { 5027 err = -errno; 5028 if (err == -ENOENT) { 5029 pr_debug("found no pinned map to reuse at '%s'\n", 5030 map->pin_path); 5031 return 0; 5032 } 5033 5034 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 5035 pr_warn("couldn't retrieve pinned map '%s': %s\n", 5036 map->pin_path, cp); 5037 return err; 5038 } 5039 5040 if (!map_is_reuse_compat(map, pin_fd)) { 5041 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", 5042 map->pin_path); 5043 close(pin_fd); 5044 return -EINVAL; 5045 } 5046 5047 err = bpf_map__reuse_fd(map, pin_fd); 5048 close(pin_fd); 5049 if (err) 5050 return err; 5051 5052 map->pinned = true; 5053 pr_debug("reused pinned map at '%s'\n", map->pin_path); 5054 5055 return 0; 5056 } 5057 5058 static int 5059 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 5060 { 5061 enum libbpf_map_type map_type = map->libbpf_type; 5062 char *cp, errmsg[STRERR_BUFSIZE]; 5063 int err, zero = 0; 5064 5065 if (obj->gen_loader) { 5066 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, 5067 map->mmaped, map->def.value_size); 5068 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) 5069 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); 5070 return 0; 5071 } 5072 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); 5073 if (err) { 5074 err = -errno; 5075 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 5076 pr_warn("Error setting initial map(%s) contents: %s\n", 5077 map->name, cp); 5078 return err; 5079 } 5080 5081 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ 5082 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { 5083 err = bpf_map_freeze(map->fd); 5084 if (err) { 5085 err = -errno; 5086 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 5087 pr_warn("Error freezing map(%s) as read-only: %s\n", 5088 map->name, cp); 5089 return err; 5090 } 5091 } 5092 return 0; 5093 } 5094 5095 static void bpf_map__destroy(struct bpf_map *map); 5096 5097 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner) 5098 { 5099 LIBBPF_OPTS(bpf_map_create_opts, create_attr); 5100 struct bpf_map_def *def = &map->def; 5101 const char *map_name = NULL; 5102 int err = 0; 5103 5104 if (kernel_supports(obj, FEAT_PROG_NAME)) 5105 map_name = map->name; 5106 create_attr.map_ifindex = map->map_ifindex; 5107 create_attr.map_flags = def->map_flags; 5108 create_attr.numa_node = map->numa_node; 5109 create_attr.map_extra = map->map_extra; 5110 5111 if (bpf_map__is_struct_ops(map)) 5112 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 5113 5114 if (obj->btf && btf__fd(obj->btf) >= 0) { 5115 create_attr.btf_fd = btf__fd(obj->btf); 5116 create_attr.btf_key_type_id = map->btf_key_type_id; 5117 create_attr.btf_value_type_id = map->btf_value_type_id; 5118 } 5119 5120 if (bpf_map_type__is_map_in_map(def->type)) { 5121 if (map->inner_map) { 5122 err = bpf_object__create_map(obj, map->inner_map, true); 5123 if (err) { 5124 pr_warn("map '%s': failed to create inner map: %d\n", 5125 map->name, err); 5126 return err; 5127 } 5128 map->inner_map_fd = bpf_map__fd(map->inner_map); 5129 } 5130 if (map->inner_map_fd >= 0) 5131 create_attr.inner_map_fd = map->inner_map_fd; 5132 } 5133 5134 switch (def->type) { 5135 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 5136 case BPF_MAP_TYPE_CGROUP_ARRAY: 5137 case BPF_MAP_TYPE_STACK_TRACE: 5138 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 5139 case BPF_MAP_TYPE_HASH_OF_MAPS: 5140 case BPF_MAP_TYPE_DEVMAP: 5141 case BPF_MAP_TYPE_DEVMAP_HASH: 5142 case BPF_MAP_TYPE_CPUMAP: 5143 case BPF_MAP_TYPE_XSKMAP: 5144 case BPF_MAP_TYPE_SOCKMAP: 5145 case BPF_MAP_TYPE_SOCKHASH: 5146 case BPF_MAP_TYPE_QUEUE: 5147 case BPF_MAP_TYPE_STACK: 5148 create_attr.btf_fd = 0; 5149 create_attr.btf_key_type_id = 0; 5150 create_attr.btf_value_type_id = 0; 5151 map->btf_key_type_id = 0; 5152 map->btf_value_type_id = 0; 5153 default: 5154 break; 5155 } 5156 5157 if (obj->gen_loader) { 5158 bpf_gen__map_create(obj->gen_loader, def->type, map_name, 5159 def->key_size, def->value_size, def->max_entries, 5160 &create_attr, is_inner ? -1 : map - obj->maps); 5161 /* Pretend to have valid FD to pass various fd >= 0 checks. 5162 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 5163 */ 5164 map->fd = 0; 5165 } else { 5166 map->fd = bpf_map_create(def->type, map_name, 5167 def->key_size, def->value_size, 5168 def->max_entries, &create_attr); 5169 } 5170 if (map->fd < 0 && (create_attr.btf_key_type_id || 5171 create_attr.btf_value_type_id)) { 5172 char *cp, errmsg[STRERR_BUFSIZE]; 5173 5174 err = -errno; 5175 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 5176 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 5177 map->name, cp, err); 5178 create_attr.btf_fd = 0; 5179 create_attr.btf_key_type_id = 0; 5180 create_attr.btf_value_type_id = 0; 5181 map->btf_key_type_id = 0; 5182 map->btf_value_type_id = 0; 5183 map->fd = bpf_map_create(def->type, map_name, 5184 def->key_size, def->value_size, 5185 def->max_entries, &create_attr); 5186 } 5187 5188 err = map->fd < 0 ? -errno : 0; 5189 5190 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { 5191 if (obj->gen_loader) 5192 map->inner_map->fd = -1; 5193 bpf_map__destroy(map->inner_map); 5194 zfree(&map->inner_map); 5195 } 5196 5197 return err; 5198 } 5199 5200 static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) 5201 { 5202 const struct bpf_map *targ_map; 5203 unsigned int i; 5204 int fd, err = 0; 5205 5206 for (i = 0; i < map->init_slots_sz; i++) { 5207 if (!map->init_slots[i]) 5208 continue; 5209 5210 targ_map = map->init_slots[i]; 5211 fd = bpf_map__fd(targ_map); 5212 5213 if (obj->gen_loader) { 5214 bpf_gen__populate_outer_map(obj->gen_loader, 5215 map - obj->maps, i, 5216 targ_map - obj->maps); 5217 } else { 5218 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 5219 } 5220 if (err) { 5221 err = -errno; 5222 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n", 5223 map->name, i, targ_map->name, fd, err); 5224 return err; 5225 } 5226 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", 5227 map->name, i, targ_map->name, fd); 5228 } 5229 5230 zfree(&map->init_slots); 5231 map->init_slots_sz = 0; 5232 5233 return 0; 5234 } 5235 5236 static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) 5237 { 5238 const struct bpf_program *targ_prog; 5239 unsigned int i; 5240 int fd, err; 5241 5242 if (obj->gen_loader) 5243 return -ENOTSUP; 5244 5245 for (i = 0; i < map->init_slots_sz; i++) { 5246 if (!map->init_slots[i]) 5247 continue; 5248 5249 targ_prog = map->init_slots[i]; 5250 fd = bpf_program__fd(targ_prog); 5251 5252 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 5253 if (err) { 5254 err = -errno; 5255 pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n", 5256 map->name, i, targ_prog->name, fd, err); 5257 return err; 5258 } 5259 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n", 5260 map->name, i, targ_prog->name, fd); 5261 } 5262 5263 zfree(&map->init_slots); 5264 map->init_slots_sz = 0; 5265 5266 return 0; 5267 } 5268 5269 static int bpf_object_init_prog_arrays(struct bpf_object *obj) 5270 { 5271 struct bpf_map *map; 5272 int i, err; 5273 5274 for (i = 0; i < obj->nr_maps; i++) { 5275 map = &obj->maps[i]; 5276 5277 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY) 5278 continue; 5279 5280 err = init_prog_array_slots(obj, map); 5281 if (err < 0) { 5282 zclose(map->fd); 5283 return err; 5284 } 5285 } 5286 return 0; 5287 } 5288 5289 static int map_set_def_max_entries(struct bpf_map *map) 5290 { 5291 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) { 5292 int nr_cpus; 5293 5294 nr_cpus = libbpf_num_possible_cpus(); 5295 if (nr_cpus < 0) { 5296 pr_warn("map '%s': failed to determine number of system CPUs: %d\n", 5297 map->name, nr_cpus); 5298 return nr_cpus; 5299 } 5300 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); 5301 map->def.max_entries = nr_cpus; 5302 } 5303 5304 return 0; 5305 } 5306 5307 static int 5308 bpf_object__create_maps(struct bpf_object *obj) 5309 { 5310 struct bpf_map *map; 5311 char *cp, errmsg[STRERR_BUFSIZE]; 5312 unsigned int i, j; 5313 int err; 5314 bool retried; 5315 5316 for (i = 0; i < obj->nr_maps; i++) { 5317 map = &obj->maps[i]; 5318 5319 /* To support old kernels, we skip creating global data maps 5320 * (.rodata, .data, .kconfig, etc); later on, during program 5321 * loading, if we detect that at least one of the to-be-loaded 5322 * programs is referencing any global data map, we'll error 5323 * out with program name and relocation index logged. 5324 * This approach allows to accommodate Clang emitting 5325 * unnecessary .rodata.str1.1 sections for string literals, 5326 * but also it allows to have CO-RE applications that use 5327 * global variables in some of BPF programs, but not others. 5328 * If those global variable-using programs are not loaded at 5329 * runtime due to bpf_program__set_autoload(prog, false), 5330 * bpf_object loading will succeed just fine even on old 5331 * kernels. 5332 */ 5333 if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA)) 5334 map->autocreate = false; 5335 5336 if (!map->autocreate) { 5337 pr_debug("map '%s': skipped auto-creating...\n", map->name); 5338 continue; 5339 } 5340 5341 err = map_set_def_max_entries(map); 5342 if (err) 5343 goto err_out; 5344 5345 retried = false; 5346 retry: 5347 if (map->pin_path) { 5348 err = bpf_object__reuse_map(map); 5349 if (err) { 5350 pr_warn("map '%s': error reusing pinned map\n", 5351 map->name); 5352 goto err_out; 5353 } 5354 if (retried && map->fd < 0) { 5355 pr_warn("map '%s': cannot find pinned map\n", 5356 map->name); 5357 err = -ENOENT; 5358 goto err_out; 5359 } 5360 } 5361 5362 if (map->fd >= 0) { 5363 pr_debug("map '%s': skipping creation (preset fd=%d)\n", 5364 map->name, map->fd); 5365 } else { 5366 err = bpf_object__create_map(obj, map, false); 5367 if (err) 5368 goto err_out; 5369 5370 pr_debug("map '%s': created successfully, fd=%d\n", 5371 map->name, map->fd); 5372 5373 if (bpf_map__is_internal(map)) { 5374 err = bpf_object__populate_internal_map(obj, map); 5375 if (err < 0) { 5376 zclose(map->fd); 5377 goto err_out; 5378 } 5379 } 5380 5381 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { 5382 err = init_map_in_map_slots(obj, map); 5383 if (err < 0) { 5384 zclose(map->fd); 5385 goto err_out; 5386 } 5387 } 5388 } 5389 5390 if (map->pin_path && !map->pinned) { 5391 err = bpf_map__pin(map, NULL); 5392 if (err) { 5393 zclose(map->fd); 5394 if (!retried && err == -EEXIST) { 5395 retried = true; 5396 goto retry; 5397 } 5398 pr_warn("map '%s': failed to auto-pin at '%s': %d\n", 5399 map->name, map->pin_path, err); 5400 goto err_out; 5401 } 5402 } 5403 } 5404 5405 return 0; 5406 5407 err_out: 5408 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 5409 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err); 5410 pr_perm_msg(err); 5411 for (j = 0; j < i; j++) 5412 zclose(obj->maps[j].fd); 5413 return err; 5414 } 5415 5416 static bool bpf_core_is_flavor_sep(const char *s) 5417 { 5418 /* check X___Y name pattern, where X and Y are not underscores */ 5419 return s[0] != '_' && /* X */ 5420 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 5421 s[4] != '_'; /* Y */ 5422 } 5423 5424 /* Given 'some_struct_name___with_flavor' return the length of a name prefix 5425 * before last triple underscore. Struct name part after last triple 5426 * underscore is ignored by BPF CO-RE relocation during relocation matching. 5427 */ 5428 size_t bpf_core_essential_name_len(const char *name) 5429 { 5430 size_t n = strlen(name); 5431 int i; 5432 5433 for (i = n - 5; i >= 0; i--) { 5434 if (bpf_core_is_flavor_sep(name + i)) 5435 return i + 1; 5436 } 5437 return n; 5438 } 5439 5440 void bpf_core_free_cands(struct bpf_core_cand_list *cands) 5441 { 5442 if (!cands) 5443 return; 5444 5445 free(cands->cands); 5446 free(cands); 5447 } 5448 5449 int bpf_core_add_cands(struct bpf_core_cand *local_cand, 5450 size_t local_essent_len, 5451 const struct btf *targ_btf, 5452 const char *targ_btf_name, 5453 int targ_start_id, 5454 struct bpf_core_cand_list *cands) 5455 { 5456 struct bpf_core_cand *new_cands, *cand; 5457 const struct btf_type *t, *local_t; 5458 const char *targ_name, *local_name; 5459 size_t targ_essent_len; 5460 int n, i; 5461 5462 local_t = btf__type_by_id(local_cand->btf, local_cand->id); 5463 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off); 5464 5465 n = btf__type_cnt(targ_btf); 5466 for (i = targ_start_id; i < n; i++) { 5467 t = btf__type_by_id(targ_btf, i); 5468 if (!btf_kind_core_compat(t, local_t)) 5469 continue; 5470 5471 targ_name = btf__name_by_offset(targ_btf, t->name_off); 5472 if (str_is_empty(targ_name)) 5473 continue; 5474 5475 targ_essent_len = bpf_core_essential_name_len(targ_name); 5476 if (targ_essent_len != local_essent_len) 5477 continue; 5478 5479 if (strncmp(local_name, targ_name, local_essent_len) != 0) 5480 continue; 5481 5482 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", 5483 local_cand->id, btf_kind_str(local_t), 5484 local_name, i, btf_kind_str(t), targ_name, 5485 targ_btf_name); 5486 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, 5487 sizeof(*cands->cands)); 5488 if (!new_cands) 5489 return -ENOMEM; 5490 5491 cand = &new_cands[cands->len]; 5492 cand->btf = targ_btf; 5493 cand->id = i; 5494 5495 cands->cands = new_cands; 5496 cands->len++; 5497 } 5498 return 0; 5499 } 5500 5501 static int load_module_btfs(struct bpf_object *obj) 5502 { 5503 struct bpf_btf_info info; 5504 struct module_btf *mod_btf; 5505 struct btf *btf; 5506 char name[64]; 5507 __u32 id = 0, len; 5508 int err, fd; 5509 5510 if (obj->btf_modules_loaded) 5511 return 0; 5512 5513 if (obj->gen_loader) 5514 return 0; 5515 5516 /* don't do this again, even if we find no module BTFs */ 5517 obj->btf_modules_loaded = true; 5518 5519 /* kernel too old to support module BTFs */ 5520 if (!kernel_supports(obj, FEAT_MODULE_BTF)) 5521 return 0; 5522 5523 while (true) { 5524 err = bpf_btf_get_next_id(id, &id); 5525 if (err && errno == ENOENT) 5526 return 0; 5527 if (err && errno == EPERM) { 5528 pr_debug("skipping module BTFs loading, missing privileges\n"); 5529 return 0; 5530 } 5531 if (err) { 5532 err = -errno; 5533 pr_warn("failed to iterate BTF objects: %d\n", err); 5534 return err; 5535 } 5536 5537 fd = bpf_btf_get_fd_by_id(id); 5538 if (fd < 0) { 5539 if (errno == ENOENT) 5540 continue; /* expected race: BTF was unloaded */ 5541 err = -errno; 5542 pr_warn("failed to get BTF object #%d FD: %d\n", id, err); 5543 return err; 5544 } 5545 5546 len = sizeof(info); 5547 memset(&info, 0, sizeof(info)); 5548 info.name = ptr_to_u64(name); 5549 info.name_len = sizeof(name); 5550 5551 err = bpf_btf_get_info_by_fd(fd, &info, &len); 5552 if (err) { 5553 err = -errno; 5554 pr_warn("failed to get BTF object #%d info: %d\n", id, err); 5555 goto err_out; 5556 } 5557 5558 /* ignore non-module BTFs */ 5559 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) { 5560 close(fd); 5561 continue; 5562 } 5563 5564 btf = btf_get_from_fd(fd, obj->btf_vmlinux); 5565 err = libbpf_get_error(btf); 5566 if (err) { 5567 pr_warn("failed to load module [%s]'s BTF object #%d: %d\n", 5568 name, id, err); 5569 goto err_out; 5570 } 5571 5572 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, 5573 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); 5574 if (err) 5575 goto err_out; 5576 5577 mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; 5578 5579 mod_btf->btf = btf; 5580 mod_btf->id = id; 5581 mod_btf->fd = fd; 5582 mod_btf->name = strdup(name); 5583 if (!mod_btf->name) { 5584 err = -ENOMEM; 5585 goto err_out; 5586 } 5587 continue; 5588 5589 err_out: 5590 close(fd); 5591 return err; 5592 } 5593 5594 return 0; 5595 } 5596 5597 static struct bpf_core_cand_list * 5598 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id) 5599 { 5600 struct bpf_core_cand local_cand = {}; 5601 struct bpf_core_cand_list *cands; 5602 const struct btf *main_btf; 5603 const struct btf_type *local_t; 5604 const char *local_name; 5605 size_t local_essent_len; 5606 int err, i; 5607 5608 local_cand.btf = local_btf; 5609 local_cand.id = local_type_id; 5610 local_t = btf__type_by_id(local_btf, local_type_id); 5611 if (!local_t) 5612 return ERR_PTR(-EINVAL); 5613 5614 local_name = btf__name_by_offset(local_btf, local_t->name_off); 5615 if (str_is_empty(local_name)) 5616 return ERR_PTR(-EINVAL); 5617 local_essent_len = bpf_core_essential_name_len(local_name); 5618 5619 cands = calloc(1, sizeof(*cands)); 5620 if (!cands) 5621 return ERR_PTR(-ENOMEM); 5622 5623 /* Attempt to find target candidates in vmlinux BTF first */ 5624 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; 5625 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands); 5626 if (err) 5627 goto err_out; 5628 5629 /* if vmlinux BTF has any candidate, don't got for module BTFs */ 5630 if (cands->len) 5631 return cands; 5632 5633 /* if vmlinux BTF was overridden, don't attempt to load module BTFs */ 5634 if (obj->btf_vmlinux_override) 5635 return cands; 5636 5637 /* now look through module BTFs, trying to still find candidates */ 5638 err = load_module_btfs(obj); 5639 if (err) 5640 goto err_out; 5641 5642 for (i = 0; i < obj->btf_module_cnt; i++) { 5643 err = bpf_core_add_cands(&local_cand, local_essent_len, 5644 obj->btf_modules[i].btf, 5645 obj->btf_modules[i].name, 5646 btf__type_cnt(obj->btf_vmlinux), 5647 cands); 5648 if (err) 5649 goto err_out; 5650 } 5651 5652 return cands; 5653 err_out: 5654 bpf_core_free_cands(cands); 5655 return ERR_PTR(err); 5656 } 5657 5658 /* Check local and target types for compatibility. This check is used for 5659 * type-based CO-RE relocations and follow slightly different rules than 5660 * field-based relocations. This function assumes that root types were already 5661 * checked for name match. Beyond that initial root-level name check, names 5662 * are completely ignored. Compatibility rules are as follows: 5663 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but 5664 * kind should match for local and target types (i.e., STRUCT is not 5665 * compatible with UNION); 5666 * - for ENUMs, the size is ignored; 5667 * - for INT, size and signedness are ignored; 5668 * - for ARRAY, dimensionality is ignored, element types are checked for 5669 * compatibility recursively; 5670 * - CONST/VOLATILE/RESTRICT modifiers are ignored; 5671 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; 5672 * - FUNC_PROTOs are compatible if they have compatible signature: same 5673 * number of input args and compatible return and argument types. 5674 * These rules are not set in stone and probably will be adjusted as we get 5675 * more experience with using BPF CO-RE relocations. 5676 */ 5677 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, 5678 const struct btf *targ_btf, __u32 targ_id) 5679 { 5680 return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32); 5681 } 5682 5683 int bpf_core_types_match(const struct btf *local_btf, __u32 local_id, 5684 const struct btf *targ_btf, __u32 targ_id) 5685 { 5686 return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32); 5687 } 5688 5689 static size_t bpf_core_hash_fn(const long key, void *ctx) 5690 { 5691 return key; 5692 } 5693 5694 static bool bpf_core_equal_fn(const long k1, const long k2, void *ctx) 5695 { 5696 return k1 == k2; 5697 } 5698 5699 static int record_relo_core(struct bpf_program *prog, 5700 const struct bpf_core_relo *core_relo, int insn_idx) 5701 { 5702 struct reloc_desc *relos, *relo; 5703 5704 relos = libbpf_reallocarray(prog->reloc_desc, 5705 prog->nr_reloc + 1, sizeof(*relos)); 5706 if (!relos) 5707 return -ENOMEM; 5708 relo = &relos[prog->nr_reloc]; 5709 relo->type = RELO_CORE; 5710 relo->insn_idx = insn_idx; 5711 relo->core_relo = core_relo; 5712 prog->reloc_desc = relos; 5713 prog->nr_reloc++; 5714 return 0; 5715 } 5716 5717 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx) 5718 { 5719 struct reloc_desc *relo; 5720 int i; 5721 5722 for (i = 0; i < prog->nr_reloc; i++) { 5723 relo = &prog->reloc_desc[i]; 5724 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx) 5725 continue; 5726 5727 return relo->core_relo; 5728 } 5729 5730 return NULL; 5731 } 5732 5733 static int bpf_core_resolve_relo(struct bpf_program *prog, 5734 const struct bpf_core_relo *relo, 5735 int relo_idx, 5736 const struct btf *local_btf, 5737 struct hashmap *cand_cache, 5738 struct bpf_core_relo_res *targ_res) 5739 { 5740 struct bpf_core_spec specs_scratch[3] = {}; 5741 struct bpf_core_cand_list *cands = NULL; 5742 const char *prog_name = prog->name; 5743 const struct btf_type *local_type; 5744 const char *local_name; 5745 __u32 local_id = relo->type_id; 5746 int err; 5747 5748 local_type = btf__type_by_id(local_btf, local_id); 5749 if (!local_type) 5750 return -EINVAL; 5751 5752 local_name = btf__name_by_offset(local_btf, local_type->name_off); 5753 if (!local_name) 5754 return -EINVAL; 5755 5756 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL && 5757 !hashmap__find(cand_cache, local_id, &cands)) { 5758 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); 5759 if (IS_ERR(cands)) { 5760 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", 5761 prog_name, relo_idx, local_id, btf_kind_str(local_type), 5762 local_name, PTR_ERR(cands)); 5763 return PTR_ERR(cands); 5764 } 5765 err = hashmap__set(cand_cache, local_id, cands, NULL, NULL); 5766 if (err) { 5767 bpf_core_free_cands(cands); 5768 return err; 5769 } 5770 } 5771 5772 return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch, 5773 targ_res); 5774 } 5775 5776 static int 5777 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) 5778 { 5779 const struct btf_ext_info_sec *sec; 5780 struct bpf_core_relo_res targ_res; 5781 const struct bpf_core_relo *rec; 5782 const struct btf_ext_info *seg; 5783 struct hashmap_entry *entry; 5784 struct hashmap *cand_cache = NULL; 5785 struct bpf_program *prog; 5786 struct bpf_insn *insn; 5787 const char *sec_name; 5788 int i, err = 0, insn_idx, sec_idx, sec_num; 5789 5790 if (obj->btf_ext->core_relo_info.len == 0) 5791 return 0; 5792 5793 if (targ_btf_path) { 5794 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); 5795 err = libbpf_get_error(obj->btf_vmlinux_override); 5796 if (err) { 5797 pr_warn("failed to parse target BTF: %d\n", err); 5798 return err; 5799 } 5800 } 5801 5802 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); 5803 if (IS_ERR(cand_cache)) { 5804 err = PTR_ERR(cand_cache); 5805 goto out; 5806 } 5807 5808 seg = &obj->btf_ext->core_relo_info; 5809 sec_num = 0; 5810 for_each_btf_ext_sec(seg, sec) { 5811 sec_idx = seg->sec_idxs[sec_num]; 5812 sec_num++; 5813 5814 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 5815 if (str_is_empty(sec_name)) { 5816 err = -EINVAL; 5817 goto out; 5818 } 5819 5820 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info); 5821 5822 for_each_btf_ext_rec(seg, sec, i, rec) { 5823 if (rec->insn_off % BPF_INSN_SZ) 5824 return -EINVAL; 5825 insn_idx = rec->insn_off / BPF_INSN_SZ; 5826 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 5827 if (!prog) { 5828 /* When __weak subprog is "overridden" by another instance 5829 * of the subprog from a different object file, linker still 5830 * appends all the .BTF.ext info that used to belong to that 5831 * eliminated subprogram. 5832 * This is similar to what x86-64 linker does for relocations. 5833 * So just ignore such relocations just like we ignore 5834 * subprog instructions when discovering subprograms. 5835 */ 5836 pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n", 5837 sec_name, i, insn_idx); 5838 continue; 5839 } 5840 /* no need to apply CO-RE relocation if the program is 5841 * not going to be loaded 5842 */ 5843 if (!prog->autoload) 5844 continue; 5845 5846 /* adjust insn_idx from section frame of reference to the local 5847 * program's frame of reference; (sub-)program code is not yet 5848 * relocated, so it's enough to just subtract in-section offset 5849 */ 5850 insn_idx = insn_idx - prog->sec_insn_off; 5851 if (insn_idx >= prog->insns_cnt) 5852 return -EINVAL; 5853 insn = &prog->insns[insn_idx]; 5854 5855 err = record_relo_core(prog, rec, insn_idx); 5856 if (err) { 5857 pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n", 5858 prog->name, i, err); 5859 goto out; 5860 } 5861 5862 if (prog->obj->gen_loader) 5863 continue; 5864 5865 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res); 5866 if (err) { 5867 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", 5868 prog->name, i, err); 5869 goto out; 5870 } 5871 5872 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); 5873 if (err) { 5874 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n", 5875 prog->name, i, insn_idx, err); 5876 goto out; 5877 } 5878 } 5879 } 5880 5881 out: 5882 /* obj->btf_vmlinux and module BTFs are freed after object load */ 5883 btf__free(obj->btf_vmlinux_override); 5884 obj->btf_vmlinux_override = NULL; 5885 5886 if (!IS_ERR_OR_NULL(cand_cache)) { 5887 hashmap__for_each_entry(cand_cache, entry, i) { 5888 bpf_core_free_cands(entry->pvalue); 5889 } 5890 hashmap__free(cand_cache); 5891 } 5892 return err; 5893 } 5894 5895 /* base map load ldimm64 special constant, used also for log fixup logic */ 5896 #define POISON_LDIMM64_MAP_BASE 2001000000 5897 #define POISON_LDIMM64_MAP_PFX "200100" 5898 5899 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx, 5900 int insn_idx, struct bpf_insn *insn, 5901 int map_idx, const struct bpf_map *map) 5902 { 5903 int i; 5904 5905 pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n", 5906 prog->name, relo_idx, insn_idx, map_idx, map->name); 5907 5908 /* we turn single ldimm64 into two identical invalid calls */ 5909 for (i = 0; i < 2; i++) { 5910 insn->code = BPF_JMP | BPF_CALL; 5911 insn->dst_reg = 0; 5912 insn->src_reg = 0; 5913 insn->off = 0; 5914 /* if this instruction is reachable (not a dead code), 5915 * verifier will complain with something like: 5916 * invalid func unknown#2001000123 5917 * where lower 123 is map index into obj->maps[] array 5918 */ 5919 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx; 5920 5921 insn++; 5922 } 5923 } 5924 5925 /* unresolved kfunc call special constant, used also for log fixup logic */ 5926 #define POISON_CALL_KFUNC_BASE 2002000000 5927 #define POISON_CALL_KFUNC_PFX "2002" 5928 5929 static void poison_kfunc_call(struct bpf_program *prog, int relo_idx, 5930 int insn_idx, struct bpf_insn *insn, 5931 int ext_idx, const struct extern_desc *ext) 5932 { 5933 pr_debug("prog '%s': relo #%d: poisoning insn #%d that calls kfunc '%s'\n", 5934 prog->name, relo_idx, insn_idx, ext->name); 5935 5936 /* we turn kfunc call into invalid helper call with identifiable constant */ 5937 insn->code = BPF_JMP | BPF_CALL; 5938 insn->dst_reg = 0; 5939 insn->src_reg = 0; 5940 insn->off = 0; 5941 /* if this instruction is reachable (not a dead code), 5942 * verifier will complain with something like: 5943 * invalid func unknown#2001000123 5944 * where lower 123 is extern index into obj->externs[] array 5945 */ 5946 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx; 5947 } 5948 5949 /* Relocate data references within program code: 5950 * - map references; 5951 * - global variable references; 5952 * - extern references. 5953 */ 5954 static int 5955 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) 5956 { 5957 int i; 5958 5959 for (i = 0; i < prog->nr_reloc; i++) { 5960 struct reloc_desc *relo = &prog->reloc_desc[i]; 5961 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 5962 const struct bpf_map *map; 5963 struct extern_desc *ext; 5964 5965 switch (relo->type) { 5966 case RELO_LD64: 5967 map = &obj->maps[relo->map_idx]; 5968 if (obj->gen_loader) { 5969 insn[0].src_reg = BPF_PSEUDO_MAP_IDX; 5970 insn[0].imm = relo->map_idx; 5971 } else if (map->autocreate) { 5972 insn[0].src_reg = BPF_PSEUDO_MAP_FD; 5973 insn[0].imm = map->fd; 5974 } else { 5975 poison_map_ldimm64(prog, i, relo->insn_idx, insn, 5976 relo->map_idx, map); 5977 } 5978 break; 5979 case RELO_DATA: 5980 map = &obj->maps[relo->map_idx]; 5981 insn[1].imm = insn[0].imm + relo->sym_off; 5982 if (obj->gen_loader) { 5983 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 5984 insn[0].imm = relo->map_idx; 5985 } else if (map->autocreate) { 5986 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 5987 insn[0].imm = map->fd; 5988 } else { 5989 poison_map_ldimm64(prog, i, relo->insn_idx, insn, 5990 relo->map_idx, map); 5991 } 5992 break; 5993 case RELO_EXTERN_LD64: 5994 ext = &obj->externs[relo->ext_idx]; 5995 if (ext->type == EXT_KCFG) { 5996 if (obj->gen_loader) { 5997 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 5998 insn[0].imm = obj->kconfig_map_idx; 5999 } else { 6000 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 6001 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; 6002 } 6003 insn[1].imm = ext->kcfg.data_off; 6004 } else /* EXT_KSYM */ { 6005 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */ 6006 insn[0].src_reg = BPF_PSEUDO_BTF_ID; 6007 insn[0].imm = ext->ksym.kernel_btf_id; 6008 insn[1].imm = ext->ksym.kernel_btf_obj_fd; 6009 } else { /* typeless ksyms or unresolved typed ksyms */ 6010 insn[0].imm = (__u32)ext->ksym.addr; 6011 insn[1].imm = ext->ksym.addr >> 32; 6012 } 6013 } 6014 break; 6015 case RELO_EXTERN_CALL: 6016 ext = &obj->externs[relo->ext_idx]; 6017 insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL; 6018 if (ext->is_set) { 6019 insn[0].imm = ext->ksym.kernel_btf_id; 6020 insn[0].off = ext->ksym.btf_fd_idx; 6021 } else { /* unresolved weak kfunc call */ 6022 poison_kfunc_call(prog, i, relo->insn_idx, insn, 6023 relo->ext_idx, ext); 6024 } 6025 break; 6026 case RELO_SUBPROG_ADDR: 6027 if (insn[0].src_reg != BPF_PSEUDO_FUNC) { 6028 pr_warn("prog '%s': relo #%d: bad insn\n", 6029 prog->name, i); 6030 return -EINVAL; 6031 } 6032 /* handled already */ 6033 break; 6034 case RELO_CALL: 6035 /* handled already */ 6036 break; 6037 case RELO_CORE: 6038 /* will be handled by bpf_program_record_relos() */ 6039 break; 6040 default: 6041 pr_warn("prog '%s': relo #%d: bad relo type %d\n", 6042 prog->name, i, relo->type); 6043 return -EINVAL; 6044 } 6045 } 6046 6047 return 0; 6048 } 6049 6050 static int adjust_prog_btf_ext_info(const struct bpf_object *obj, 6051 const struct bpf_program *prog, 6052 const struct btf_ext_info *ext_info, 6053 void **prog_info, __u32 *prog_rec_cnt, 6054 __u32 *prog_rec_sz) 6055 { 6056 void *copy_start = NULL, *copy_end = NULL; 6057 void *rec, *rec_end, *new_prog_info; 6058 const struct btf_ext_info_sec *sec; 6059 size_t old_sz, new_sz; 6060 int i, sec_num, sec_idx, off_adj; 6061 6062 sec_num = 0; 6063 for_each_btf_ext_sec(ext_info, sec) { 6064 sec_idx = ext_info->sec_idxs[sec_num]; 6065 sec_num++; 6066 if (prog->sec_idx != sec_idx) 6067 continue; 6068 6069 for_each_btf_ext_rec(ext_info, sec, i, rec) { 6070 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ; 6071 6072 if (insn_off < prog->sec_insn_off) 6073 continue; 6074 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) 6075 break; 6076 6077 if (!copy_start) 6078 copy_start = rec; 6079 copy_end = rec + ext_info->rec_size; 6080 } 6081 6082 if (!copy_start) 6083 return -ENOENT; 6084 6085 /* append func/line info of a given (sub-)program to the main 6086 * program func/line info 6087 */ 6088 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size; 6089 new_sz = old_sz + (copy_end - copy_start); 6090 new_prog_info = realloc(*prog_info, new_sz); 6091 if (!new_prog_info) 6092 return -ENOMEM; 6093 *prog_info = new_prog_info; 6094 *prog_rec_cnt = new_sz / ext_info->rec_size; 6095 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start); 6096 6097 /* Kernel instruction offsets are in units of 8-byte 6098 * instructions, while .BTF.ext instruction offsets generated 6099 * by Clang are in units of bytes. So convert Clang offsets 6100 * into kernel offsets and adjust offset according to program 6101 * relocated position. 6102 */ 6103 off_adj = prog->sub_insn_off - prog->sec_insn_off; 6104 rec = new_prog_info + old_sz; 6105 rec_end = new_prog_info + new_sz; 6106 for (; rec < rec_end; rec += ext_info->rec_size) { 6107 __u32 *insn_off = rec; 6108 6109 *insn_off = *insn_off / BPF_INSN_SZ + off_adj; 6110 } 6111 *prog_rec_sz = ext_info->rec_size; 6112 return 0; 6113 } 6114 6115 return -ENOENT; 6116 } 6117 6118 static int 6119 reloc_prog_func_and_line_info(const struct bpf_object *obj, 6120 struct bpf_program *main_prog, 6121 const struct bpf_program *prog) 6122 { 6123 int err; 6124 6125 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't 6126 * supprot func/line info 6127 */ 6128 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC)) 6129 return 0; 6130 6131 /* only attempt func info relocation if main program's func_info 6132 * relocation was successful 6133 */ 6134 if (main_prog != prog && !main_prog->func_info) 6135 goto line_info; 6136 6137 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, 6138 &main_prog->func_info, 6139 &main_prog->func_info_cnt, 6140 &main_prog->func_info_rec_size); 6141 if (err) { 6142 if (err != -ENOENT) { 6143 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n", 6144 prog->name, err); 6145 return err; 6146 } 6147 if (main_prog->func_info) { 6148 /* 6149 * Some info has already been found but has problem 6150 * in the last btf_ext reloc. Must have to error out. 6151 */ 6152 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); 6153 return err; 6154 } 6155 /* Have problem loading the very first info. Ignore the rest. */ 6156 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n", 6157 prog->name); 6158 } 6159 6160 line_info: 6161 /* don't relocate line info if main program's relocation failed */ 6162 if (main_prog != prog && !main_prog->line_info) 6163 return 0; 6164 6165 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, 6166 &main_prog->line_info, 6167 &main_prog->line_info_cnt, 6168 &main_prog->line_info_rec_size); 6169 if (err) { 6170 if (err != -ENOENT) { 6171 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n", 6172 prog->name, err); 6173 return err; 6174 } 6175 if (main_prog->line_info) { 6176 /* 6177 * Some info has already been found but has problem 6178 * in the last btf_ext reloc. Must have to error out. 6179 */ 6180 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); 6181 return err; 6182 } 6183 /* Have problem loading the very first info. Ignore the rest. */ 6184 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n", 6185 prog->name); 6186 } 6187 return 0; 6188 } 6189 6190 static int cmp_relo_by_insn_idx(const void *key, const void *elem) 6191 { 6192 size_t insn_idx = *(const size_t *)key; 6193 const struct reloc_desc *relo = elem; 6194 6195 if (insn_idx == relo->insn_idx) 6196 return 0; 6197 return insn_idx < relo->insn_idx ? -1 : 1; 6198 } 6199 6200 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx) 6201 { 6202 if (!prog->nr_reloc) 6203 return NULL; 6204 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, 6205 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); 6206 } 6207 6208 static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog) 6209 { 6210 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc; 6211 struct reloc_desc *relos; 6212 int i; 6213 6214 if (main_prog == subprog) 6215 return 0; 6216 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); 6217 /* if new count is zero, reallocarray can return a valid NULL result; 6218 * in this case the previous pointer will be freed, so we *have to* 6219 * reassign old pointer to the new value (even if it's NULL) 6220 */ 6221 if (!relos && new_cnt) 6222 return -ENOMEM; 6223 if (subprog->nr_reloc) 6224 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, 6225 sizeof(*relos) * subprog->nr_reloc); 6226 6227 for (i = main_prog->nr_reloc; i < new_cnt; i++) 6228 relos[i].insn_idx += subprog->sub_insn_off; 6229 /* After insn_idx adjustment the 'relos' array is still sorted 6230 * by insn_idx and doesn't break bsearch. 6231 */ 6232 main_prog->reloc_desc = relos; 6233 main_prog->nr_reloc = new_cnt; 6234 return 0; 6235 } 6236 6237 static int 6238 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, 6239 struct bpf_program *prog) 6240 { 6241 size_t sub_insn_idx, insn_idx, new_cnt; 6242 struct bpf_program *subprog; 6243 struct bpf_insn *insns, *insn; 6244 struct reloc_desc *relo; 6245 int err; 6246 6247 err = reloc_prog_func_and_line_info(obj, main_prog, prog); 6248 if (err) 6249 return err; 6250 6251 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { 6252 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 6253 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn)) 6254 continue; 6255 6256 relo = find_prog_insn_relo(prog, insn_idx); 6257 if (relo && relo->type == RELO_EXTERN_CALL) 6258 /* kfunc relocations will be handled later 6259 * in bpf_object__relocate_data() 6260 */ 6261 continue; 6262 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { 6263 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", 6264 prog->name, insn_idx, relo->type); 6265 return -LIBBPF_ERRNO__RELOC; 6266 } 6267 if (relo) { 6268 /* sub-program instruction index is a combination of 6269 * an offset of a symbol pointed to by relocation and 6270 * call instruction's imm field; for global functions, 6271 * call always has imm = -1, but for static functions 6272 * relocation is against STT_SECTION and insn->imm 6273 * points to a start of a static function 6274 * 6275 * for subprog addr relocation, the relo->sym_off + insn->imm is 6276 * the byte offset in the corresponding section. 6277 */ 6278 if (relo->type == RELO_CALL) 6279 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; 6280 else 6281 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; 6282 } else if (insn_is_pseudo_func(insn)) { 6283 /* 6284 * RELO_SUBPROG_ADDR relo is always emitted even if both 6285 * functions are in the same section, so it shouldn't reach here. 6286 */ 6287 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n", 6288 prog->name, insn_idx); 6289 return -LIBBPF_ERRNO__RELOC; 6290 } else { 6291 /* if subprogram call is to a static function within 6292 * the same ELF section, there won't be any relocation 6293 * emitted, but it also means there is no additional 6294 * offset necessary, insns->imm is relative to 6295 * instruction's original position within the section 6296 */ 6297 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; 6298 } 6299 6300 /* we enforce that sub-programs should be in .text section */ 6301 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); 6302 if (!subprog) { 6303 pr_warn("prog '%s': no .text section found yet sub-program call exists\n", 6304 prog->name); 6305 return -LIBBPF_ERRNO__RELOC; 6306 } 6307 6308 /* if it's the first call instruction calling into this 6309 * subprogram (meaning this subprog hasn't been processed 6310 * yet) within the context of current main program: 6311 * - append it at the end of main program's instructions blog; 6312 * - process is recursively, while current program is put on hold; 6313 * - if that subprogram calls some other not yet processes 6314 * subprogram, same thing will happen recursively until 6315 * there are no more unprocesses subprograms left to append 6316 * and relocate. 6317 */ 6318 if (subprog->sub_insn_off == 0) { 6319 subprog->sub_insn_off = main_prog->insns_cnt; 6320 6321 new_cnt = main_prog->insns_cnt + subprog->insns_cnt; 6322 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); 6323 if (!insns) { 6324 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); 6325 return -ENOMEM; 6326 } 6327 main_prog->insns = insns; 6328 main_prog->insns_cnt = new_cnt; 6329 6330 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, 6331 subprog->insns_cnt * sizeof(*insns)); 6332 6333 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", 6334 main_prog->name, subprog->insns_cnt, subprog->name); 6335 6336 /* The subprog insns are now appended. Append its relos too. */ 6337 err = append_subprog_relos(main_prog, subprog); 6338 if (err) 6339 return err; 6340 err = bpf_object__reloc_code(obj, main_prog, subprog); 6341 if (err) 6342 return err; 6343 } 6344 6345 /* main_prog->insns memory could have been re-allocated, so 6346 * calculate pointer again 6347 */ 6348 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 6349 /* calculate correct instruction position within current main 6350 * prog; each main prog can have a different set of 6351 * subprograms appended (potentially in different order as 6352 * well), so position of any subprog can be different for 6353 * different main programs 6354 */ 6355 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; 6356 6357 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n", 6358 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); 6359 } 6360 6361 return 0; 6362 } 6363 6364 /* 6365 * Relocate sub-program calls. 6366 * 6367 * Algorithm operates as follows. Each entry-point BPF program (referred to as 6368 * main prog) is processed separately. For each subprog (non-entry functions, 6369 * that can be called from either entry progs or other subprogs) gets their 6370 * sub_insn_off reset to zero. This serves as indicator that this subprogram 6371 * hasn't been yet appended and relocated within current main prog. Once its 6372 * relocated, sub_insn_off will point at the position within current main prog 6373 * where given subprog was appended. This will further be used to relocate all 6374 * the call instructions jumping into this subprog. 6375 * 6376 * We start with main program and process all call instructions. If the call 6377 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off 6378 * is zero), subprog instructions are appended at the end of main program's 6379 * instruction array. Then main program is "put on hold" while we recursively 6380 * process newly appended subprogram. If that subprogram calls into another 6381 * subprogram that hasn't been appended, new subprogram is appended again to 6382 * the *main* prog's instructions (subprog's instructions are always left 6383 * untouched, as they need to be in unmodified state for subsequent main progs 6384 * and subprog instructions are always sent only as part of a main prog) and 6385 * the process continues recursively. Once all the subprogs called from a main 6386 * prog or any of its subprogs are appended (and relocated), all their 6387 * positions within finalized instructions array are known, so it's easy to 6388 * rewrite call instructions with correct relative offsets, corresponding to 6389 * desired target subprog. 6390 * 6391 * Its important to realize that some subprogs might not be called from some 6392 * main prog and any of its called/used subprogs. Those will keep their 6393 * subprog->sub_insn_off as zero at all times and won't be appended to current 6394 * main prog and won't be relocated within the context of current main prog. 6395 * They might still be used from other main progs later. 6396 * 6397 * Visually this process can be shown as below. Suppose we have two main 6398 * programs mainA and mainB and BPF object contains three subprogs: subA, 6399 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and 6400 * subC both call subB: 6401 * 6402 * +--------+ +-------+ 6403 * | v v | 6404 * +--+---+ +--+-+-+ +---+--+ 6405 * | subA | | subB | | subC | 6406 * +--+---+ +------+ +---+--+ 6407 * ^ ^ 6408 * | | 6409 * +---+-------+ +------+----+ 6410 * | mainA | | mainB | 6411 * +-----------+ +-----------+ 6412 * 6413 * We'll start relocating mainA, will find subA, append it and start 6414 * processing sub A recursively: 6415 * 6416 * +-----------+------+ 6417 * | mainA | subA | 6418 * +-----------+------+ 6419 * 6420 * At this point we notice that subB is used from subA, so we append it and 6421 * relocate (there are no further subcalls from subB): 6422 * 6423 * +-----------+------+------+ 6424 * | mainA | subA | subB | 6425 * +-----------+------+------+ 6426 * 6427 * At this point, we relocate subA calls, then go one level up and finish with 6428 * relocatin mainA calls. mainA is done. 6429 * 6430 * For mainB process is similar but results in different order. We start with 6431 * mainB and skip subA and subB, as mainB never calls them (at least 6432 * directly), but we see subC is needed, so we append and start processing it: 6433 * 6434 * +-----------+------+ 6435 * | mainB | subC | 6436 * +-----------+------+ 6437 * Now we see subC needs subB, so we go back to it, append and relocate it: 6438 * 6439 * +-----------+------+------+ 6440 * | mainB | subC | subB | 6441 * +-----------+------+------+ 6442 * 6443 * At this point we unwind recursion, relocate calls in subC, then in mainB. 6444 */ 6445 static int 6446 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) 6447 { 6448 struct bpf_program *subprog; 6449 int i, err; 6450 6451 /* mark all subprogs as not relocated (yet) within the context of 6452 * current main program 6453 */ 6454 for (i = 0; i < obj->nr_programs; i++) { 6455 subprog = &obj->programs[i]; 6456 if (!prog_is_subprog(obj, subprog)) 6457 continue; 6458 6459 subprog->sub_insn_off = 0; 6460 } 6461 6462 err = bpf_object__reloc_code(obj, prog, prog); 6463 if (err) 6464 return err; 6465 6466 return 0; 6467 } 6468 6469 static void 6470 bpf_object__free_relocs(struct bpf_object *obj) 6471 { 6472 struct bpf_program *prog; 6473 int i; 6474 6475 /* free up relocation descriptors */ 6476 for (i = 0; i < obj->nr_programs; i++) { 6477 prog = &obj->programs[i]; 6478 zfree(&prog->reloc_desc); 6479 prog->nr_reloc = 0; 6480 } 6481 } 6482 6483 static int cmp_relocs(const void *_a, const void *_b) 6484 { 6485 const struct reloc_desc *a = _a; 6486 const struct reloc_desc *b = _b; 6487 6488 if (a->insn_idx != b->insn_idx) 6489 return a->insn_idx < b->insn_idx ? -1 : 1; 6490 6491 /* no two relocations should have the same insn_idx, but ... */ 6492 if (a->type != b->type) 6493 return a->type < b->type ? -1 : 1; 6494 6495 return 0; 6496 } 6497 6498 static void bpf_object__sort_relos(struct bpf_object *obj) 6499 { 6500 int i; 6501 6502 for (i = 0; i < obj->nr_programs; i++) { 6503 struct bpf_program *p = &obj->programs[i]; 6504 6505 if (!p->nr_reloc) 6506 continue; 6507 6508 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); 6509 } 6510 } 6511 6512 static int 6513 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) 6514 { 6515 struct bpf_program *prog; 6516 size_t i, j; 6517 int err; 6518 6519 if (obj->btf_ext) { 6520 err = bpf_object__relocate_core(obj, targ_btf_path); 6521 if (err) { 6522 pr_warn("failed to perform CO-RE relocations: %d\n", 6523 err); 6524 return err; 6525 } 6526 bpf_object__sort_relos(obj); 6527 } 6528 6529 /* Before relocating calls pre-process relocations and mark 6530 * few ld_imm64 instructions that points to subprogs. 6531 * Otherwise bpf_object__reloc_code() later would have to consider 6532 * all ld_imm64 insns as relocation candidates. That would 6533 * reduce relocation speed, since amount of find_prog_insn_relo() 6534 * would increase and most of them will fail to find a relo. 6535 */ 6536 for (i = 0; i < obj->nr_programs; i++) { 6537 prog = &obj->programs[i]; 6538 for (j = 0; j < prog->nr_reloc; j++) { 6539 struct reloc_desc *relo = &prog->reloc_desc[j]; 6540 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 6541 6542 /* mark the insn, so it's recognized by insn_is_pseudo_func() */ 6543 if (relo->type == RELO_SUBPROG_ADDR) 6544 insn[0].src_reg = BPF_PSEUDO_FUNC; 6545 } 6546 } 6547 6548 /* relocate subprogram calls and append used subprograms to main 6549 * programs; each copy of subprogram code needs to be relocated 6550 * differently for each main program, because its code location might 6551 * have changed. 6552 * Append subprog relos to main programs to allow data relos to be 6553 * processed after text is completely relocated. 6554 */ 6555 for (i = 0; i < obj->nr_programs; i++) { 6556 prog = &obj->programs[i]; 6557 /* sub-program's sub-calls are relocated within the context of 6558 * its main program only 6559 */ 6560 if (prog_is_subprog(obj, prog)) 6561 continue; 6562 if (!prog->autoload) 6563 continue; 6564 6565 err = bpf_object__relocate_calls(obj, prog); 6566 if (err) { 6567 pr_warn("prog '%s': failed to relocate calls: %d\n", 6568 prog->name, err); 6569 return err; 6570 } 6571 } 6572 /* Process data relos for main programs */ 6573 for (i = 0; i < obj->nr_programs; i++) { 6574 prog = &obj->programs[i]; 6575 if (prog_is_subprog(obj, prog)) 6576 continue; 6577 if (!prog->autoload) 6578 continue; 6579 err = bpf_object__relocate_data(obj, prog); 6580 if (err) { 6581 pr_warn("prog '%s': failed to relocate data references: %d\n", 6582 prog->name, err); 6583 return err; 6584 } 6585 } 6586 6587 return 0; 6588 } 6589 6590 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 6591 Elf64_Shdr *shdr, Elf_Data *data); 6592 6593 static int bpf_object__collect_map_relos(struct bpf_object *obj, 6594 Elf64_Shdr *shdr, Elf_Data *data) 6595 { 6596 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *); 6597 int i, j, nrels, new_sz; 6598 const struct btf_var_secinfo *vi = NULL; 6599 const struct btf_type *sec, *var, *def; 6600 struct bpf_map *map = NULL, *targ_map = NULL; 6601 struct bpf_program *targ_prog = NULL; 6602 bool is_prog_array, is_map_in_map; 6603 const struct btf_member *member; 6604 const char *name, *mname, *type; 6605 unsigned int moff; 6606 Elf64_Sym *sym; 6607 Elf64_Rel *rel; 6608 void *tmp; 6609 6610 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) 6611 return -EINVAL; 6612 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); 6613 if (!sec) 6614 return -EINVAL; 6615 6616 nrels = shdr->sh_size / shdr->sh_entsize; 6617 for (i = 0; i < nrels; i++) { 6618 rel = elf_rel_by_idx(data, i); 6619 if (!rel) { 6620 pr_warn(".maps relo #%d: failed to get ELF relo\n", i); 6621 return -LIBBPF_ERRNO__FORMAT; 6622 } 6623 6624 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); 6625 if (!sym) { 6626 pr_warn(".maps relo #%d: symbol %zx not found\n", 6627 i, (size_t)ELF64_R_SYM(rel->r_info)); 6628 return -LIBBPF_ERRNO__FORMAT; 6629 } 6630 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; 6631 6632 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n", 6633 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value, 6634 (size_t)rel->r_offset, sym->st_name, name); 6635 6636 for (j = 0; j < obj->nr_maps; j++) { 6637 map = &obj->maps[j]; 6638 if (map->sec_idx != obj->efile.btf_maps_shndx) 6639 continue; 6640 6641 vi = btf_var_secinfos(sec) + map->btf_var_idx; 6642 if (vi->offset <= rel->r_offset && 6643 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size) 6644 break; 6645 } 6646 if (j == obj->nr_maps) { 6647 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n", 6648 i, name, (size_t)rel->r_offset); 6649 return -EINVAL; 6650 } 6651 6652 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type); 6653 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY; 6654 type = is_map_in_map ? "map" : "prog"; 6655 if (is_map_in_map) { 6656 if (sym->st_shndx != obj->efile.btf_maps_shndx) { 6657 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", 6658 i, name); 6659 return -LIBBPF_ERRNO__RELOC; 6660 } 6661 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && 6662 map->def.key_size != sizeof(int)) { 6663 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", 6664 i, map->name, sizeof(int)); 6665 return -EINVAL; 6666 } 6667 targ_map = bpf_object__find_map_by_name(obj, name); 6668 if (!targ_map) { 6669 pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n", 6670 i, name); 6671 return -ESRCH; 6672 } 6673 } else if (is_prog_array) { 6674 targ_prog = bpf_object__find_program_by_name(obj, name); 6675 if (!targ_prog) { 6676 pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n", 6677 i, name); 6678 return -ESRCH; 6679 } 6680 if (targ_prog->sec_idx != sym->st_shndx || 6681 targ_prog->sec_insn_off * 8 != sym->st_value || 6682 prog_is_subprog(obj, targ_prog)) { 6683 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n", 6684 i, name); 6685 return -LIBBPF_ERRNO__RELOC; 6686 } 6687 } else { 6688 return -EINVAL; 6689 } 6690 6691 var = btf__type_by_id(obj->btf, vi->type); 6692 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 6693 if (btf_vlen(def) == 0) 6694 return -EINVAL; 6695 member = btf_members(def) + btf_vlen(def) - 1; 6696 mname = btf__name_by_offset(obj->btf, member->name_off); 6697 if (strcmp(mname, "values")) 6698 return -EINVAL; 6699 6700 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; 6701 if (rel->r_offset - vi->offset < moff) 6702 return -EINVAL; 6703 6704 moff = rel->r_offset - vi->offset - moff; 6705 /* here we use BPF pointer size, which is always 64 bit, as we 6706 * are parsing ELF that was built for BPF target 6707 */ 6708 if (moff % bpf_ptr_sz) 6709 return -EINVAL; 6710 moff /= bpf_ptr_sz; 6711 if (moff >= map->init_slots_sz) { 6712 new_sz = moff + 1; 6713 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz); 6714 if (!tmp) 6715 return -ENOMEM; 6716 map->init_slots = tmp; 6717 memset(map->init_slots + map->init_slots_sz, 0, 6718 (new_sz - map->init_slots_sz) * host_ptr_sz); 6719 map->init_slots_sz = new_sz; 6720 } 6721 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog; 6722 6723 pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n", 6724 i, map->name, moff, type, name); 6725 } 6726 6727 return 0; 6728 } 6729 6730 static int bpf_object__collect_relos(struct bpf_object *obj) 6731 { 6732 int i, err; 6733 6734 for (i = 0; i < obj->efile.sec_cnt; i++) { 6735 struct elf_sec_desc *sec_desc = &obj->efile.secs[i]; 6736 Elf64_Shdr *shdr; 6737 Elf_Data *data; 6738 int idx; 6739 6740 if (sec_desc->sec_type != SEC_RELO) 6741 continue; 6742 6743 shdr = sec_desc->shdr; 6744 data = sec_desc->data; 6745 idx = shdr->sh_info; 6746 6747 if (shdr->sh_type != SHT_REL) { 6748 pr_warn("internal error at %d\n", __LINE__); 6749 return -LIBBPF_ERRNO__INTERNAL; 6750 } 6751 6752 if (idx == obj->efile.st_ops_shndx || idx == obj->efile.st_ops_link_shndx) 6753 err = bpf_object__collect_st_ops_relos(obj, shdr, data); 6754 else if (idx == obj->efile.btf_maps_shndx) 6755 err = bpf_object__collect_map_relos(obj, shdr, data); 6756 else 6757 err = bpf_object__collect_prog_relos(obj, shdr, data); 6758 if (err) 6759 return err; 6760 } 6761 6762 bpf_object__sort_relos(obj); 6763 return 0; 6764 } 6765 6766 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id) 6767 { 6768 if (BPF_CLASS(insn->code) == BPF_JMP && 6769 BPF_OP(insn->code) == BPF_CALL && 6770 BPF_SRC(insn->code) == BPF_K && 6771 insn->src_reg == 0 && 6772 insn->dst_reg == 0) { 6773 *func_id = insn->imm; 6774 return true; 6775 } 6776 return false; 6777 } 6778 6779 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) 6780 { 6781 struct bpf_insn *insn = prog->insns; 6782 enum bpf_func_id func_id; 6783 int i; 6784 6785 if (obj->gen_loader) 6786 return 0; 6787 6788 for (i = 0; i < prog->insns_cnt; i++, insn++) { 6789 if (!insn_is_helper_call(insn, &func_id)) 6790 continue; 6791 6792 /* on kernels that don't yet support 6793 * bpf_probe_read_{kernel,user}[_str] helpers, fall back 6794 * to bpf_probe_read() which works well for old kernels 6795 */ 6796 switch (func_id) { 6797 case BPF_FUNC_probe_read_kernel: 6798 case BPF_FUNC_probe_read_user: 6799 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 6800 insn->imm = BPF_FUNC_probe_read; 6801 break; 6802 case BPF_FUNC_probe_read_kernel_str: 6803 case BPF_FUNC_probe_read_user_str: 6804 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 6805 insn->imm = BPF_FUNC_probe_read_str; 6806 break; 6807 default: 6808 break; 6809 } 6810 } 6811 return 0; 6812 } 6813 6814 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, 6815 int *btf_obj_fd, int *btf_type_id); 6816 6817 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */ 6818 static int libbpf_prepare_prog_load(struct bpf_program *prog, 6819 struct bpf_prog_load_opts *opts, long cookie) 6820 { 6821 enum sec_def_flags def = cookie; 6822 6823 /* old kernels might not support specifying expected_attach_type */ 6824 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) 6825 opts->expected_attach_type = 0; 6826 6827 if (def & SEC_SLEEPABLE) 6828 opts->prog_flags |= BPF_F_SLEEPABLE; 6829 6830 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) 6831 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; 6832 6833 /* special check for usdt to use uprobe_multi link */ 6834 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) 6835 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; 6836 6837 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { 6838 int btf_obj_fd = 0, btf_type_id = 0, err; 6839 const char *attach_name; 6840 6841 attach_name = strchr(prog->sec_name, '/'); 6842 if (!attach_name) { 6843 /* if BPF program is annotated with just SEC("fentry") 6844 * (or similar) without declaratively specifying 6845 * target, then it is expected that target will be 6846 * specified with bpf_program__set_attach_target() at 6847 * runtime before BPF object load step. If not, then 6848 * there is nothing to load into the kernel as BPF 6849 * verifier won't be able to validate BPF program 6850 * correctness anyways. 6851 */ 6852 pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n", 6853 prog->name); 6854 return -EINVAL; 6855 } 6856 attach_name++; /* skip over / */ 6857 6858 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id); 6859 if (err) 6860 return err; 6861 6862 /* cache resolved BTF FD and BTF type ID in the prog */ 6863 prog->attach_btf_obj_fd = btf_obj_fd; 6864 prog->attach_btf_id = btf_type_id; 6865 6866 /* but by now libbpf common logic is not utilizing 6867 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because 6868 * this callback is called after opts were populated by 6869 * libbpf, so this callback has to update opts explicitly here 6870 */ 6871 opts->attach_btf_obj_fd = btf_obj_fd; 6872 opts->attach_btf_id = btf_type_id; 6873 } 6874 return 0; 6875 } 6876 6877 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz); 6878 6879 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, 6880 struct bpf_insn *insns, int insns_cnt, 6881 const char *license, __u32 kern_version, int *prog_fd) 6882 { 6883 LIBBPF_OPTS(bpf_prog_load_opts, load_attr); 6884 const char *prog_name = NULL; 6885 char *cp, errmsg[STRERR_BUFSIZE]; 6886 size_t log_buf_size = 0; 6887 char *log_buf = NULL, *tmp; 6888 int btf_fd, ret, err; 6889 bool own_log_buf = true; 6890 __u32 log_level = prog->log_level; 6891 6892 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 6893 /* 6894 * The program type must be set. Most likely we couldn't find a proper 6895 * section definition at load time, and thus we didn't infer the type. 6896 */ 6897 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n", 6898 prog->name, prog->sec_name); 6899 return -EINVAL; 6900 } 6901 6902 if (!insns || !insns_cnt) 6903 return -EINVAL; 6904 6905 if (kernel_supports(obj, FEAT_PROG_NAME)) 6906 prog_name = prog->name; 6907 load_attr.attach_prog_fd = prog->attach_prog_fd; 6908 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; 6909 load_attr.attach_btf_id = prog->attach_btf_id; 6910 load_attr.kern_version = kern_version; 6911 load_attr.prog_ifindex = prog->prog_ifindex; 6912 6913 /* specify func_info/line_info only if kernel supports them */ 6914 btf_fd = bpf_object__btf_fd(obj); 6915 if (btf_fd >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) { 6916 load_attr.prog_btf_fd = btf_fd; 6917 load_attr.func_info = prog->func_info; 6918 load_attr.func_info_rec_size = prog->func_info_rec_size; 6919 load_attr.func_info_cnt = prog->func_info_cnt; 6920 load_attr.line_info = prog->line_info; 6921 load_attr.line_info_rec_size = prog->line_info_rec_size; 6922 load_attr.line_info_cnt = prog->line_info_cnt; 6923 } 6924 load_attr.log_level = log_level; 6925 load_attr.prog_flags = prog->prog_flags; 6926 load_attr.fd_array = obj->fd_array; 6927 6928 /* adjust load_attr if sec_def provides custom preload callback */ 6929 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) { 6930 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie); 6931 if (err < 0) { 6932 pr_warn("prog '%s': failed to prepare load attributes: %d\n", 6933 prog->name, err); 6934 return err; 6935 } 6936 insns = prog->insns; 6937 insns_cnt = prog->insns_cnt; 6938 } 6939 6940 /* allow prog_prepare_load_fn to change expected_attach_type */ 6941 load_attr.expected_attach_type = prog->expected_attach_type; 6942 6943 if (obj->gen_loader) { 6944 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, 6945 license, insns, insns_cnt, &load_attr, 6946 prog - obj->programs); 6947 *prog_fd = -1; 6948 return 0; 6949 } 6950 6951 retry_load: 6952 /* if log_level is zero, we don't request logs initially even if 6953 * custom log_buf is specified; if the program load fails, then we'll 6954 * bump log_level to 1 and use either custom log_buf or we'll allocate 6955 * our own and retry the load to get details on what failed 6956 */ 6957 if (log_level) { 6958 if (prog->log_buf) { 6959 log_buf = prog->log_buf; 6960 log_buf_size = prog->log_size; 6961 own_log_buf = false; 6962 } else if (obj->log_buf) { 6963 log_buf = obj->log_buf; 6964 log_buf_size = obj->log_size; 6965 own_log_buf = false; 6966 } else { 6967 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2); 6968 tmp = realloc(log_buf, log_buf_size); 6969 if (!tmp) { 6970 ret = -ENOMEM; 6971 goto out; 6972 } 6973 log_buf = tmp; 6974 log_buf[0] = '\0'; 6975 own_log_buf = true; 6976 } 6977 } 6978 6979 load_attr.log_buf = log_buf; 6980 load_attr.log_size = log_buf_size; 6981 load_attr.log_level = log_level; 6982 6983 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); 6984 if (ret >= 0) { 6985 if (log_level && own_log_buf) { 6986 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", 6987 prog->name, log_buf); 6988 } 6989 6990 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) { 6991 struct bpf_map *map; 6992 int i; 6993 6994 for (i = 0; i < obj->nr_maps; i++) { 6995 map = &prog->obj->maps[i]; 6996 if (map->libbpf_type != LIBBPF_MAP_RODATA) 6997 continue; 6998 6999 if (bpf_prog_bind_map(ret, bpf_map__fd(map), NULL)) { 7000 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 7001 pr_warn("prog '%s': failed to bind map '%s': %s\n", 7002 prog->name, map->real_name, cp); 7003 /* Don't fail hard if can't bind rodata. */ 7004 } 7005 } 7006 } 7007 7008 *prog_fd = ret; 7009 ret = 0; 7010 goto out; 7011 } 7012 7013 if (log_level == 0) { 7014 log_level = 1; 7015 goto retry_load; 7016 } 7017 /* On ENOSPC, increase log buffer size and retry, unless custom 7018 * log_buf is specified. 7019 * Be careful to not overflow u32, though. Kernel's log buf size limit 7020 * isn't part of UAPI so it can always be bumped to full 4GB. So don't 7021 * multiply by 2 unless we are sure we'll fit within 32 bits. 7022 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2). 7023 */ 7024 if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2) 7025 goto retry_load; 7026 7027 ret = -errno; 7028 7029 /* post-process verifier log to improve error descriptions */ 7030 fixup_verifier_log(prog, log_buf, log_buf_size); 7031 7032 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 7033 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp); 7034 pr_perm_msg(ret); 7035 7036 if (own_log_buf && log_buf && log_buf[0] != '\0') { 7037 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", 7038 prog->name, log_buf); 7039 } 7040 7041 out: 7042 if (own_log_buf) 7043 free(log_buf); 7044 return ret; 7045 } 7046 7047 static char *find_prev_line(char *buf, char *cur) 7048 { 7049 char *p; 7050 7051 if (cur == buf) /* end of a log buf */ 7052 return NULL; 7053 7054 p = cur - 1; 7055 while (p - 1 >= buf && *(p - 1) != '\n') 7056 p--; 7057 7058 return p; 7059 } 7060 7061 static void patch_log(char *buf, size_t buf_sz, size_t log_sz, 7062 char *orig, size_t orig_sz, const char *patch) 7063 { 7064 /* size of the remaining log content to the right from the to-be-replaced part */ 7065 size_t rem_sz = (buf + log_sz) - (orig + orig_sz); 7066 size_t patch_sz = strlen(patch); 7067 7068 if (patch_sz != orig_sz) { 7069 /* If patch line(s) are longer than original piece of verifier log, 7070 * shift log contents by (patch_sz - orig_sz) bytes to the right 7071 * starting from after to-be-replaced part of the log. 7072 * 7073 * If patch line(s) are shorter than original piece of verifier log, 7074 * shift log contents by (orig_sz - patch_sz) bytes to the left 7075 * starting from after to-be-replaced part of the log 7076 * 7077 * We need to be careful about not overflowing available 7078 * buf_sz capacity. If that's the case, we'll truncate the end 7079 * of the original log, as necessary. 7080 */ 7081 if (patch_sz > orig_sz) { 7082 if (orig + patch_sz >= buf + buf_sz) { 7083 /* patch is big enough to cover remaining space completely */ 7084 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1; 7085 rem_sz = 0; 7086 } else if (patch_sz - orig_sz > buf_sz - log_sz) { 7087 /* patch causes part of remaining log to be truncated */ 7088 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz); 7089 } 7090 } 7091 /* shift remaining log to the right by calculated amount */ 7092 memmove(orig + patch_sz, orig + orig_sz, rem_sz); 7093 } 7094 7095 memcpy(orig, patch, patch_sz); 7096 } 7097 7098 static void fixup_log_failed_core_relo(struct bpf_program *prog, 7099 char *buf, size_t buf_sz, size_t log_sz, 7100 char *line1, char *line2, char *line3) 7101 { 7102 /* Expected log for failed and not properly guarded CO-RE relocation: 7103 * line1 -> 123: (85) call unknown#195896080 7104 * line2 -> invalid func unknown#195896080 7105 * line3 -> <anything else or end of buffer> 7106 * 7107 * "123" is the index of the instruction that was poisoned. We extract 7108 * instruction index to find corresponding CO-RE relocation and 7109 * replace this part of the log with more relevant information about 7110 * failed CO-RE relocation. 7111 */ 7112 const struct bpf_core_relo *relo; 7113 struct bpf_core_spec spec; 7114 char patch[512], spec_buf[256]; 7115 int insn_idx, err, spec_len; 7116 7117 if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1) 7118 return; 7119 7120 relo = find_relo_core(prog, insn_idx); 7121 if (!relo) 7122 return; 7123 7124 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec); 7125 if (err) 7126 return; 7127 7128 spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec); 7129 snprintf(patch, sizeof(patch), 7130 "%d: <invalid CO-RE relocation>\n" 7131 "failed to resolve CO-RE relocation %s%s\n", 7132 insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : ""); 7133 7134 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 7135 } 7136 7137 static void fixup_log_missing_map_load(struct bpf_program *prog, 7138 char *buf, size_t buf_sz, size_t log_sz, 7139 char *line1, char *line2, char *line3) 7140 { 7141 /* Expected log for failed and not properly guarded map reference: 7142 * line1 -> 123: (85) call unknown#2001000345 7143 * line2 -> invalid func unknown#2001000345 7144 * line3 -> <anything else or end of buffer> 7145 * 7146 * "123" is the index of the instruction that was poisoned. 7147 * "345" in "2001000345" is a map index in obj->maps to fetch map name. 7148 */ 7149 struct bpf_object *obj = prog->obj; 7150 const struct bpf_map *map; 7151 int insn_idx, map_idx; 7152 char patch[128]; 7153 7154 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2) 7155 return; 7156 7157 map_idx -= POISON_LDIMM64_MAP_BASE; 7158 if (map_idx < 0 || map_idx >= obj->nr_maps) 7159 return; 7160 map = &obj->maps[map_idx]; 7161 7162 snprintf(patch, sizeof(patch), 7163 "%d: <invalid BPF map reference>\n" 7164 "BPF map '%s' is referenced but wasn't created\n", 7165 insn_idx, map->name); 7166 7167 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 7168 } 7169 7170 static void fixup_log_missing_kfunc_call(struct bpf_program *prog, 7171 char *buf, size_t buf_sz, size_t log_sz, 7172 char *line1, char *line2, char *line3) 7173 { 7174 /* Expected log for failed and not properly guarded kfunc call: 7175 * line1 -> 123: (85) call unknown#2002000345 7176 * line2 -> invalid func unknown#2002000345 7177 * line3 -> <anything else or end of buffer> 7178 * 7179 * "123" is the index of the instruction that was poisoned. 7180 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name. 7181 */ 7182 struct bpf_object *obj = prog->obj; 7183 const struct extern_desc *ext; 7184 int insn_idx, ext_idx; 7185 char patch[128]; 7186 7187 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &ext_idx) != 2) 7188 return; 7189 7190 ext_idx -= POISON_CALL_KFUNC_BASE; 7191 if (ext_idx < 0 || ext_idx >= obj->nr_extern) 7192 return; 7193 ext = &obj->externs[ext_idx]; 7194 7195 snprintf(patch, sizeof(patch), 7196 "%d: <invalid kfunc call>\n" 7197 "kfunc '%s' is referenced but wasn't resolved\n", 7198 insn_idx, ext->name); 7199 7200 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 7201 } 7202 7203 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz) 7204 { 7205 /* look for familiar error patterns in last N lines of the log */ 7206 const size_t max_last_line_cnt = 10; 7207 char *prev_line, *cur_line, *next_line; 7208 size_t log_sz; 7209 int i; 7210 7211 if (!buf) 7212 return; 7213 7214 log_sz = strlen(buf) + 1; 7215 next_line = buf + log_sz - 1; 7216 7217 for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) { 7218 cur_line = find_prev_line(buf, next_line); 7219 if (!cur_line) 7220 return; 7221 7222 if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) { 7223 prev_line = find_prev_line(buf, cur_line); 7224 if (!prev_line) 7225 continue; 7226 7227 /* failed CO-RE relocation case */ 7228 fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz, 7229 prev_line, cur_line, next_line); 7230 return; 7231 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_LDIMM64_MAP_PFX)) { 7232 prev_line = find_prev_line(buf, cur_line); 7233 if (!prev_line) 7234 continue; 7235 7236 /* reference to uncreated BPF map */ 7237 fixup_log_missing_map_load(prog, buf, buf_sz, log_sz, 7238 prev_line, cur_line, next_line); 7239 return; 7240 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_CALL_KFUNC_PFX)) { 7241 prev_line = find_prev_line(buf, cur_line); 7242 if (!prev_line) 7243 continue; 7244 7245 /* reference to unresolved kfunc */ 7246 fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz, 7247 prev_line, cur_line, next_line); 7248 return; 7249 } 7250 } 7251 } 7252 7253 static int bpf_program_record_relos(struct bpf_program *prog) 7254 { 7255 struct bpf_object *obj = prog->obj; 7256 int i; 7257 7258 for (i = 0; i < prog->nr_reloc; i++) { 7259 struct reloc_desc *relo = &prog->reloc_desc[i]; 7260 struct extern_desc *ext = &obj->externs[relo->ext_idx]; 7261 int kind; 7262 7263 switch (relo->type) { 7264 case RELO_EXTERN_LD64: 7265 if (ext->type != EXT_KSYM) 7266 continue; 7267 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ? 7268 BTF_KIND_VAR : BTF_KIND_FUNC; 7269 bpf_gen__record_extern(obj->gen_loader, ext->name, 7270 ext->is_weak, !ext->ksym.type_id, 7271 true, kind, relo->insn_idx); 7272 break; 7273 case RELO_EXTERN_CALL: 7274 bpf_gen__record_extern(obj->gen_loader, ext->name, 7275 ext->is_weak, false, false, BTF_KIND_FUNC, 7276 relo->insn_idx); 7277 break; 7278 case RELO_CORE: { 7279 struct bpf_core_relo cr = { 7280 .insn_off = relo->insn_idx * 8, 7281 .type_id = relo->core_relo->type_id, 7282 .access_str_off = relo->core_relo->access_str_off, 7283 .kind = relo->core_relo->kind, 7284 }; 7285 7286 bpf_gen__record_relo_core(obj->gen_loader, &cr); 7287 break; 7288 } 7289 default: 7290 continue; 7291 } 7292 } 7293 return 0; 7294 } 7295 7296 static int 7297 bpf_object__load_progs(struct bpf_object *obj, int log_level) 7298 { 7299 struct bpf_program *prog; 7300 size_t i; 7301 int err; 7302 7303 for (i = 0; i < obj->nr_programs; i++) { 7304 prog = &obj->programs[i]; 7305 err = bpf_object__sanitize_prog(obj, prog); 7306 if (err) 7307 return err; 7308 } 7309 7310 for (i = 0; i < obj->nr_programs; i++) { 7311 prog = &obj->programs[i]; 7312 if (prog_is_subprog(obj, prog)) 7313 continue; 7314 if (!prog->autoload) { 7315 pr_debug("prog '%s': skipped loading\n", prog->name); 7316 continue; 7317 } 7318 prog->log_level |= log_level; 7319 7320 if (obj->gen_loader) 7321 bpf_program_record_relos(prog); 7322 7323 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, 7324 obj->license, obj->kern_version, &prog->fd); 7325 if (err) { 7326 pr_warn("prog '%s': failed to load: %d\n", prog->name, err); 7327 return err; 7328 } 7329 } 7330 7331 bpf_object__free_relocs(obj); 7332 return 0; 7333 } 7334 7335 static const struct bpf_sec_def *find_sec_def(const char *sec_name); 7336 7337 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts) 7338 { 7339 struct bpf_program *prog; 7340 int err; 7341 7342 bpf_object__for_each_program(prog, obj) { 7343 prog->sec_def = find_sec_def(prog->sec_name); 7344 if (!prog->sec_def) { 7345 /* couldn't guess, but user might manually specify */ 7346 pr_debug("prog '%s': unrecognized ELF section name '%s'\n", 7347 prog->name, prog->sec_name); 7348 continue; 7349 } 7350 7351 prog->type = prog->sec_def->prog_type; 7352 prog->expected_attach_type = prog->sec_def->expected_attach_type; 7353 7354 /* sec_def can have custom callback which should be called 7355 * after bpf_program is initialized to adjust its properties 7356 */ 7357 if (prog->sec_def->prog_setup_fn) { 7358 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie); 7359 if (err < 0) { 7360 pr_warn("prog '%s': failed to initialize: %d\n", 7361 prog->name, err); 7362 return err; 7363 } 7364 } 7365 } 7366 7367 return 0; 7368 } 7369 7370 static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz, 7371 const struct bpf_object_open_opts *opts) 7372 { 7373 const char *obj_name, *kconfig, *btf_tmp_path; 7374 struct bpf_object *obj; 7375 char tmp_name[64]; 7376 int err; 7377 char *log_buf; 7378 size_t log_size; 7379 __u32 log_level; 7380 7381 if (elf_version(EV_CURRENT) == EV_NONE) { 7382 pr_warn("failed to init libelf for %s\n", 7383 path ? : "(mem buf)"); 7384 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 7385 } 7386 7387 if (!OPTS_VALID(opts, bpf_object_open_opts)) 7388 return ERR_PTR(-EINVAL); 7389 7390 obj_name = OPTS_GET(opts, object_name, NULL); 7391 if (obj_buf) { 7392 if (!obj_name) { 7393 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 7394 (unsigned long)obj_buf, 7395 (unsigned long)obj_buf_sz); 7396 obj_name = tmp_name; 7397 } 7398 path = obj_name; 7399 pr_debug("loading object '%s' from buffer\n", obj_name); 7400 } 7401 7402 log_buf = OPTS_GET(opts, kernel_log_buf, NULL); 7403 log_size = OPTS_GET(opts, kernel_log_size, 0); 7404 log_level = OPTS_GET(opts, kernel_log_level, 0); 7405 if (log_size > UINT_MAX) 7406 return ERR_PTR(-EINVAL); 7407 if (log_size && !log_buf) 7408 return ERR_PTR(-EINVAL); 7409 7410 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); 7411 if (IS_ERR(obj)) 7412 return obj; 7413 7414 obj->log_buf = log_buf; 7415 obj->log_size = log_size; 7416 obj->log_level = log_level; 7417 7418 btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL); 7419 if (btf_tmp_path) { 7420 if (strlen(btf_tmp_path) >= PATH_MAX) { 7421 err = -ENAMETOOLONG; 7422 goto out; 7423 } 7424 obj->btf_custom_path = strdup(btf_tmp_path); 7425 if (!obj->btf_custom_path) { 7426 err = -ENOMEM; 7427 goto out; 7428 } 7429 } 7430 7431 kconfig = OPTS_GET(opts, kconfig, NULL); 7432 if (kconfig) { 7433 obj->kconfig = strdup(kconfig); 7434 if (!obj->kconfig) { 7435 err = -ENOMEM; 7436 goto out; 7437 } 7438 } 7439 7440 err = bpf_object__elf_init(obj); 7441 err = err ? : bpf_object__check_endianness(obj); 7442 err = err ? : bpf_object__elf_collect(obj); 7443 err = err ? : bpf_object__collect_externs(obj); 7444 err = err ? : bpf_object_fixup_btf(obj); 7445 err = err ? : bpf_object__init_maps(obj, opts); 7446 err = err ? : bpf_object_init_progs(obj, opts); 7447 err = err ? : bpf_object__collect_relos(obj); 7448 if (err) 7449 goto out; 7450 7451 bpf_object__elf_finish(obj); 7452 7453 return obj; 7454 out: 7455 bpf_object__close(obj); 7456 return ERR_PTR(err); 7457 } 7458 7459 struct bpf_object * 7460 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) 7461 { 7462 if (!path) 7463 return libbpf_err_ptr(-EINVAL); 7464 7465 pr_debug("loading %s\n", path); 7466 7467 return libbpf_ptr(bpf_object_open(path, NULL, 0, opts)); 7468 } 7469 7470 struct bpf_object *bpf_object__open(const char *path) 7471 { 7472 return bpf_object__open_file(path, NULL); 7473 } 7474 7475 struct bpf_object * 7476 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, 7477 const struct bpf_object_open_opts *opts) 7478 { 7479 if (!obj_buf || obj_buf_sz == 0) 7480 return libbpf_err_ptr(-EINVAL); 7481 7482 return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts)); 7483 } 7484 7485 static int bpf_object_unload(struct bpf_object *obj) 7486 { 7487 size_t i; 7488 7489 if (!obj) 7490 return libbpf_err(-EINVAL); 7491 7492 for (i = 0; i < obj->nr_maps; i++) { 7493 zclose(obj->maps[i].fd); 7494 if (obj->maps[i].st_ops) 7495 zfree(&obj->maps[i].st_ops->kern_vdata); 7496 } 7497 7498 for (i = 0; i < obj->nr_programs; i++) 7499 bpf_program__unload(&obj->programs[i]); 7500 7501 return 0; 7502 } 7503 7504 static int bpf_object__sanitize_maps(struct bpf_object *obj) 7505 { 7506 struct bpf_map *m; 7507 7508 bpf_object__for_each_map(m, obj) { 7509 if (!bpf_map__is_internal(m)) 7510 continue; 7511 if (!kernel_supports(obj, FEAT_ARRAY_MMAP)) 7512 m->def.map_flags &= ~BPF_F_MMAPABLE; 7513 } 7514 7515 return 0; 7516 } 7517 7518 int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx) 7519 { 7520 char sym_type, sym_name[500]; 7521 unsigned long long sym_addr; 7522 int ret, err = 0; 7523 FILE *f; 7524 7525 f = fopen("/proc/kallsyms", "re"); 7526 if (!f) { 7527 err = -errno; 7528 pr_warn("failed to open /proc/kallsyms: %d\n", err); 7529 return err; 7530 } 7531 7532 while (true) { 7533 ret = fscanf(f, "%llx %c %499s%*[^\n]\n", 7534 &sym_addr, &sym_type, sym_name); 7535 if (ret == EOF && feof(f)) 7536 break; 7537 if (ret != 3) { 7538 pr_warn("failed to read kallsyms entry: %d\n", ret); 7539 err = -EINVAL; 7540 break; 7541 } 7542 7543 err = cb(sym_addr, sym_type, sym_name, ctx); 7544 if (err) 7545 break; 7546 } 7547 7548 fclose(f); 7549 return err; 7550 } 7551 7552 static int kallsyms_cb(unsigned long long sym_addr, char sym_type, 7553 const char *sym_name, void *ctx) 7554 { 7555 struct bpf_object *obj = ctx; 7556 const struct btf_type *t; 7557 struct extern_desc *ext; 7558 7559 ext = find_extern_by_name(obj, sym_name); 7560 if (!ext || ext->type != EXT_KSYM) 7561 return 0; 7562 7563 t = btf__type_by_id(obj->btf, ext->btf_id); 7564 if (!btf_is_var(t)) 7565 return 0; 7566 7567 if (ext->is_set && ext->ksym.addr != sym_addr) { 7568 pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n", 7569 sym_name, ext->ksym.addr, sym_addr); 7570 return -EINVAL; 7571 } 7572 if (!ext->is_set) { 7573 ext->is_set = true; 7574 ext->ksym.addr = sym_addr; 7575 pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr); 7576 } 7577 return 0; 7578 } 7579 7580 static int bpf_object__read_kallsyms_file(struct bpf_object *obj) 7581 { 7582 return libbpf_kallsyms_parse(kallsyms_cb, obj); 7583 } 7584 7585 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, 7586 __u16 kind, struct btf **res_btf, 7587 struct module_btf **res_mod_btf) 7588 { 7589 struct module_btf *mod_btf; 7590 struct btf *btf; 7591 int i, id, err; 7592 7593 btf = obj->btf_vmlinux; 7594 mod_btf = NULL; 7595 id = btf__find_by_name_kind(btf, ksym_name, kind); 7596 7597 if (id == -ENOENT) { 7598 err = load_module_btfs(obj); 7599 if (err) 7600 return err; 7601 7602 for (i = 0; i < obj->btf_module_cnt; i++) { 7603 /* we assume module_btf's BTF FD is always >0 */ 7604 mod_btf = &obj->btf_modules[i]; 7605 btf = mod_btf->btf; 7606 id = btf__find_by_name_kind_own(btf, ksym_name, kind); 7607 if (id != -ENOENT) 7608 break; 7609 } 7610 } 7611 if (id <= 0) 7612 return -ESRCH; 7613 7614 *res_btf = btf; 7615 *res_mod_btf = mod_btf; 7616 return id; 7617 } 7618 7619 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, 7620 struct extern_desc *ext) 7621 { 7622 const struct btf_type *targ_var, *targ_type; 7623 __u32 targ_type_id, local_type_id; 7624 struct module_btf *mod_btf = NULL; 7625 const char *targ_var_name; 7626 struct btf *btf = NULL; 7627 int id, err; 7628 7629 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf); 7630 if (id < 0) { 7631 if (id == -ESRCH && ext->is_weak) 7632 return 0; 7633 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n", 7634 ext->name); 7635 return id; 7636 } 7637 7638 /* find local type_id */ 7639 local_type_id = ext->ksym.type_id; 7640 7641 /* find target type_id */ 7642 targ_var = btf__type_by_id(btf, id); 7643 targ_var_name = btf__name_by_offset(btf, targ_var->name_off); 7644 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id); 7645 7646 err = bpf_core_types_are_compat(obj->btf, local_type_id, 7647 btf, targ_type_id); 7648 if (err <= 0) { 7649 const struct btf_type *local_type; 7650 const char *targ_name, *local_name; 7651 7652 local_type = btf__type_by_id(obj->btf, local_type_id); 7653 local_name = btf__name_by_offset(obj->btf, local_type->name_off); 7654 targ_name = btf__name_by_offset(btf, targ_type->name_off); 7655 7656 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", 7657 ext->name, local_type_id, 7658 btf_kind_str(local_type), local_name, targ_type_id, 7659 btf_kind_str(targ_type), targ_name); 7660 return -EINVAL; 7661 } 7662 7663 ext->is_set = true; 7664 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; 7665 ext->ksym.kernel_btf_id = id; 7666 pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n", 7667 ext->name, id, btf_kind_str(targ_var), targ_var_name); 7668 7669 return 0; 7670 } 7671 7672 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, 7673 struct extern_desc *ext) 7674 { 7675 int local_func_proto_id, kfunc_proto_id, kfunc_id; 7676 struct module_btf *mod_btf = NULL; 7677 const struct btf_type *kern_func; 7678 struct btf *kern_btf = NULL; 7679 int ret; 7680 7681 local_func_proto_id = ext->ksym.type_id; 7682 7683 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf, 7684 &mod_btf); 7685 if (kfunc_id < 0) { 7686 if (kfunc_id == -ESRCH && ext->is_weak) 7687 return 0; 7688 pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n", 7689 ext->name); 7690 return kfunc_id; 7691 } 7692 7693 kern_func = btf__type_by_id(kern_btf, kfunc_id); 7694 kfunc_proto_id = kern_func->type; 7695 7696 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, 7697 kern_btf, kfunc_proto_id); 7698 if (ret <= 0) { 7699 if (ext->is_weak) 7700 return 0; 7701 7702 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with %s [%d]\n", 7703 ext->name, local_func_proto_id, 7704 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id); 7705 return -EINVAL; 7706 } 7707 7708 /* set index for module BTF fd in fd_array, if unset */ 7709 if (mod_btf && !mod_btf->fd_array_idx) { 7710 /* insn->off is s16 */ 7711 if (obj->fd_array_cnt == INT16_MAX) { 7712 pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n", 7713 ext->name, mod_btf->fd_array_idx); 7714 return -E2BIG; 7715 } 7716 /* Cannot use index 0 for module BTF fd */ 7717 if (!obj->fd_array_cnt) 7718 obj->fd_array_cnt = 1; 7719 7720 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), 7721 obj->fd_array_cnt + 1); 7722 if (ret) 7723 return ret; 7724 mod_btf->fd_array_idx = obj->fd_array_cnt; 7725 /* we assume module BTF FD is always >0 */ 7726 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd; 7727 } 7728 7729 ext->is_set = true; 7730 ext->ksym.kernel_btf_id = kfunc_id; 7731 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0; 7732 /* Also set kernel_btf_obj_fd to make sure that bpf_object__relocate_data() 7733 * populates FD into ld_imm64 insn when it's used to point to kfunc. 7734 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call. 7735 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64. 7736 */ 7737 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; 7738 pr_debug("extern (func ksym) '%s': resolved to %s [%d]\n", 7739 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id); 7740 7741 return 0; 7742 } 7743 7744 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj) 7745 { 7746 const struct btf_type *t; 7747 struct extern_desc *ext; 7748 int i, err; 7749 7750 for (i = 0; i < obj->nr_extern; i++) { 7751 ext = &obj->externs[i]; 7752 if (ext->type != EXT_KSYM || !ext->ksym.type_id) 7753 continue; 7754 7755 if (obj->gen_loader) { 7756 ext->is_set = true; 7757 ext->ksym.kernel_btf_obj_fd = 0; 7758 ext->ksym.kernel_btf_id = 0; 7759 continue; 7760 } 7761 t = btf__type_by_id(obj->btf, ext->btf_id); 7762 if (btf_is_var(t)) 7763 err = bpf_object__resolve_ksym_var_btf_id(obj, ext); 7764 else 7765 err = bpf_object__resolve_ksym_func_btf_id(obj, ext); 7766 if (err) 7767 return err; 7768 } 7769 return 0; 7770 } 7771 7772 static int bpf_object__resolve_externs(struct bpf_object *obj, 7773 const char *extra_kconfig) 7774 { 7775 bool need_config = false, need_kallsyms = false; 7776 bool need_vmlinux_btf = false; 7777 struct extern_desc *ext; 7778 void *kcfg_data = NULL; 7779 int err, i; 7780 7781 if (obj->nr_extern == 0) 7782 return 0; 7783 7784 if (obj->kconfig_map_idx >= 0) 7785 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; 7786 7787 for (i = 0; i < obj->nr_extern; i++) { 7788 ext = &obj->externs[i]; 7789 7790 if (ext->type == EXT_KSYM) { 7791 if (ext->ksym.type_id) 7792 need_vmlinux_btf = true; 7793 else 7794 need_kallsyms = true; 7795 continue; 7796 } else if (ext->type == EXT_KCFG) { 7797 void *ext_ptr = kcfg_data + ext->kcfg.data_off; 7798 __u64 value = 0; 7799 7800 /* Kconfig externs need actual /proc/config.gz */ 7801 if (str_has_pfx(ext->name, "CONFIG_")) { 7802 need_config = true; 7803 continue; 7804 } 7805 7806 /* Virtual kcfg externs are customly handled by libbpf */ 7807 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { 7808 value = get_kernel_version(); 7809 if (!value) { 7810 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name); 7811 return -EINVAL; 7812 } 7813 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) { 7814 value = kernel_supports(obj, FEAT_BPF_COOKIE); 7815 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) { 7816 value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER); 7817 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) { 7818 /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed 7819 * __kconfig externs, where LINUX_ ones are virtual and filled out 7820 * customly by libbpf (their values don't come from Kconfig). 7821 * If LINUX_xxx variable is not recognized by libbpf, but is marked 7822 * __weak, it defaults to zero value, just like for CONFIG_xxx 7823 * externs. 7824 */ 7825 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name); 7826 return -EINVAL; 7827 } 7828 7829 err = set_kcfg_value_num(ext, ext_ptr, value); 7830 if (err) 7831 return err; 7832 pr_debug("extern (kcfg) '%s': set to 0x%llx\n", 7833 ext->name, (long long)value); 7834 } else { 7835 pr_warn("extern '%s': unrecognized extern kind\n", ext->name); 7836 return -EINVAL; 7837 } 7838 } 7839 if (need_config && extra_kconfig) { 7840 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data); 7841 if (err) 7842 return -EINVAL; 7843 need_config = false; 7844 for (i = 0; i < obj->nr_extern; i++) { 7845 ext = &obj->externs[i]; 7846 if (ext->type == EXT_KCFG && !ext->is_set) { 7847 need_config = true; 7848 break; 7849 } 7850 } 7851 } 7852 if (need_config) { 7853 err = bpf_object__read_kconfig_file(obj, kcfg_data); 7854 if (err) 7855 return -EINVAL; 7856 } 7857 if (need_kallsyms) { 7858 err = bpf_object__read_kallsyms_file(obj); 7859 if (err) 7860 return -EINVAL; 7861 } 7862 if (need_vmlinux_btf) { 7863 err = bpf_object__resolve_ksyms_btf_id(obj); 7864 if (err) 7865 return -EINVAL; 7866 } 7867 for (i = 0; i < obj->nr_extern; i++) { 7868 ext = &obj->externs[i]; 7869 7870 if (!ext->is_set && !ext->is_weak) { 7871 pr_warn("extern '%s' (strong): not resolved\n", ext->name); 7872 return -ESRCH; 7873 } else if (!ext->is_set) { 7874 pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n", 7875 ext->name); 7876 } 7877 } 7878 7879 return 0; 7880 } 7881 7882 static void bpf_map_prepare_vdata(const struct bpf_map *map) 7883 { 7884 struct bpf_struct_ops *st_ops; 7885 __u32 i; 7886 7887 st_ops = map->st_ops; 7888 for (i = 0; i < btf_vlen(st_ops->type); i++) { 7889 struct bpf_program *prog = st_ops->progs[i]; 7890 void *kern_data; 7891 int prog_fd; 7892 7893 if (!prog) 7894 continue; 7895 7896 prog_fd = bpf_program__fd(prog); 7897 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; 7898 *(unsigned long *)kern_data = prog_fd; 7899 } 7900 } 7901 7902 static int bpf_object_prepare_struct_ops(struct bpf_object *obj) 7903 { 7904 int i; 7905 7906 for (i = 0; i < obj->nr_maps; i++) 7907 if (bpf_map__is_struct_ops(&obj->maps[i])) 7908 bpf_map_prepare_vdata(&obj->maps[i]); 7909 7910 return 0; 7911 } 7912 7913 static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path) 7914 { 7915 int err, i; 7916 7917 if (!obj) 7918 return libbpf_err(-EINVAL); 7919 7920 if (obj->loaded) { 7921 pr_warn("object '%s': load can't be attempted twice\n", obj->name); 7922 return libbpf_err(-EINVAL); 7923 } 7924 7925 if (obj->gen_loader) 7926 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); 7927 7928 err = bpf_object__probe_loading(obj); 7929 err = err ? : bpf_object__load_vmlinux_btf(obj, false); 7930 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); 7931 err = err ? : bpf_object__sanitize_and_load_btf(obj); 7932 err = err ? : bpf_object__sanitize_maps(obj); 7933 err = err ? : bpf_object__init_kern_struct_ops_maps(obj); 7934 err = err ? : bpf_object__create_maps(obj); 7935 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path); 7936 err = err ? : bpf_object__load_progs(obj, extra_log_level); 7937 err = err ? : bpf_object_init_prog_arrays(obj); 7938 err = err ? : bpf_object_prepare_struct_ops(obj); 7939 7940 if (obj->gen_loader) { 7941 /* reset FDs */ 7942 if (obj->btf) 7943 btf__set_fd(obj->btf, -1); 7944 for (i = 0; i < obj->nr_maps; i++) 7945 obj->maps[i].fd = -1; 7946 if (!err) 7947 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps); 7948 } 7949 7950 /* clean up fd_array */ 7951 zfree(&obj->fd_array); 7952 7953 /* clean up module BTFs */ 7954 for (i = 0; i < obj->btf_module_cnt; i++) { 7955 close(obj->btf_modules[i].fd); 7956 btf__free(obj->btf_modules[i].btf); 7957 free(obj->btf_modules[i].name); 7958 } 7959 free(obj->btf_modules); 7960 7961 /* clean up vmlinux BTF */ 7962 btf__free(obj->btf_vmlinux); 7963 obj->btf_vmlinux = NULL; 7964 7965 obj->loaded = true; /* doesn't matter if successfully or not */ 7966 7967 if (err) 7968 goto out; 7969 7970 return 0; 7971 out: 7972 /* unpin any maps that were auto-pinned during load */ 7973 for (i = 0; i < obj->nr_maps; i++) 7974 if (obj->maps[i].pinned && !obj->maps[i].reused) 7975 bpf_map__unpin(&obj->maps[i], NULL); 7976 7977 bpf_object_unload(obj); 7978 pr_warn("failed to load object '%s'\n", obj->path); 7979 return libbpf_err(err); 7980 } 7981 7982 int bpf_object__load(struct bpf_object *obj) 7983 { 7984 return bpf_object_load(obj, 0, NULL); 7985 } 7986 7987 static int make_parent_dir(const char *path) 7988 { 7989 char *cp, errmsg[STRERR_BUFSIZE]; 7990 char *dname, *dir; 7991 int err = 0; 7992 7993 dname = strdup(path); 7994 if (dname == NULL) 7995 return -ENOMEM; 7996 7997 dir = dirname(dname); 7998 if (mkdir(dir, 0700) && errno != EEXIST) 7999 err = -errno; 8000 8001 free(dname); 8002 if (err) { 8003 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 8004 pr_warn("failed to mkdir %s: %s\n", path, cp); 8005 } 8006 return err; 8007 } 8008 8009 static int check_path(const char *path) 8010 { 8011 char *cp, errmsg[STRERR_BUFSIZE]; 8012 struct statfs st_fs; 8013 char *dname, *dir; 8014 int err = 0; 8015 8016 if (path == NULL) 8017 return -EINVAL; 8018 8019 dname = strdup(path); 8020 if (dname == NULL) 8021 return -ENOMEM; 8022 8023 dir = dirname(dname); 8024 if (statfs(dir, &st_fs)) { 8025 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 8026 pr_warn("failed to statfs %s: %s\n", dir, cp); 8027 err = -errno; 8028 } 8029 free(dname); 8030 8031 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 8032 pr_warn("specified path %s is not on BPF FS\n", path); 8033 err = -EINVAL; 8034 } 8035 8036 return err; 8037 } 8038 8039 int bpf_program__pin(struct bpf_program *prog, const char *path) 8040 { 8041 char *cp, errmsg[STRERR_BUFSIZE]; 8042 int err; 8043 8044 if (prog->fd < 0) { 8045 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name); 8046 return libbpf_err(-EINVAL); 8047 } 8048 8049 err = make_parent_dir(path); 8050 if (err) 8051 return libbpf_err(err); 8052 8053 err = check_path(path); 8054 if (err) 8055 return libbpf_err(err); 8056 8057 if (bpf_obj_pin(prog->fd, path)) { 8058 err = -errno; 8059 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 8060 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp); 8061 return libbpf_err(err); 8062 } 8063 8064 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path); 8065 return 0; 8066 } 8067 8068 int bpf_program__unpin(struct bpf_program *prog, const char *path) 8069 { 8070 int err; 8071 8072 if (prog->fd < 0) { 8073 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name); 8074 return libbpf_err(-EINVAL); 8075 } 8076 8077 err = check_path(path); 8078 if (err) 8079 return libbpf_err(err); 8080 8081 err = unlink(path); 8082 if (err) 8083 return libbpf_err(-errno); 8084 8085 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path); 8086 return 0; 8087 } 8088 8089 int bpf_map__pin(struct bpf_map *map, const char *path) 8090 { 8091 char *cp, errmsg[STRERR_BUFSIZE]; 8092 int err; 8093 8094 if (map == NULL) { 8095 pr_warn("invalid map pointer\n"); 8096 return libbpf_err(-EINVAL); 8097 } 8098 8099 if (map->pin_path) { 8100 if (path && strcmp(path, map->pin_path)) { 8101 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 8102 bpf_map__name(map), map->pin_path, path); 8103 return libbpf_err(-EINVAL); 8104 } else if (map->pinned) { 8105 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", 8106 bpf_map__name(map), map->pin_path); 8107 return 0; 8108 } 8109 } else { 8110 if (!path) { 8111 pr_warn("missing a path to pin map '%s' at\n", 8112 bpf_map__name(map)); 8113 return libbpf_err(-EINVAL); 8114 } else if (map->pinned) { 8115 pr_warn("map '%s' already pinned\n", bpf_map__name(map)); 8116 return libbpf_err(-EEXIST); 8117 } 8118 8119 map->pin_path = strdup(path); 8120 if (!map->pin_path) { 8121 err = -errno; 8122 goto out_err; 8123 } 8124 } 8125 8126 err = make_parent_dir(map->pin_path); 8127 if (err) 8128 return libbpf_err(err); 8129 8130 err = check_path(map->pin_path); 8131 if (err) 8132 return libbpf_err(err); 8133 8134 if (bpf_obj_pin(map->fd, map->pin_path)) { 8135 err = -errno; 8136 goto out_err; 8137 } 8138 8139 map->pinned = true; 8140 pr_debug("pinned map '%s'\n", map->pin_path); 8141 8142 return 0; 8143 8144 out_err: 8145 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 8146 pr_warn("failed to pin map: %s\n", cp); 8147 return libbpf_err(err); 8148 } 8149 8150 int bpf_map__unpin(struct bpf_map *map, const char *path) 8151 { 8152 int err; 8153 8154 if (map == NULL) { 8155 pr_warn("invalid map pointer\n"); 8156 return libbpf_err(-EINVAL); 8157 } 8158 8159 if (map->pin_path) { 8160 if (path && strcmp(path, map->pin_path)) { 8161 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 8162 bpf_map__name(map), map->pin_path, path); 8163 return libbpf_err(-EINVAL); 8164 } 8165 path = map->pin_path; 8166 } else if (!path) { 8167 pr_warn("no path to unpin map '%s' from\n", 8168 bpf_map__name(map)); 8169 return libbpf_err(-EINVAL); 8170 } 8171 8172 err = check_path(path); 8173 if (err) 8174 return libbpf_err(err); 8175 8176 err = unlink(path); 8177 if (err != 0) 8178 return libbpf_err(-errno); 8179 8180 map->pinned = false; 8181 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); 8182 8183 return 0; 8184 } 8185 8186 int bpf_map__set_pin_path(struct bpf_map *map, const char *path) 8187 { 8188 char *new = NULL; 8189 8190 if (path) { 8191 new = strdup(path); 8192 if (!new) 8193 return libbpf_err(-errno); 8194 } 8195 8196 free(map->pin_path); 8197 map->pin_path = new; 8198 return 0; 8199 } 8200 8201 __alias(bpf_map__pin_path) 8202 const char *bpf_map__get_pin_path(const struct bpf_map *map); 8203 8204 const char *bpf_map__pin_path(const struct bpf_map *map) 8205 { 8206 return map->pin_path; 8207 } 8208 8209 bool bpf_map__is_pinned(const struct bpf_map *map) 8210 { 8211 return map->pinned; 8212 } 8213 8214 static void sanitize_pin_path(char *s) 8215 { 8216 /* bpffs disallows periods in path names */ 8217 while (*s) { 8218 if (*s == '.') 8219 *s = '_'; 8220 s++; 8221 } 8222 } 8223 8224 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 8225 { 8226 struct bpf_map *map; 8227 int err; 8228 8229 if (!obj) 8230 return libbpf_err(-ENOENT); 8231 8232 if (!obj->loaded) { 8233 pr_warn("object not yet loaded; load it first\n"); 8234 return libbpf_err(-ENOENT); 8235 } 8236 8237 bpf_object__for_each_map(map, obj) { 8238 char *pin_path = NULL; 8239 char buf[PATH_MAX]; 8240 8241 if (!map->autocreate) 8242 continue; 8243 8244 if (path) { 8245 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 8246 if (err) 8247 goto err_unpin_maps; 8248 sanitize_pin_path(buf); 8249 pin_path = buf; 8250 } else if (!map->pin_path) { 8251 continue; 8252 } 8253 8254 err = bpf_map__pin(map, pin_path); 8255 if (err) 8256 goto err_unpin_maps; 8257 } 8258 8259 return 0; 8260 8261 err_unpin_maps: 8262 while ((map = bpf_object__prev_map(obj, map))) { 8263 if (!map->pin_path) 8264 continue; 8265 8266 bpf_map__unpin(map, NULL); 8267 } 8268 8269 return libbpf_err(err); 8270 } 8271 8272 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 8273 { 8274 struct bpf_map *map; 8275 int err; 8276 8277 if (!obj) 8278 return libbpf_err(-ENOENT); 8279 8280 bpf_object__for_each_map(map, obj) { 8281 char *pin_path = NULL; 8282 char buf[PATH_MAX]; 8283 8284 if (path) { 8285 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 8286 if (err) 8287 return libbpf_err(err); 8288 sanitize_pin_path(buf); 8289 pin_path = buf; 8290 } else if (!map->pin_path) { 8291 continue; 8292 } 8293 8294 err = bpf_map__unpin(map, pin_path); 8295 if (err) 8296 return libbpf_err(err); 8297 } 8298 8299 return 0; 8300 } 8301 8302 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 8303 { 8304 struct bpf_program *prog; 8305 char buf[PATH_MAX]; 8306 int err; 8307 8308 if (!obj) 8309 return libbpf_err(-ENOENT); 8310 8311 if (!obj->loaded) { 8312 pr_warn("object not yet loaded; load it first\n"); 8313 return libbpf_err(-ENOENT); 8314 } 8315 8316 bpf_object__for_each_program(prog, obj) { 8317 err = pathname_concat(buf, sizeof(buf), path, prog->name); 8318 if (err) 8319 goto err_unpin_programs; 8320 8321 err = bpf_program__pin(prog, buf); 8322 if (err) 8323 goto err_unpin_programs; 8324 } 8325 8326 return 0; 8327 8328 err_unpin_programs: 8329 while ((prog = bpf_object__prev_program(obj, prog))) { 8330 if (pathname_concat(buf, sizeof(buf), path, prog->name)) 8331 continue; 8332 8333 bpf_program__unpin(prog, buf); 8334 } 8335 8336 return libbpf_err(err); 8337 } 8338 8339 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 8340 { 8341 struct bpf_program *prog; 8342 int err; 8343 8344 if (!obj) 8345 return libbpf_err(-ENOENT); 8346 8347 bpf_object__for_each_program(prog, obj) { 8348 char buf[PATH_MAX]; 8349 8350 err = pathname_concat(buf, sizeof(buf), path, prog->name); 8351 if (err) 8352 return libbpf_err(err); 8353 8354 err = bpf_program__unpin(prog, buf); 8355 if (err) 8356 return libbpf_err(err); 8357 } 8358 8359 return 0; 8360 } 8361 8362 int bpf_object__pin(struct bpf_object *obj, const char *path) 8363 { 8364 int err; 8365 8366 err = bpf_object__pin_maps(obj, path); 8367 if (err) 8368 return libbpf_err(err); 8369 8370 err = bpf_object__pin_programs(obj, path); 8371 if (err) { 8372 bpf_object__unpin_maps(obj, path); 8373 return libbpf_err(err); 8374 } 8375 8376 return 0; 8377 } 8378 8379 int bpf_object__unpin(struct bpf_object *obj, const char *path) 8380 { 8381 int err; 8382 8383 err = bpf_object__unpin_programs(obj, path); 8384 if (err) 8385 return libbpf_err(err); 8386 8387 err = bpf_object__unpin_maps(obj, path); 8388 if (err) 8389 return libbpf_err(err); 8390 8391 return 0; 8392 } 8393 8394 static void bpf_map__destroy(struct bpf_map *map) 8395 { 8396 if (map->inner_map) { 8397 bpf_map__destroy(map->inner_map); 8398 zfree(&map->inner_map); 8399 } 8400 8401 zfree(&map->init_slots); 8402 map->init_slots_sz = 0; 8403 8404 if (map->mmaped) { 8405 size_t mmap_sz; 8406 8407 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 8408 munmap(map->mmaped, mmap_sz); 8409 map->mmaped = NULL; 8410 } 8411 8412 if (map->st_ops) { 8413 zfree(&map->st_ops->data); 8414 zfree(&map->st_ops->progs); 8415 zfree(&map->st_ops->kern_func_off); 8416 zfree(&map->st_ops); 8417 } 8418 8419 zfree(&map->name); 8420 zfree(&map->real_name); 8421 zfree(&map->pin_path); 8422 8423 if (map->fd >= 0) 8424 zclose(map->fd); 8425 } 8426 8427 void bpf_object__close(struct bpf_object *obj) 8428 { 8429 size_t i; 8430 8431 if (IS_ERR_OR_NULL(obj)) 8432 return; 8433 8434 usdt_manager_free(obj->usdt_man); 8435 obj->usdt_man = NULL; 8436 8437 bpf_gen__free(obj->gen_loader); 8438 bpf_object__elf_finish(obj); 8439 bpf_object_unload(obj); 8440 btf__free(obj->btf); 8441 btf__free(obj->btf_vmlinux); 8442 btf_ext__free(obj->btf_ext); 8443 8444 for (i = 0; i < obj->nr_maps; i++) 8445 bpf_map__destroy(&obj->maps[i]); 8446 8447 zfree(&obj->btf_custom_path); 8448 zfree(&obj->kconfig); 8449 8450 for (i = 0; i < obj->nr_extern; i++) 8451 zfree(&obj->externs[i].essent_name); 8452 8453 zfree(&obj->externs); 8454 obj->nr_extern = 0; 8455 8456 zfree(&obj->maps); 8457 obj->nr_maps = 0; 8458 8459 if (obj->programs && obj->nr_programs) { 8460 for (i = 0; i < obj->nr_programs; i++) 8461 bpf_program__exit(&obj->programs[i]); 8462 } 8463 zfree(&obj->programs); 8464 8465 free(obj); 8466 } 8467 8468 const char *bpf_object__name(const struct bpf_object *obj) 8469 { 8470 return obj ? obj->name : libbpf_err_ptr(-EINVAL); 8471 } 8472 8473 unsigned int bpf_object__kversion(const struct bpf_object *obj) 8474 { 8475 return obj ? obj->kern_version : 0; 8476 } 8477 8478 struct btf *bpf_object__btf(const struct bpf_object *obj) 8479 { 8480 return obj ? obj->btf : NULL; 8481 } 8482 8483 int bpf_object__btf_fd(const struct bpf_object *obj) 8484 { 8485 return obj->btf ? btf__fd(obj->btf) : -1; 8486 } 8487 8488 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version) 8489 { 8490 if (obj->loaded) 8491 return libbpf_err(-EINVAL); 8492 8493 obj->kern_version = kern_version; 8494 8495 return 0; 8496 } 8497 8498 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) 8499 { 8500 struct bpf_gen *gen; 8501 8502 if (!opts) 8503 return -EFAULT; 8504 if (!OPTS_VALID(opts, gen_loader_opts)) 8505 return -EINVAL; 8506 gen = calloc(sizeof(*gen), 1); 8507 if (!gen) 8508 return -ENOMEM; 8509 gen->opts = opts; 8510 obj->gen_loader = gen; 8511 return 0; 8512 } 8513 8514 static struct bpf_program * 8515 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 8516 bool forward) 8517 { 8518 size_t nr_programs = obj->nr_programs; 8519 ssize_t idx; 8520 8521 if (!nr_programs) 8522 return NULL; 8523 8524 if (!p) 8525 /* Iter from the beginning */ 8526 return forward ? &obj->programs[0] : 8527 &obj->programs[nr_programs - 1]; 8528 8529 if (p->obj != obj) { 8530 pr_warn("error: program handler doesn't match object\n"); 8531 return errno = EINVAL, NULL; 8532 } 8533 8534 idx = (p - obj->programs) + (forward ? 1 : -1); 8535 if (idx >= obj->nr_programs || idx < 0) 8536 return NULL; 8537 return &obj->programs[idx]; 8538 } 8539 8540 struct bpf_program * 8541 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev) 8542 { 8543 struct bpf_program *prog = prev; 8544 8545 do { 8546 prog = __bpf_program__iter(prog, obj, true); 8547 } while (prog && prog_is_subprog(obj, prog)); 8548 8549 return prog; 8550 } 8551 8552 struct bpf_program * 8553 bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next) 8554 { 8555 struct bpf_program *prog = next; 8556 8557 do { 8558 prog = __bpf_program__iter(prog, obj, false); 8559 } while (prog && prog_is_subprog(obj, prog)); 8560 8561 return prog; 8562 } 8563 8564 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 8565 { 8566 prog->prog_ifindex = ifindex; 8567 } 8568 8569 const char *bpf_program__name(const struct bpf_program *prog) 8570 { 8571 return prog->name; 8572 } 8573 8574 const char *bpf_program__section_name(const struct bpf_program *prog) 8575 { 8576 return prog->sec_name; 8577 } 8578 8579 bool bpf_program__autoload(const struct bpf_program *prog) 8580 { 8581 return prog->autoload; 8582 } 8583 8584 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) 8585 { 8586 if (prog->obj->loaded) 8587 return libbpf_err(-EINVAL); 8588 8589 prog->autoload = autoload; 8590 return 0; 8591 } 8592 8593 bool bpf_program__autoattach(const struct bpf_program *prog) 8594 { 8595 return prog->autoattach; 8596 } 8597 8598 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach) 8599 { 8600 prog->autoattach = autoattach; 8601 } 8602 8603 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog) 8604 { 8605 return prog->insns; 8606 } 8607 8608 size_t bpf_program__insn_cnt(const struct bpf_program *prog) 8609 { 8610 return prog->insns_cnt; 8611 } 8612 8613 int bpf_program__set_insns(struct bpf_program *prog, 8614 struct bpf_insn *new_insns, size_t new_insn_cnt) 8615 { 8616 struct bpf_insn *insns; 8617 8618 if (prog->obj->loaded) 8619 return -EBUSY; 8620 8621 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); 8622 /* NULL is a valid return from reallocarray if the new count is zero */ 8623 if (!insns && new_insn_cnt) { 8624 pr_warn("prog '%s': failed to realloc prog code\n", prog->name); 8625 return -ENOMEM; 8626 } 8627 memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns)); 8628 8629 prog->insns = insns; 8630 prog->insns_cnt = new_insn_cnt; 8631 return 0; 8632 } 8633 8634 int bpf_program__fd(const struct bpf_program *prog) 8635 { 8636 if (!prog) 8637 return libbpf_err(-EINVAL); 8638 8639 if (prog->fd < 0) 8640 return libbpf_err(-ENOENT); 8641 8642 return prog->fd; 8643 } 8644 8645 __alias(bpf_program__type) 8646 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog); 8647 8648 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog) 8649 { 8650 return prog->type; 8651 } 8652 8653 static size_t custom_sec_def_cnt; 8654 static struct bpf_sec_def *custom_sec_defs; 8655 static struct bpf_sec_def custom_fallback_def; 8656 static bool has_custom_fallback_def; 8657 static int last_custom_sec_def_handler_id; 8658 8659 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 8660 { 8661 if (prog->obj->loaded) 8662 return libbpf_err(-EBUSY); 8663 8664 /* if type is not changed, do nothing */ 8665 if (prog->type == type) 8666 return 0; 8667 8668 prog->type = type; 8669 8670 /* If a program type was changed, we need to reset associated SEC() 8671 * handler, as it will be invalid now. The only exception is a generic 8672 * fallback handler, which by definition is program type-agnostic and 8673 * is a catch-all custom handler, optionally set by the application, 8674 * so should be able to handle any type of BPF program. 8675 */ 8676 if (prog->sec_def != &custom_fallback_def) 8677 prog->sec_def = NULL; 8678 return 0; 8679 } 8680 8681 __alias(bpf_program__expected_attach_type) 8682 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog); 8683 8684 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog) 8685 { 8686 return prog->expected_attach_type; 8687 } 8688 8689 int bpf_program__set_expected_attach_type(struct bpf_program *prog, 8690 enum bpf_attach_type type) 8691 { 8692 if (prog->obj->loaded) 8693 return libbpf_err(-EBUSY); 8694 8695 prog->expected_attach_type = type; 8696 return 0; 8697 } 8698 8699 __u32 bpf_program__flags(const struct bpf_program *prog) 8700 { 8701 return prog->prog_flags; 8702 } 8703 8704 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags) 8705 { 8706 if (prog->obj->loaded) 8707 return libbpf_err(-EBUSY); 8708 8709 prog->prog_flags = flags; 8710 return 0; 8711 } 8712 8713 __u32 bpf_program__log_level(const struct bpf_program *prog) 8714 { 8715 return prog->log_level; 8716 } 8717 8718 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level) 8719 { 8720 if (prog->obj->loaded) 8721 return libbpf_err(-EBUSY); 8722 8723 prog->log_level = log_level; 8724 return 0; 8725 } 8726 8727 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size) 8728 { 8729 *log_size = prog->log_size; 8730 return prog->log_buf; 8731 } 8732 8733 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size) 8734 { 8735 if (log_size && !log_buf) 8736 return -EINVAL; 8737 if (prog->log_size > UINT_MAX) 8738 return -EINVAL; 8739 if (prog->obj->loaded) 8740 return -EBUSY; 8741 8742 prog->log_buf = log_buf; 8743 prog->log_size = log_size; 8744 return 0; 8745 } 8746 8747 #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \ 8748 .sec = (char *)sec_pfx, \ 8749 .prog_type = BPF_PROG_TYPE_##ptype, \ 8750 .expected_attach_type = atype, \ 8751 .cookie = (long)(flags), \ 8752 .prog_prepare_load_fn = libbpf_prepare_prog_load, \ 8753 __VA_ARGS__ \ 8754 } 8755 8756 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8757 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8758 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8759 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8760 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8761 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8762 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8763 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8764 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8765 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8766 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8767 8768 static const struct bpf_sec_def section_defs[] = { 8769 SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE), 8770 SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE), 8771 SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE), 8772 SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), 8773 SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), 8774 SEC_DEF("uprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), 8775 SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), 8776 SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), 8777 SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), 8778 SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), 8779 SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), 8780 SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi), 8781 SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi), 8782 SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), 8783 SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), 8784 SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), 8785 SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), 8786 SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt), 8787 SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT | SEC_SLEEPABLE, attach_usdt), 8788 SEC_DEF("tc/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */ 8789 SEC_DEF("tc/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), /* alias for tcx */ 8790 SEC_DEF("tcx/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), 8791 SEC_DEF("tcx/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), 8792 SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 8793 SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 8794 SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 8795 SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp), 8796 SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp), 8797 SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), 8798 SEC_DEF("raw_tp+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), 8799 SEC_DEF("raw_tracepoint.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), 8800 SEC_DEF("raw_tp.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), 8801 SEC_DEF("tp_btf+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace), 8802 SEC_DEF("fentry+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace), 8803 SEC_DEF("fmod_ret+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace), 8804 SEC_DEF("fexit+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace), 8805 SEC_DEF("fentry.s+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 8806 SEC_DEF("fmod_ret.s+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 8807 SEC_DEF("fexit.s+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 8808 SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace), 8809 SEC_DEF("lsm+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm), 8810 SEC_DEF("lsm.s+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm), 8811 SEC_DEF("lsm_cgroup+", LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF), 8812 SEC_DEF("iter+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter), 8813 SEC_DEF("iter.s+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter), 8814 SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE), 8815 SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS), 8816 SEC_DEF("xdp/devmap", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE), 8817 SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS), 8818 SEC_DEF("xdp/cpumap", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE), 8819 SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS), 8820 SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT), 8821 SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE), 8822 SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE), 8823 SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE), 8824 SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE), 8825 SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE), 8826 SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT), 8827 SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT), 8828 SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT), 8829 SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE), 8830 SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT), 8831 SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT), 8832 SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT), 8833 SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT), 8834 SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT), 8835 SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE), 8836 SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE), 8837 SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE), 8838 SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT), 8839 SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE), 8840 SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE), 8841 SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE), 8842 SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE), 8843 SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE), 8844 SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE), 8845 SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE), 8846 SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE), 8847 SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE), 8848 SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE), 8849 SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE), 8850 SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE), 8851 SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE), 8852 SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE), 8853 SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE), 8854 SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE), 8855 SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE), 8856 SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT), 8857 SEC_DEF("struct_ops+", STRUCT_OPS, 0, SEC_NONE), 8858 SEC_DEF("struct_ops.s+", STRUCT_OPS, 0, SEC_SLEEPABLE), 8859 SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE), 8860 SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE), 8861 }; 8862 8863 int libbpf_register_prog_handler(const char *sec, 8864 enum bpf_prog_type prog_type, 8865 enum bpf_attach_type exp_attach_type, 8866 const struct libbpf_prog_handler_opts *opts) 8867 { 8868 struct bpf_sec_def *sec_def; 8869 8870 if (!OPTS_VALID(opts, libbpf_prog_handler_opts)) 8871 return libbpf_err(-EINVAL); 8872 8873 if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */ 8874 return libbpf_err(-E2BIG); 8875 8876 if (sec) { 8877 sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1, 8878 sizeof(*sec_def)); 8879 if (!sec_def) 8880 return libbpf_err(-ENOMEM); 8881 8882 custom_sec_defs = sec_def; 8883 sec_def = &custom_sec_defs[custom_sec_def_cnt]; 8884 } else { 8885 if (has_custom_fallback_def) 8886 return libbpf_err(-EBUSY); 8887 8888 sec_def = &custom_fallback_def; 8889 } 8890 8891 sec_def->sec = sec ? strdup(sec) : NULL; 8892 if (sec && !sec_def->sec) 8893 return libbpf_err(-ENOMEM); 8894 8895 sec_def->prog_type = prog_type; 8896 sec_def->expected_attach_type = exp_attach_type; 8897 sec_def->cookie = OPTS_GET(opts, cookie, 0); 8898 8899 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL); 8900 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL); 8901 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL); 8902 8903 sec_def->handler_id = ++last_custom_sec_def_handler_id; 8904 8905 if (sec) 8906 custom_sec_def_cnt++; 8907 else 8908 has_custom_fallback_def = true; 8909 8910 return sec_def->handler_id; 8911 } 8912 8913 int libbpf_unregister_prog_handler(int handler_id) 8914 { 8915 struct bpf_sec_def *sec_defs; 8916 int i; 8917 8918 if (handler_id <= 0) 8919 return libbpf_err(-EINVAL); 8920 8921 if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) { 8922 memset(&custom_fallback_def, 0, sizeof(custom_fallback_def)); 8923 has_custom_fallback_def = false; 8924 return 0; 8925 } 8926 8927 for (i = 0; i < custom_sec_def_cnt; i++) { 8928 if (custom_sec_defs[i].handler_id == handler_id) 8929 break; 8930 } 8931 8932 if (i == custom_sec_def_cnt) 8933 return libbpf_err(-ENOENT); 8934 8935 free(custom_sec_defs[i].sec); 8936 for (i = i + 1; i < custom_sec_def_cnt; i++) 8937 custom_sec_defs[i - 1] = custom_sec_defs[i]; 8938 custom_sec_def_cnt--; 8939 8940 /* try to shrink the array, but it's ok if we couldn't */ 8941 sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs)); 8942 /* if new count is zero, reallocarray can return a valid NULL result; 8943 * in this case the previous pointer will be freed, so we *have to* 8944 * reassign old pointer to the new value (even if it's NULL) 8945 */ 8946 if (sec_defs || custom_sec_def_cnt == 0) 8947 custom_sec_defs = sec_defs; 8948 8949 return 0; 8950 } 8951 8952 static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name) 8953 { 8954 size_t len = strlen(sec_def->sec); 8955 8956 /* "type/" always has to have proper SEC("type/extras") form */ 8957 if (sec_def->sec[len - 1] == '/') { 8958 if (str_has_pfx(sec_name, sec_def->sec)) 8959 return true; 8960 return false; 8961 } 8962 8963 /* "type+" means it can be either exact SEC("type") or 8964 * well-formed SEC("type/extras") with proper '/' separator 8965 */ 8966 if (sec_def->sec[len - 1] == '+') { 8967 len--; 8968 /* not even a prefix */ 8969 if (strncmp(sec_name, sec_def->sec, len) != 0) 8970 return false; 8971 /* exact match or has '/' separator */ 8972 if (sec_name[len] == '\0' || sec_name[len] == '/') 8973 return true; 8974 return false; 8975 } 8976 8977 return strcmp(sec_name, sec_def->sec) == 0; 8978 } 8979 8980 static const struct bpf_sec_def *find_sec_def(const char *sec_name) 8981 { 8982 const struct bpf_sec_def *sec_def; 8983 int i, n; 8984 8985 n = custom_sec_def_cnt; 8986 for (i = 0; i < n; i++) { 8987 sec_def = &custom_sec_defs[i]; 8988 if (sec_def_matches(sec_def, sec_name)) 8989 return sec_def; 8990 } 8991 8992 n = ARRAY_SIZE(section_defs); 8993 for (i = 0; i < n; i++) { 8994 sec_def = §ion_defs[i]; 8995 if (sec_def_matches(sec_def, sec_name)) 8996 return sec_def; 8997 } 8998 8999 if (has_custom_fallback_def) 9000 return &custom_fallback_def; 9001 9002 return NULL; 9003 } 9004 9005 #define MAX_TYPE_NAME_SIZE 32 9006 9007 static char *libbpf_get_type_names(bool attach_type) 9008 { 9009 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; 9010 char *buf; 9011 9012 buf = malloc(len); 9013 if (!buf) 9014 return NULL; 9015 9016 buf[0] = '\0'; 9017 /* Forge string buf with all available names */ 9018 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 9019 const struct bpf_sec_def *sec_def = §ion_defs[i]; 9020 9021 if (attach_type) { 9022 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) 9023 continue; 9024 9025 if (!(sec_def->cookie & SEC_ATTACHABLE)) 9026 continue; 9027 } 9028 9029 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { 9030 free(buf); 9031 return NULL; 9032 } 9033 strcat(buf, " "); 9034 strcat(buf, section_defs[i].sec); 9035 } 9036 9037 return buf; 9038 } 9039 9040 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 9041 enum bpf_attach_type *expected_attach_type) 9042 { 9043 const struct bpf_sec_def *sec_def; 9044 char *type_names; 9045 9046 if (!name) 9047 return libbpf_err(-EINVAL); 9048 9049 sec_def = find_sec_def(name); 9050 if (sec_def) { 9051 *prog_type = sec_def->prog_type; 9052 *expected_attach_type = sec_def->expected_attach_type; 9053 return 0; 9054 } 9055 9056 pr_debug("failed to guess program type from ELF section '%s'\n", name); 9057 type_names = libbpf_get_type_names(false); 9058 if (type_names != NULL) { 9059 pr_debug("supported section(type) names are:%s\n", type_names); 9060 free(type_names); 9061 } 9062 9063 return libbpf_err(-ESRCH); 9064 } 9065 9066 const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t) 9067 { 9068 if (t < 0 || t >= ARRAY_SIZE(attach_type_name)) 9069 return NULL; 9070 9071 return attach_type_name[t]; 9072 } 9073 9074 const char *libbpf_bpf_link_type_str(enum bpf_link_type t) 9075 { 9076 if (t < 0 || t >= ARRAY_SIZE(link_type_name)) 9077 return NULL; 9078 9079 return link_type_name[t]; 9080 } 9081 9082 const char *libbpf_bpf_map_type_str(enum bpf_map_type t) 9083 { 9084 if (t < 0 || t >= ARRAY_SIZE(map_type_name)) 9085 return NULL; 9086 9087 return map_type_name[t]; 9088 } 9089 9090 const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t) 9091 { 9092 if (t < 0 || t >= ARRAY_SIZE(prog_type_name)) 9093 return NULL; 9094 9095 return prog_type_name[t]; 9096 } 9097 9098 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, 9099 int sec_idx, 9100 size_t offset) 9101 { 9102 struct bpf_map *map; 9103 size_t i; 9104 9105 for (i = 0; i < obj->nr_maps; i++) { 9106 map = &obj->maps[i]; 9107 if (!bpf_map__is_struct_ops(map)) 9108 continue; 9109 if (map->sec_idx == sec_idx && 9110 map->sec_offset <= offset && 9111 offset - map->sec_offset < map->def.value_size) 9112 return map; 9113 } 9114 9115 return NULL; 9116 } 9117 9118 /* Collect the reloc from ELF and populate the st_ops->progs[] */ 9119 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 9120 Elf64_Shdr *shdr, Elf_Data *data) 9121 { 9122 const struct btf_member *member; 9123 struct bpf_struct_ops *st_ops; 9124 struct bpf_program *prog; 9125 unsigned int shdr_idx; 9126 const struct btf *btf; 9127 struct bpf_map *map; 9128 unsigned int moff, insn_idx; 9129 const char *name; 9130 __u32 member_idx; 9131 Elf64_Sym *sym; 9132 Elf64_Rel *rel; 9133 int i, nrels; 9134 9135 btf = obj->btf; 9136 nrels = shdr->sh_size / shdr->sh_entsize; 9137 for (i = 0; i < nrels; i++) { 9138 rel = elf_rel_by_idx(data, i); 9139 if (!rel) { 9140 pr_warn("struct_ops reloc: failed to get %d reloc\n", i); 9141 return -LIBBPF_ERRNO__FORMAT; 9142 } 9143 9144 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); 9145 if (!sym) { 9146 pr_warn("struct_ops reloc: symbol %zx not found\n", 9147 (size_t)ELF64_R_SYM(rel->r_info)); 9148 return -LIBBPF_ERRNO__FORMAT; 9149 } 9150 9151 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; 9152 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset); 9153 if (!map) { 9154 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n", 9155 (size_t)rel->r_offset); 9156 return -EINVAL; 9157 } 9158 9159 moff = rel->r_offset - map->sec_offset; 9160 shdr_idx = sym->st_shndx; 9161 st_ops = map->st_ops; 9162 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", 9163 map->name, 9164 (long long)(rel->r_info >> 32), 9165 (long long)sym->st_value, 9166 shdr_idx, (size_t)rel->r_offset, 9167 map->sec_offset, sym->st_name, name); 9168 9169 if (shdr_idx >= SHN_LORESERVE) { 9170 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n", 9171 map->name, (size_t)rel->r_offset, shdr_idx); 9172 return -LIBBPF_ERRNO__RELOC; 9173 } 9174 if (sym->st_value % BPF_INSN_SZ) { 9175 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n", 9176 map->name, (unsigned long long)sym->st_value); 9177 return -LIBBPF_ERRNO__FORMAT; 9178 } 9179 insn_idx = sym->st_value / BPF_INSN_SZ; 9180 9181 member = find_member_by_offset(st_ops->type, moff * 8); 9182 if (!member) { 9183 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", 9184 map->name, moff); 9185 return -EINVAL; 9186 } 9187 member_idx = member - btf_members(st_ops->type); 9188 name = btf__name_by_offset(btf, member->name_off); 9189 9190 if (!resolve_func_ptr(btf, member->type, NULL)) { 9191 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", 9192 map->name, name); 9193 return -EINVAL; 9194 } 9195 9196 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); 9197 if (!prog) { 9198 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", 9199 map->name, shdr_idx, name); 9200 return -EINVAL; 9201 } 9202 9203 /* prevent the use of BPF prog with invalid type */ 9204 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) { 9205 pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n", 9206 map->name, prog->name); 9207 return -EINVAL; 9208 } 9209 9210 /* if we haven't yet processed this BPF program, record proper 9211 * attach_btf_id and member_idx 9212 */ 9213 if (!prog->attach_btf_id) { 9214 prog->attach_btf_id = st_ops->type_id; 9215 prog->expected_attach_type = member_idx; 9216 } 9217 9218 /* struct_ops BPF prog can be re-used between multiple 9219 * .struct_ops & .struct_ops.link as long as it's the 9220 * same struct_ops struct definition and the same 9221 * function pointer field 9222 */ 9223 if (prog->attach_btf_id != st_ops->type_id || 9224 prog->expected_attach_type != member_idx) { 9225 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n", 9226 map->name, prog->name, prog->sec_name, prog->type, 9227 prog->attach_btf_id, prog->expected_attach_type, name); 9228 return -EINVAL; 9229 } 9230 9231 st_ops->progs[member_idx] = prog; 9232 } 9233 9234 return 0; 9235 } 9236 9237 #define BTF_TRACE_PREFIX "btf_trace_" 9238 #define BTF_LSM_PREFIX "bpf_lsm_" 9239 #define BTF_ITER_PREFIX "bpf_iter_" 9240 #define BTF_MAX_NAME_SIZE 128 9241 9242 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, 9243 const char **prefix, int *kind) 9244 { 9245 switch (attach_type) { 9246 case BPF_TRACE_RAW_TP: 9247 *prefix = BTF_TRACE_PREFIX; 9248 *kind = BTF_KIND_TYPEDEF; 9249 break; 9250 case BPF_LSM_MAC: 9251 case BPF_LSM_CGROUP: 9252 *prefix = BTF_LSM_PREFIX; 9253 *kind = BTF_KIND_FUNC; 9254 break; 9255 case BPF_TRACE_ITER: 9256 *prefix = BTF_ITER_PREFIX; 9257 *kind = BTF_KIND_FUNC; 9258 break; 9259 default: 9260 *prefix = ""; 9261 *kind = BTF_KIND_FUNC; 9262 } 9263 } 9264 9265 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 9266 const char *name, __u32 kind) 9267 { 9268 char btf_type_name[BTF_MAX_NAME_SIZE]; 9269 int ret; 9270 9271 ret = snprintf(btf_type_name, sizeof(btf_type_name), 9272 "%s%s", prefix, name); 9273 /* snprintf returns the number of characters written excluding the 9274 * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it 9275 * indicates truncation. 9276 */ 9277 if (ret < 0 || ret >= sizeof(btf_type_name)) 9278 return -ENAMETOOLONG; 9279 return btf__find_by_name_kind(btf, btf_type_name, kind); 9280 } 9281 9282 static inline int find_attach_btf_id(struct btf *btf, const char *name, 9283 enum bpf_attach_type attach_type) 9284 { 9285 const char *prefix; 9286 int kind; 9287 9288 btf_get_kernel_prefix_kind(attach_type, &prefix, &kind); 9289 return find_btf_by_prefix_kind(btf, prefix, name, kind); 9290 } 9291 9292 int libbpf_find_vmlinux_btf_id(const char *name, 9293 enum bpf_attach_type attach_type) 9294 { 9295 struct btf *btf; 9296 int err; 9297 9298 btf = btf__load_vmlinux_btf(); 9299 err = libbpf_get_error(btf); 9300 if (err) { 9301 pr_warn("vmlinux BTF is not found\n"); 9302 return libbpf_err(err); 9303 } 9304 9305 err = find_attach_btf_id(btf, name, attach_type); 9306 if (err <= 0) 9307 pr_warn("%s is not found in vmlinux BTF\n", name); 9308 9309 btf__free(btf); 9310 return libbpf_err(err); 9311 } 9312 9313 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) 9314 { 9315 struct bpf_prog_info info; 9316 __u32 info_len = sizeof(info); 9317 struct btf *btf; 9318 int err; 9319 9320 memset(&info, 0, info_len); 9321 err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len); 9322 if (err) { 9323 pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %d\n", 9324 attach_prog_fd, err); 9325 return err; 9326 } 9327 9328 err = -EINVAL; 9329 if (!info.btf_id) { 9330 pr_warn("The target program doesn't have BTF\n"); 9331 goto out; 9332 } 9333 btf = btf__load_from_kernel_by_id(info.btf_id); 9334 err = libbpf_get_error(btf); 9335 if (err) { 9336 pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err); 9337 goto out; 9338 } 9339 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 9340 btf__free(btf); 9341 if (err <= 0) { 9342 pr_warn("%s is not found in prog's BTF\n", name); 9343 goto out; 9344 } 9345 out: 9346 return err; 9347 } 9348 9349 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name, 9350 enum bpf_attach_type attach_type, 9351 int *btf_obj_fd, int *btf_type_id) 9352 { 9353 int ret, i; 9354 9355 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type); 9356 if (ret > 0) { 9357 *btf_obj_fd = 0; /* vmlinux BTF */ 9358 *btf_type_id = ret; 9359 return 0; 9360 } 9361 if (ret != -ENOENT) 9362 return ret; 9363 9364 ret = load_module_btfs(obj); 9365 if (ret) 9366 return ret; 9367 9368 for (i = 0; i < obj->btf_module_cnt; i++) { 9369 const struct module_btf *mod = &obj->btf_modules[i]; 9370 9371 ret = find_attach_btf_id(mod->btf, attach_name, attach_type); 9372 if (ret > 0) { 9373 *btf_obj_fd = mod->fd; 9374 *btf_type_id = ret; 9375 return 0; 9376 } 9377 if (ret == -ENOENT) 9378 continue; 9379 9380 return ret; 9381 } 9382 9383 return -ESRCH; 9384 } 9385 9386 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, 9387 int *btf_obj_fd, int *btf_type_id) 9388 { 9389 enum bpf_attach_type attach_type = prog->expected_attach_type; 9390 __u32 attach_prog_fd = prog->attach_prog_fd; 9391 int err = 0; 9392 9393 /* BPF program's BTF ID */ 9394 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) { 9395 if (!attach_prog_fd) { 9396 pr_warn("prog '%s': attach program FD is not set\n", prog->name); 9397 return -EINVAL; 9398 } 9399 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd); 9400 if (err < 0) { 9401 pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n", 9402 prog->name, attach_prog_fd, attach_name, err); 9403 return err; 9404 } 9405 *btf_obj_fd = 0; 9406 *btf_type_id = err; 9407 return 0; 9408 } 9409 9410 /* kernel/module BTF ID */ 9411 if (prog->obj->gen_loader) { 9412 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); 9413 *btf_obj_fd = 0; 9414 *btf_type_id = 1; 9415 } else { 9416 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id); 9417 } 9418 if (err) { 9419 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n", 9420 prog->name, attach_name, err); 9421 return err; 9422 } 9423 return 0; 9424 } 9425 9426 int libbpf_attach_type_by_name(const char *name, 9427 enum bpf_attach_type *attach_type) 9428 { 9429 char *type_names; 9430 const struct bpf_sec_def *sec_def; 9431 9432 if (!name) 9433 return libbpf_err(-EINVAL); 9434 9435 sec_def = find_sec_def(name); 9436 if (!sec_def) { 9437 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); 9438 type_names = libbpf_get_type_names(true); 9439 if (type_names != NULL) { 9440 pr_debug("attachable section(type) names are:%s\n", type_names); 9441 free(type_names); 9442 } 9443 9444 return libbpf_err(-EINVAL); 9445 } 9446 9447 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) 9448 return libbpf_err(-EINVAL); 9449 if (!(sec_def->cookie & SEC_ATTACHABLE)) 9450 return libbpf_err(-EINVAL); 9451 9452 *attach_type = sec_def->expected_attach_type; 9453 return 0; 9454 } 9455 9456 int bpf_map__fd(const struct bpf_map *map) 9457 { 9458 return map ? map->fd : libbpf_err(-EINVAL); 9459 } 9460 9461 static bool map_uses_real_name(const struct bpf_map *map) 9462 { 9463 /* Since libbpf started to support custom .data.* and .rodata.* maps, 9464 * their user-visible name differs from kernel-visible name. Users see 9465 * such map's corresponding ELF section name as a map name. 9466 * This check distinguishes .data/.rodata from .data.* and .rodata.* 9467 * maps to know which name has to be returned to the user. 9468 */ 9469 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0) 9470 return true; 9471 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0) 9472 return true; 9473 return false; 9474 } 9475 9476 const char *bpf_map__name(const struct bpf_map *map) 9477 { 9478 if (!map) 9479 return NULL; 9480 9481 if (map_uses_real_name(map)) 9482 return map->real_name; 9483 9484 return map->name; 9485 } 9486 9487 enum bpf_map_type bpf_map__type(const struct bpf_map *map) 9488 { 9489 return map->def.type; 9490 } 9491 9492 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type) 9493 { 9494 if (map->fd >= 0) 9495 return libbpf_err(-EBUSY); 9496 map->def.type = type; 9497 return 0; 9498 } 9499 9500 __u32 bpf_map__map_flags(const struct bpf_map *map) 9501 { 9502 return map->def.map_flags; 9503 } 9504 9505 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags) 9506 { 9507 if (map->fd >= 0) 9508 return libbpf_err(-EBUSY); 9509 map->def.map_flags = flags; 9510 return 0; 9511 } 9512 9513 __u64 bpf_map__map_extra(const struct bpf_map *map) 9514 { 9515 return map->map_extra; 9516 } 9517 9518 int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra) 9519 { 9520 if (map->fd >= 0) 9521 return libbpf_err(-EBUSY); 9522 map->map_extra = map_extra; 9523 return 0; 9524 } 9525 9526 __u32 bpf_map__numa_node(const struct bpf_map *map) 9527 { 9528 return map->numa_node; 9529 } 9530 9531 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node) 9532 { 9533 if (map->fd >= 0) 9534 return libbpf_err(-EBUSY); 9535 map->numa_node = numa_node; 9536 return 0; 9537 } 9538 9539 __u32 bpf_map__key_size(const struct bpf_map *map) 9540 { 9541 return map->def.key_size; 9542 } 9543 9544 int bpf_map__set_key_size(struct bpf_map *map, __u32 size) 9545 { 9546 if (map->fd >= 0) 9547 return libbpf_err(-EBUSY); 9548 map->def.key_size = size; 9549 return 0; 9550 } 9551 9552 __u32 bpf_map__value_size(const struct bpf_map *map) 9553 { 9554 return map->def.value_size; 9555 } 9556 9557 static int map_btf_datasec_resize(struct bpf_map *map, __u32 size) 9558 { 9559 struct btf *btf; 9560 struct btf_type *datasec_type, *var_type; 9561 struct btf_var_secinfo *var; 9562 const struct btf_type *array_type; 9563 const struct btf_array *array; 9564 int vlen, element_sz, new_array_id; 9565 __u32 nr_elements; 9566 9567 /* check btf existence */ 9568 btf = bpf_object__btf(map->obj); 9569 if (!btf) 9570 return -ENOENT; 9571 9572 /* verify map is datasec */ 9573 datasec_type = btf_type_by_id(btf, bpf_map__btf_value_type_id(map)); 9574 if (!btf_is_datasec(datasec_type)) { 9575 pr_warn("map '%s': cannot be resized, map value type is not a datasec\n", 9576 bpf_map__name(map)); 9577 return -EINVAL; 9578 } 9579 9580 /* verify datasec has at least one var */ 9581 vlen = btf_vlen(datasec_type); 9582 if (vlen == 0) { 9583 pr_warn("map '%s': cannot be resized, map value datasec is empty\n", 9584 bpf_map__name(map)); 9585 return -EINVAL; 9586 } 9587 9588 /* verify last var in the datasec is an array */ 9589 var = &btf_var_secinfos(datasec_type)[vlen - 1]; 9590 var_type = btf_type_by_id(btf, var->type); 9591 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL); 9592 if (!btf_is_array(array_type)) { 9593 pr_warn("map '%s': cannot be resized, last var must be an array\n", 9594 bpf_map__name(map)); 9595 return -EINVAL; 9596 } 9597 9598 /* verify request size aligns with array */ 9599 array = btf_array(array_type); 9600 element_sz = btf__resolve_size(btf, array->type); 9601 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) { 9602 pr_warn("map '%s': cannot be resized, element size (%d) doesn't align with new total size (%u)\n", 9603 bpf_map__name(map), element_sz, size); 9604 return -EINVAL; 9605 } 9606 9607 /* create a new array based on the existing array, but with new length */ 9608 nr_elements = (size - var->offset) / element_sz; 9609 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements); 9610 if (new_array_id < 0) 9611 return new_array_id; 9612 9613 /* adding a new btf type invalidates existing pointers to btf objects, 9614 * so refresh pointers before proceeding 9615 */ 9616 datasec_type = btf_type_by_id(btf, map->btf_value_type_id); 9617 var = &btf_var_secinfos(datasec_type)[vlen - 1]; 9618 var_type = btf_type_by_id(btf, var->type); 9619 9620 /* finally update btf info */ 9621 datasec_type->size = size; 9622 var->size = size - var->offset; 9623 var_type->type = new_array_id; 9624 9625 return 0; 9626 } 9627 9628 int bpf_map__set_value_size(struct bpf_map *map, __u32 size) 9629 { 9630 if (map->fd >= 0) 9631 return libbpf_err(-EBUSY); 9632 9633 if (map->mmaped) { 9634 int err; 9635 size_t mmap_old_sz, mmap_new_sz; 9636 9637 mmap_old_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 9638 mmap_new_sz = bpf_map_mmap_sz(size, map->def.max_entries); 9639 err = bpf_map_mmap_resize(map, mmap_old_sz, mmap_new_sz); 9640 if (err) { 9641 pr_warn("map '%s': failed to resize memory-mapped region: %d\n", 9642 bpf_map__name(map), err); 9643 return err; 9644 } 9645 err = map_btf_datasec_resize(map, size); 9646 if (err && err != -ENOENT) { 9647 pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %d\n", 9648 bpf_map__name(map), err); 9649 map->btf_value_type_id = 0; 9650 map->btf_key_type_id = 0; 9651 } 9652 } 9653 9654 map->def.value_size = size; 9655 return 0; 9656 } 9657 9658 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 9659 { 9660 return map ? map->btf_key_type_id : 0; 9661 } 9662 9663 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 9664 { 9665 return map ? map->btf_value_type_id : 0; 9666 } 9667 9668 int bpf_map__set_initial_value(struct bpf_map *map, 9669 const void *data, size_t size) 9670 { 9671 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG || 9672 size != map->def.value_size || map->fd >= 0) 9673 return libbpf_err(-EINVAL); 9674 9675 memcpy(map->mmaped, data, size); 9676 return 0; 9677 } 9678 9679 void *bpf_map__initial_value(struct bpf_map *map, size_t *psize) 9680 { 9681 if (!map->mmaped) 9682 return NULL; 9683 *psize = map->def.value_size; 9684 return map->mmaped; 9685 } 9686 9687 bool bpf_map__is_internal(const struct bpf_map *map) 9688 { 9689 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 9690 } 9691 9692 __u32 bpf_map__ifindex(const struct bpf_map *map) 9693 { 9694 return map->map_ifindex; 9695 } 9696 9697 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 9698 { 9699 if (map->fd >= 0) 9700 return libbpf_err(-EBUSY); 9701 map->map_ifindex = ifindex; 9702 return 0; 9703 } 9704 9705 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 9706 { 9707 if (!bpf_map_type__is_map_in_map(map->def.type)) { 9708 pr_warn("error: unsupported map type\n"); 9709 return libbpf_err(-EINVAL); 9710 } 9711 if (map->inner_map_fd != -1) { 9712 pr_warn("error: inner_map_fd already specified\n"); 9713 return libbpf_err(-EINVAL); 9714 } 9715 if (map->inner_map) { 9716 bpf_map__destroy(map->inner_map); 9717 zfree(&map->inner_map); 9718 } 9719 map->inner_map_fd = fd; 9720 return 0; 9721 } 9722 9723 static struct bpf_map * 9724 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 9725 { 9726 ssize_t idx; 9727 struct bpf_map *s, *e; 9728 9729 if (!obj || !obj->maps) 9730 return errno = EINVAL, NULL; 9731 9732 s = obj->maps; 9733 e = obj->maps + obj->nr_maps; 9734 9735 if ((m < s) || (m >= e)) { 9736 pr_warn("error in %s: map handler doesn't belong to object\n", 9737 __func__); 9738 return errno = EINVAL, NULL; 9739 } 9740 9741 idx = (m - obj->maps) + i; 9742 if (idx >= obj->nr_maps || idx < 0) 9743 return NULL; 9744 return &obj->maps[idx]; 9745 } 9746 9747 struct bpf_map * 9748 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev) 9749 { 9750 if (prev == NULL) 9751 return obj->maps; 9752 9753 return __bpf_map__iter(prev, obj, 1); 9754 } 9755 9756 struct bpf_map * 9757 bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next) 9758 { 9759 if (next == NULL) { 9760 if (!obj->nr_maps) 9761 return NULL; 9762 return obj->maps + obj->nr_maps - 1; 9763 } 9764 9765 return __bpf_map__iter(next, obj, -1); 9766 } 9767 9768 struct bpf_map * 9769 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 9770 { 9771 struct bpf_map *pos; 9772 9773 bpf_object__for_each_map(pos, obj) { 9774 /* if it's a special internal map name (which always starts 9775 * with dot) then check if that special name matches the 9776 * real map name (ELF section name) 9777 */ 9778 if (name[0] == '.') { 9779 if (pos->real_name && strcmp(pos->real_name, name) == 0) 9780 return pos; 9781 continue; 9782 } 9783 /* otherwise map name has to be an exact match */ 9784 if (map_uses_real_name(pos)) { 9785 if (strcmp(pos->real_name, name) == 0) 9786 return pos; 9787 continue; 9788 } 9789 if (strcmp(pos->name, name) == 0) 9790 return pos; 9791 } 9792 return errno = ENOENT, NULL; 9793 } 9794 9795 int 9796 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 9797 { 9798 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 9799 } 9800 9801 static int validate_map_op(const struct bpf_map *map, size_t key_sz, 9802 size_t value_sz, bool check_value_sz) 9803 { 9804 if (map->fd <= 0) 9805 return -ENOENT; 9806 9807 if (map->def.key_size != key_sz) { 9808 pr_warn("map '%s': unexpected key size %zu provided, expected %u\n", 9809 map->name, key_sz, map->def.key_size); 9810 return -EINVAL; 9811 } 9812 9813 if (!check_value_sz) 9814 return 0; 9815 9816 switch (map->def.type) { 9817 case BPF_MAP_TYPE_PERCPU_ARRAY: 9818 case BPF_MAP_TYPE_PERCPU_HASH: 9819 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 9820 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: { 9821 int num_cpu = libbpf_num_possible_cpus(); 9822 size_t elem_sz = roundup(map->def.value_size, 8); 9823 9824 if (value_sz != num_cpu * elem_sz) { 9825 pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n", 9826 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); 9827 return -EINVAL; 9828 } 9829 break; 9830 } 9831 default: 9832 if (map->def.value_size != value_sz) { 9833 pr_warn("map '%s': unexpected value size %zu provided, expected %u\n", 9834 map->name, value_sz, map->def.value_size); 9835 return -EINVAL; 9836 } 9837 break; 9838 } 9839 return 0; 9840 } 9841 9842 int bpf_map__lookup_elem(const struct bpf_map *map, 9843 const void *key, size_t key_sz, 9844 void *value, size_t value_sz, __u64 flags) 9845 { 9846 int err; 9847 9848 err = validate_map_op(map, key_sz, value_sz, true); 9849 if (err) 9850 return libbpf_err(err); 9851 9852 return bpf_map_lookup_elem_flags(map->fd, key, value, flags); 9853 } 9854 9855 int bpf_map__update_elem(const struct bpf_map *map, 9856 const void *key, size_t key_sz, 9857 const void *value, size_t value_sz, __u64 flags) 9858 { 9859 int err; 9860 9861 err = validate_map_op(map, key_sz, value_sz, true); 9862 if (err) 9863 return libbpf_err(err); 9864 9865 return bpf_map_update_elem(map->fd, key, value, flags); 9866 } 9867 9868 int bpf_map__delete_elem(const struct bpf_map *map, 9869 const void *key, size_t key_sz, __u64 flags) 9870 { 9871 int err; 9872 9873 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */); 9874 if (err) 9875 return libbpf_err(err); 9876 9877 return bpf_map_delete_elem_flags(map->fd, key, flags); 9878 } 9879 9880 int bpf_map__lookup_and_delete_elem(const struct bpf_map *map, 9881 const void *key, size_t key_sz, 9882 void *value, size_t value_sz, __u64 flags) 9883 { 9884 int err; 9885 9886 err = validate_map_op(map, key_sz, value_sz, true); 9887 if (err) 9888 return libbpf_err(err); 9889 9890 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags); 9891 } 9892 9893 int bpf_map__get_next_key(const struct bpf_map *map, 9894 const void *cur_key, void *next_key, size_t key_sz) 9895 { 9896 int err; 9897 9898 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */); 9899 if (err) 9900 return libbpf_err(err); 9901 9902 return bpf_map_get_next_key(map->fd, cur_key, next_key); 9903 } 9904 9905 long libbpf_get_error(const void *ptr) 9906 { 9907 if (!IS_ERR_OR_NULL(ptr)) 9908 return 0; 9909 9910 if (IS_ERR(ptr)) 9911 errno = -PTR_ERR(ptr); 9912 9913 /* If ptr == NULL, then errno should be already set by the failing 9914 * API, because libbpf never returns NULL on success and it now always 9915 * sets errno on error. So no extra errno handling for ptr == NULL 9916 * case. 9917 */ 9918 return -errno; 9919 } 9920 9921 /* Replace link's underlying BPF program with the new one */ 9922 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) 9923 { 9924 int ret; 9925 9926 ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL); 9927 return libbpf_err_errno(ret); 9928 } 9929 9930 /* Release "ownership" of underlying BPF resource (typically, BPF program 9931 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected 9932 * link, when destructed through bpf_link__destroy() call won't attempt to 9933 * detach/unregisted that BPF resource. This is useful in situations where, 9934 * say, attached BPF program has to outlive userspace program that attached it 9935 * in the system. Depending on type of BPF program, though, there might be 9936 * additional steps (like pinning BPF program in BPF FS) necessary to ensure 9937 * exit of userspace program doesn't trigger automatic detachment and clean up 9938 * inside the kernel. 9939 */ 9940 void bpf_link__disconnect(struct bpf_link *link) 9941 { 9942 link->disconnected = true; 9943 } 9944 9945 int bpf_link__destroy(struct bpf_link *link) 9946 { 9947 int err = 0; 9948 9949 if (IS_ERR_OR_NULL(link)) 9950 return 0; 9951 9952 if (!link->disconnected && link->detach) 9953 err = link->detach(link); 9954 if (link->pin_path) 9955 free(link->pin_path); 9956 if (link->dealloc) 9957 link->dealloc(link); 9958 else 9959 free(link); 9960 9961 return libbpf_err(err); 9962 } 9963 9964 int bpf_link__fd(const struct bpf_link *link) 9965 { 9966 return link->fd; 9967 } 9968 9969 const char *bpf_link__pin_path(const struct bpf_link *link) 9970 { 9971 return link->pin_path; 9972 } 9973 9974 static int bpf_link__detach_fd(struct bpf_link *link) 9975 { 9976 return libbpf_err_errno(close(link->fd)); 9977 } 9978 9979 struct bpf_link *bpf_link__open(const char *path) 9980 { 9981 struct bpf_link *link; 9982 int fd; 9983 9984 fd = bpf_obj_get(path); 9985 if (fd < 0) { 9986 fd = -errno; 9987 pr_warn("failed to open link at %s: %d\n", path, fd); 9988 return libbpf_err_ptr(fd); 9989 } 9990 9991 link = calloc(1, sizeof(*link)); 9992 if (!link) { 9993 close(fd); 9994 return libbpf_err_ptr(-ENOMEM); 9995 } 9996 link->detach = &bpf_link__detach_fd; 9997 link->fd = fd; 9998 9999 link->pin_path = strdup(path); 10000 if (!link->pin_path) { 10001 bpf_link__destroy(link); 10002 return libbpf_err_ptr(-ENOMEM); 10003 } 10004 10005 return link; 10006 } 10007 10008 int bpf_link__detach(struct bpf_link *link) 10009 { 10010 return bpf_link_detach(link->fd) ? -errno : 0; 10011 } 10012 10013 int bpf_link__pin(struct bpf_link *link, const char *path) 10014 { 10015 int err; 10016 10017 if (link->pin_path) 10018 return libbpf_err(-EBUSY); 10019 err = make_parent_dir(path); 10020 if (err) 10021 return libbpf_err(err); 10022 err = check_path(path); 10023 if (err) 10024 return libbpf_err(err); 10025 10026 link->pin_path = strdup(path); 10027 if (!link->pin_path) 10028 return libbpf_err(-ENOMEM); 10029 10030 if (bpf_obj_pin(link->fd, link->pin_path)) { 10031 err = -errno; 10032 zfree(&link->pin_path); 10033 return libbpf_err(err); 10034 } 10035 10036 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); 10037 return 0; 10038 } 10039 10040 int bpf_link__unpin(struct bpf_link *link) 10041 { 10042 int err; 10043 10044 if (!link->pin_path) 10045 return libbpf_err(-EINVAL); 10046 10047 err = unlink(link->pin_path); 10048 if (err != 0) 10049 return -errno; 10050 10051 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); 10052 zfree(&link->pin_path); 10053 return 0; 10054 } 10055 10056 struct bpf_link_perf { 10057 struct bpf_link link; 10058 int perf_event_fd; 10059 /* legacy kprobe support: keep track of probe identifier and type */ 10060 char *legacy_probe_name; 10061 bool legacy_is_kprobe; 10062 bool legacy_is_retprobe; 10063 }; 10064 10065 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe); 10066 static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe); 10067 10068 static int bpf_link_perf_detach(struct bpf_link *link) 10069 { 10070 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 10071 int err = 0; 10072 10073 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0) 10074 err = -errno; 10075 10076 if (perf_link->perf_event_fd != link->fd) 10077 close(perf_link->perf_event_fd); 10078 close(link->fd); 10079 10080 /* legacy uprobe/kprobe needs to be removed after perf event fd closure */ 10081 if (perf_link->legacy_probe_name) { 10082 if (perf_link->legacy_is_kprobe) { 10083 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name, 10084 perf_link->legacy_is_retprobe); 10085 } else { 10086 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name, 10087 perf_link->legacy_is_retprobe); 10088 } 10089 } 10090 10091 return err; 10092 } 10093 10094 static void bpf_link_perf_dealloc(struct bpf_link *link) 10095 { 10096 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 10097 10098 free(perf_link->legacy_probe_name); 10099 free(perf_link); 10100 } 10101 10102 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd, 10103 const struct bpf_perf_event_opts *opts) 10104 { 10105 char errmsg[STRERR_BUFSIZE]; 10106 struct bpf_link_perf *link; 10107 int prog_fd, link_fd = -1, err; 10108 bool force_ioctl_attach; 10109 10110 if (!OPTS_VALID(opts, bpf_perf_event_opts)) 10111 return libbpf_err_ptr(-EINVAL); 10112 10113 if (pfd < 0) { 10114 pr_warn("prog '%s': invalid perf event FD %d\n", 10115 prog->name, pfd); 10116 return libbpf_err_ptr(-EINVAL); 10117 } 10118 prog_fd = bpf_program__fd(prog); 10119 if (prog_fd < 0) { 10120 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n", 10121 prog->name); 10122 return libbpf_err_ptr(-EINVAL); 10123 } 10124 10125 link = calloc(1, sizeof(*link)); 10126 if (!link) 10127 return libbpf_err_ptr(-ENOMEM); 10128 link->link.detach = &bpf_link_perf_detach; 10129 link->link.dealloc = &bpf_link_perf_dealloc; 10130 link->perf_event_fd = pfd; 10131 10132 force_ioctl_attach = OPTS_GET(opts, force_ioctl_attach, false); 10133 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) { 10134 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts, 10135 .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0)); 10136 10137 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts); 10138 if (link_fd < 0) { 10139 err = -errno; 10140 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n", 10141 prog->name, pfd, 10142 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10143 goto err_out; 10144 } 10145 link->link.fd = link_fd; 10146 } else { 10147 if (OPTS_GET(opts, bpf_cookie, 0)) { 10148 pr_warn("prog '%s': user context value is not supported\n", prog->name); 10149 err = -EOPNOTSUPP; 10150 goto err_out; 10151 } 10152 10153 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 10154 err = -errno; 10155 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n", 10156 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10157 if (err == -EPROTO) 10158 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n", 10159 prog->name, pfd); 10160 goto err_out; 10161 } 10162 link->link.fd = pfd; 10163 } 10164 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 10165 err = -errno; 10166 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n", 10167 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10168 goto err_out; 10169 } 10170 10171 return &link->link; 10172 err_out: 10173 if (link_fd >= 0) 10174 close(link_fd); 10175 free(link); 10176 return libbpf_err_ptr(err); 10177 } 10178 10179 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd) 10180 { 10181 return bpf_program__attach_perf_event_opts(prog, pfd, NULL); 10182 } 10183 10184 /* 10185 * this function is expected to parse integer in the range of [0, 2^31-1] from 10186 * given file using scanf format string fmt. If actual parsed value is 10187 * negative, the result might be indistinguishable from error 10188 */ 10189 static int parse_uint_from_file(const char *file, const char *fmt) 10190 { 10191 char buf[STRERR_BUFSIZE]; 10192 int err, ret; 10193 FILE *f; 10194 10195 f = fopen(file, "re"); 10196 if (!f) { 10197 err = -errno; 10198 pr_debug("failed to open '%s': %s\n", file, 10199 libbpf_strerror_r(err, buf, sizeof(buf))); 10200 return err; 10201 } 10202 err = fscanf(f, fmt, &ret); 10203 if (err != 1) { 10204 err = err == EOF ? -EIO : -errno; 10205 pr_debug("failed to parse '%s': %s\n", file, 10206 libbpf_strerror_r(err, buf, sizeof(buf))); 10207 fclose(f); 10208 return err; 10209 } 10210 fclose(f); 10211 return ret; 10212 } 10213 10214 static int determine_kprobe_perf_type(void) 10215 { 10216 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 10217 10218 return parse_uint_from_file(file, "%d\n"); 10219 } 10220 10221 static int determine_uprobe_perf_type(void) 10222 { 10223 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 10224 10225 return parse_uint_from_file(file, "%d\n"); 10226 } 10227 10228 static int determine_kprobe_retprobe_bit(void) 10229 { 10230 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 10231 10232 return parse_uint_from_file(file, "config:%d\n"); 10233 } 10234 10235 static int determine_uprobe_retprobe_bit(void) 10236 { 10237 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 10238 10239 return parse_uint_from_file(file, "config:%d\n"); 10240 } 10241 10242 #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32 10243 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32 10244 10245 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 10246 uint64_t offset, int pid, size_t ref_ctr_off) 10247 { 10248 const size_t attr_sz = sizeof(struct perf_event_attr); 10249 struct perf_event_attr attr; 10250 char errmsg[STRERR_BUFSIZE]; 10251 int type, pfd; 10252 10253 if ((__u64)ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS)) 10254 return -EINVAL; 10255 10256 memset(&attr, 0, attr_sz); 10257 10258 type = uprobe ? determine_uprobe_perf_type() 10259 : determine_kprobe_perf_type(); 10260 if (type < 0) { 10261 pr_warn("failed to determine %s perf type: %s\n", 10262 uprobe ? "uprobe" : "kprobe", 10263 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 10264 return type; 10265 } 10266 if (retprobe) { 10267 int bit = uprobe ? determine_uprobe_retprobe_bit() 10268 : determine_kprobe_retprobe_bit(); 10269 10270 if (bit < 0) { 10271 pr_warn("failed to determine %s retprobe bit: %s\n", 10272 uprobe ? "uprobe" : "kprobe", 10273 libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); 10274 return bit; 10275 } 10276 attr.config |= 1 << bit; 10277 } 10278 attr.size = attr_sz; 10279 attr.type = type; 10280 attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT; 10281 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 10282 attr.config2 = offset; /* kprobe_addr or probe_offset */ 10283 10284 /* pid filter is meaningful only for uprobes */ 10285 pfd = syscall(__NR_perf_event_open, &attr, 10286 pid < 0 ? -1 : pid /* pid */, 10287 pid == -1 ? 0 : -1 /* cpu */, 10288 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 10289 return pfd >= 0 ? pfd : -errno; 10290 } 10291 10292 static int append_to_file(const char *file, const char *fmt, ...) 10293 { 10294 int fd, n, err = 0; 10295 va_list ap; 10296 char buf[1024]; 10297 10298 va_start(ap, fmt); 10299 n = vsnprintf(buf, sizeof(buf), fmt, ap); 10300 va_end(ap); 10301 10302 if (n < 0 || n >= sizeof(buf)) 10303 return -EINVAL; 10304 10305 fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0); 10306 if (fd < 0) 10307 return -errno; 10308 10309 if (write(fd, buf, n) < 0) 10310 err = -errno; 10311 10312 close(fd); 10313 return err; 10314 } 10315 10316 #define DEBUGFS "/sys/kernel/debug/tracing" 10317 #define TRACEFS "/sys/kernel/tracing" 10318 10319 static bool use_debugfs(void) 10320 { 10321 static int has_debugfs = -1; 10322 10323 if (has_debugfs < 0) 10324 has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0; 10325 10326 return has_debugfs == 1; 10327 } 10328 10329 static const char *tracefs_path(void) 10330 { 10331 return use_debugfs() ? DEBUGFS : TRACEFS; 10332 } 10333 10334 static const char *tracefs_kprobe_events(void) 10335 { 10336 return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events"; 10337 } 10338 10339 static const char *tracefs_uprobe_events(void) 10340 { 10341 return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events"; 10342 } 10343 10344 static const char *tracefs_available_filter_functions(void) 10345 { 10346 return use_debugfs() ? DEBUGFS"/available_filter_functions" 10347 : TRACEFS"/available_filter_functions"; 10348 } 10349 10350 static const char *tracefs_available_filter_functions_addrs(void) 10351 { 10352 return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" 10353 : TRACEFS"/available_filter_functions_addrs"; 10354 } 10355 10356 static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz, 10357 const char *kfunc_name, size_t offset) 10358 { 10359 static int index = 0; 10360 int i; 10361 10362 snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset, 10363 __sync_fetch_and_add(&index, 1)); 10364 10365 /* sanitize binary_path in the probe name */ 10366 for (i = 0; buf[i]; i++) { 10367 if (!isalnum(buf[i])) 10368 buf[i] = '_'; 10369 } 10370 } 10371 10372 static int add_kprobe_event_legacy(const char *probe_name, bool retprobe, 10373 const char *kfunc_name, size_t offset) 10374 { 10375 return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx", 10376 retprobe ? 'r' : 'p', 10377 retprobe ? "kretprobes" : "kprobes", 10378 probe_name, kfunc_name, offset); 10379 } 10380 10381 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe) 10382 { 10383 return append_to_file(tracefs_kprobe_events(), "-:%s/%s", 10384 retprobe ? "kretprobes" : "kprobes", probe_name); 10385 } 10386 10387 static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe) 10388 { 10389 char file[256]; 10390 10391 snprintf(file, sizeof(file), "%s/events/%s/%s/id", 10392 tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name); 10393 10394 return parse_uint_from_file(file, "%d\n"); 10395 } 10396 10397 static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, 10398 const char *kfunc_name, size_t offset, int pid) 10399 { 10400 const size_t attr_sz = sizeof(struct perf_event_attr); 10401 struct perf_event_attr attr; 10402 char errmsg[STRERR_BUFSIZE]; 10403 int type, pfd, err; 10404 10405 err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset); 10406 if (err < 0) { 10407 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n", 10408 kfunc_name, offset, 10409 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10410 return err; 10411 } 10412 type = determine_kprobe_perf_type_legacy(probe_name, retprobe); 10413 if (type < 0) { 10414 err = type; 10415 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n", 10416 kfunc_name, offset, 10417 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10418 goto err_clean_legacy; 10419 } 10420 10421 memset(&attr, 0, attr_sz); 10422 attr.size = attr_sz; 10423 attr.config = type; 10424 attr.type = PERF_TYPE_TRACEPOINT; 10425 10426 pfd = syscall(__NR_perf_event_open, &attr, 10427 pid < 0 ? -1 : pid, /* pid */ 10428 pid == -1 ? 0 : -1, /* cpu */ 10429 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 10430 if (pfd < 0) { 10431 err = -errno; 10432 pr_warn("legacy kprobe perf_event_open() failed: %s\n", 10433 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10434 goto err_clean_legacy; 10435 } 10436 return pfd; 10437 10438 err_clean_legacy: 10439 /* Clear the newly added legacy kprobe_event */ 10440 remove_kprobe_event_legacy(probe_name, retprobe); 10441 return err; 10442 } 10443 10444 static const char *arch_specific_syscall_pfx(void) 10445 { 10446 #if defined(__x86_64__) 10447 return "x64"; 10448 #elif defined(__i386__) 10449 return "ia32"; 10450 #elif defined(__s390x__) 10451 return "s390x"; 10452 #elif defined(__s390__) 10453 return "s390"; 10454 #elif defined(__arm__) 10455 return "arm"; 10456 #elif defined(__aarch64__) 10457 return "arm64"; 10458 #elif defined(__mips__) 10459 return "mips"; 10460 #elif defined(__riscv) 10461 return "riscv"; 10462 #elif defined(__powerpc__) 10463 return "powerpc"; 10464 #elif defined(__powerpc64__) 10465 return "powerpc64"; 10466 #else 10467 return NULL; 10468 #endif 10469 } 10470 10471 static int probe_kern_syscall_wrapper(void) 10472 { 10473 char syscall_name[64]; 10474 const char *ksys_pfx; 10475 10476 ksys_pfx = arch_specific_syscall_pfx(); 10477 if (!ksys_pfx) 10478 return 0; 10479 10480 snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx); 10481 10482 if (determine_kprobe_perf_type() >= 0) { 10483 int pfd; 10484 10485 pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0); 10486 if (pfd >= 0) 10487 close(pfd); 10488 10489 return pfd >= 0 ? 1 : 0; 10490 } else { /* legacy mode */ 10491 char probe_name[128]; 10492 10493 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0); 10494 if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0) 10495 return 0; 10496 10497 (void)remove_kprobe_event_legacy(probe_name, false); 10498 return 1; 10499 } 10500 } 10501 10502 struct bpf_link * 10503 bpf_program__attach_kprobe_opts(const struct bpf_program *prog, 10504 const char *func_name, 10505 const struct bpf_kprobe_opts *opts) 10506 { 10507 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 10508 enum probe_attach_mode attach_mode; 10509 char errmsg[STRERR_BUFSIZE]; 10510 char *legacy_probe = NULL; 10511 struct bpf_link *link; 10512 size_t offset; 10513 bool retprobe, legacy; 10514 int pfd, err; 10515 10516 if (!OPTS_VALID(opts, bpf_kprobe_opts)) 10517 return libbpf_err_ptr(-EINVAL); 10518 10519 attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT); 10520 retprobe = OPTS_GET(opts, retprobe, false); 10521 offset = OPTS_GET(opts, offset, 0); 10522 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 10523 10524 legacy = determine_kprobe_perf_type() < 0; 10525 switch (attach_mode) { 10526 case PROBE_ATTACH_MODE_LEGACY: 10527 legacy = true; 10528 pe_opts.force_ioctl_attach = true; 10529 break; 10530 case PROBE_ATTACH_MODE_PERF: 10531 if (legacy) 10532 return libbpf_err_ptr(-ENOTSUP); 10533 pe_opts.force_ioctl_attach = true; 10534 break; 10535 case PROBE_ATTACH_MODE_LINK: 10536 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) 10537 return libbpf_err_ptr(-ENOTSUP); 10538 break; 10539 case PROBE_ATTACH_MODE_DEFAULT: 10540 break; 10541 default: 10542 return libbpf_err_ptr(-EINVAL); 10543 } 10544 10545 if (!legacy) { 10546 pfd = perf_event_open_probe(false /* uprobe */, retprobe, 10547 func_name, offset, 10548 -1 /* pid */, 0 /* ref_ctr_off */); 10549 } else { 10550 char probe_name[256]; 10551 10552 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), 10553 func_name, offset); 10554 10555 legacy_probe = strdup(probe_name); 10556 if (!legacy_probe) 10557 return libbpf_err_ptr(-ENOMEM); 10558 10559 pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name, 10560 offset, -1 /* pid */); 10561 } 10562 if (pfd < 0) { 10563 err = -errno; 10564 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n", 10565 prog->name, retprobe ? "kretprobe" : "kprobe", 10566 func_name, offset, 10567 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10568 goto err_out; 10569 } 10570 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 10571 err = libbpf_get_error(link); 10572 if (err) { 10573 close(pfd); 10574 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n", 10575 prog->name, retprobe ? "kretprobe" : "kprobe", 10576 func_name, offset, 10577 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10578 goto err_clean_legacy; 10579 } 10580 if (legacy) { 10581 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 10582 10583 perf_link->legacy_probe_name = legacy_probe; 10584 perf_link->legacy_is_kprobe = true; 10585 perf_link->legacy_is_retprobe = retprobe; 10586 } 10587 10588 return link; 10589 10590 err_clean_legacy: 10591 if (legacy) 10592 remove_kprobe_event_legacy(legacy_probe, retprobe); 10593 err_out: 10594 free(legacy_probe); 10595 return libbpf_err_ptr(err); 10596 } 10597 10598 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog, 10599 bool retprobe, 10600 const char *func_name) 10601 { 10602 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts, 10603 .retprobe = retprobe, 10604 ); 10605 10606 return bpf_program__attach_kprobe_opts(prog, func_name, &opts); 10607 } 10608 10609 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog, 10610 const char *syscall_name, 10611 const struct bpf_ksyscall_opts *opts) 10612 { 10613 LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); 10614 char func_name[128]; 10615 10616 if (!OPTS_VALID(opts, bpf_ksyscall_opts)) 10617 return libbpf_err_ptr(-EINVAL); 10618 10619 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { 10620 /* arch_specific_syscall_pfx() should never return NULL here 10621 * because it is guarded by kernel_supports(). However, since 10622 * compiler does not know that we have an explicit conditional 10623 * as well. 10624 */ 10625 snprintf(func_name, sizeof(func_name), "__%s_sys_%s", 10626 arch_specific_syscall_pfx() ? : "", syscall_name); 10627 } else { 10628 snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name); 10629 } 10630 10631 kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false); 10632 kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 10633 10634 return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts); 10635 } 10636 10637 /* Adapted from perf/util/string.c */ 10638 bool glob_match(const char *str, const char *pat) 10639 { 10640 while (*str && *pat && *pat != '*') { 10641 if (*pat == '?') { /* Matches any single character */ 10642 str++; 10643 pat++; 10644 continue; 10645 } 10646 if (*str != *pat) 10647 return false; 10648 str++; 10649 pat++; 10650 } 10651 /* Check wild card */ 10652 if (*pat == '*') { 10653 while (*pat == '*') 10654 pat++; 10655 if (!*pat) /* Tail wild card matches all */ 10656 return true; 10657 while (*str) 10658 if (glob_match(str++, pat)) 10659 return true; 10660 } 10661 return !*str && !*pat; 10662 } 10663 10664 struct kprobe_multi_resolve { 10665 const char *pattern; 10666 unsigned long *addrs; 10667 size_t cap; 10668 size_t cnt; 10669 }; 10670 10671 struct avail_kallsyms_data { 10672 char **syms; 10673 size_t cnt; 10674 struct kprobe_multi_resolve *res; 10675 }; 10676 10677 static int avail_func_cmp(const void *a, const void *b) 10678 { 10679 return strcmp(*(const char **)a, *(const char **)b); 10680 } 10681 10682 static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type, 10683 const char *sym_name, void *ctx) 10684 { 10685 struct avail_kallsyms_data *data = ctx; 10686 struct kprobe_multi_resolve *res = data->res; 10687 int err; 10688 10689 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) 10690 return 0; 10691 10692 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1); 10693 if (err) 10694 return err; 10695 10696 res->addrs[res->cnt++] = (unsigned long)sym_addr; 10697 return 0; 10698 } 10699 10700 static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res) 10701 { 10702 const char *available_functions_file = tracefs_available_filter_functions(); 10703 struct avail_kallsyms_data data; 10704 char sym_name[500]; 10705 FILE *f; 10706 int err = 0, ret, i; 10707 char **syms = NULL; 10708 size_t cap = 0, cnt = 0; 10709 10710 f = fopen(available_functions_file, "re"); 10711 if (!f) { 10712 err = -errno; 10713 pr_warn("failed to open %s: %d\n", available_functions_file, err); 10714 return err; 10715 } 10716 10717 while (true) { 10718 char *name; 10719 10720 ret = fscanf(f, "%499s%*[^\n]\n", sym_name); 10721 if (ret == EOF && feof(f)) 10722 break; 10723 10724 if (ret != 1) { 10725 pr_warn("failed to parse available_filter_functions entry: %d\n", ret); 10726 err = -EINVAL; 10727 goto cleanup; 10728 } 10729 10730 if (!glob_match(sym_name, res->pattern)) 10731 continue; 10732 10733 err = libbpf_ensure_mem((void **)&syms, &cap, sizeof(*syms), cnt + 1); 10734 if (err) 10735 goto cleanup; 10736 10737 name = strdup(sym_name); 10738 if (!name) { 10739 err = -errno; 10740 goto cleanup; 10741 } 10742 10743 syms[cnt++] = name; 10744 } 10745 10746 /* no entries found, bail out */ 10747 if (cnt == 0) { 10748 err = -ENOENT; 10749 goto cleanup; 10750 } 10751 10752 /* sort available functions */ 10753 qsort(syms, cnt, sizeof(*syms), avail_func_cmp); 10754 10755 data.syms = syms; 10756 data.res = res; 10757 data.cnt = cnt; 10758 libbpf_kallsyms_parse(avail_kallsyms_cb, &data); 10759 10760 if (res->cnt == 0) 10761 err = -ENOENT; 10762 10763 cleanup: 10764 for (i = 0; i < cnt; i++) 10765 free((char *)syms[i]); 10766 free(syms); 10767 10768 fclose(f); 10769 return err; 10770 } 10771 10772 static bool has_available_filter_functions_addrs(void) 10773 { 10774 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1; 10775 } 10776 10777 static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res) 10778 { 10779 const char *available_path = tracefs_available_filter_functions_addrs(); 10780 char sym_name[500]; 10781 FILE *f; 10782 int ret, err = 0; 10783 unsigned long long sym_addr; 10784 10785 f = fopen(available_path, "re"); 10786 if (!f) { 10787 err = -errno; 10788 pr_warn("failed to open %s: %d\n", available_path, err); 10789 return err; 10790 } 10791 10792 while (true) { 10793 ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name); 10794 if (ret == EOF && feof(f)) 10795 break; 10796 10797 if (ret != 2) { 10798 pr_warn("failed to parse available_filter_functions_addrs entry: %d\n", 10799 ret); 10800 err = -EINVAL; 10801 goto cleanup; 10802 } 10803 10804 if (!glob_match(sym_name, res->pattern)) 10805 continue; 10806 10807 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, 10808 sizeof(*res->addrs), res->cnt + 1); 10809 if (err) 10810 goto cleanup; 10811 10812 res->addrs[res->cnt++] = (unsigned long)sym_addr; 10813 } 10814 10815 if (res->cnt == 0) 10816 err = -ENOENT; 10817 10818 cleanup: 10819 fclose(f); 10820 return err; 10821 } 10822 10823 struct bpf_link * 10824 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, 10825 const char *pattern, 10826 const struct bpf_kprobe_multi_opts *opts) 10827 { 10828 LIBBPF_OPTS(bpf_link_create_opts, lopts); 10829 struct kprobe_multi_resolve res = { 10830 .pattern = pattern, 10831 }; 10832 struct bpf_link *link = NULL; 10833 char errmsg[STRERR_BUFSIZE]; 10834 const unsigned long *addrs; 10835 int err, link_fd, prog_fd; 10836 const __u64 *cookies; 10837 const char **syms; 10838 bool retprobe; 10839 size_t cnt; 10840 10841 if (!OPTS_VALID(opts, bpf_kprobe_multi_opts)) 10842 return libbpf_err_ptr(-EINVAL); 10843 10844 syms = OPTS_GET(opts, syms, false); 10845 addrs = OPTS_GET(opts, addrs, false); 10846 cnt = OPTS_GET(opts, cnt, false); 10847 cookies = OPTS_GET(opts, cookies, false); 10848 10849 if (!pattern && !addrs && !syms) 10850 return libbpf_err_ptr(-EINVAL); 10851 if (pattern && (addrs || syms || cookies || cnt)) 10852 return libbpf_err_ptr(-EINVAL); 10853 if (!pattern && !cnt) 10854 return libbpf_err_ptr(-EINVAL); 10855 if (addrs && syms) 10856 return libbpf_err_ptr(-EINVAL); 10857 10858 if (pattern) { 10859 if (has_available_filter_functions_addrs()) 10860 err = libbpf_available_kprobes_parse(&res); 10861 else 10862 err = libbpf_available_kallsyms_parse(&res); 10863 if (err) 10864 goto error; 10865 addrs = res.addrs; 10866 cnt = res.cnt; 10867 } 10868 10869 retprobe = OPTS_GET(opts, retprobe, false); 10870 10871 lopts.kprobe_multi.syms = syms; 10872 lopts.kprobe_multi.addrs = addrs; 10873 lopts.kprobe_multi.cookies = cookies; 10874 lopts.kprobe_multi.cnt = cnt; 10875 lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0; 10876 10877 link = calloc(1, sizeof(*link)); 10878 if (!link) { 10879 err = -ENOMEM; 10880 goto error; 10881 } 10882 link->detach = &bpf_link__detach_fd; 10883 10884 prog_fd = bpf_program__fd(prog); 10885 link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, &lopts); 10886 if (link_fd < 0) { 10887 err = -errno; 10888 pr_warn("prog '%s': failed to attach: %s\n", 10889 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10890 goto error; 10891 } 10892 link->fd = link_fd; 10893 free(res.addrs); 10894 return link; 10895 10896 error: 10897 free(link); 10898 free(res.addrs); 10899 return libbpf_err_ptr(err); 10900 } 10901 10902 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) 10903 { 10904 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts); 10905 unsigned long offset = 0; 10906 const char *func_name; 10907 char *func; 10908 int n; 10909 10910 *link = NULL; 10911 10912 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */ 10913 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0) 10914 return 0; 10915 10916 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/"); 10917 if (opts.retprobe) 10918 func_name = prog->sec_name + sizeof("kretprobe/") - 1; 10919 else 10920 func_name = prog->sec_name + sizeof("kprobe/") - 1; 10921 10922 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset); 10923 if (n < 1) { 10924 pr_warn("kprobe name is invalid: %s\n", func_name); 10925 return -EINVAL; 10926 } 10927 if (opts.retprobe && offset != 0) { 10928 free(func); 10929 pr_warn("kretprobes do not support offset specification\n"); 10930 return -EINVAL; 10931 } 10932 10933 opts.offset = offset; 10934 *link = bpf_program__attach_kprobe_opts(prog, func, &opts); 10935 free(func); 10936 return libbpf_get_error(*link); 10937 } 10938 10939 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link) 10940 { 10941 LIBBPF_OPTS(bpf_ksyscall_opts, opts); 10942 const char *syscall_name; 10943 10944 *link = NULL; 10945 10946 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */ 10947 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0) 10948 return 0; 10949 10950 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/"); 10951 if (opts.retprobe) 10952 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1; 10953 else 10954 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1; 10955 10956 *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts); 10957 return *link ? 0 : -errno; 10958 } 10959 10960 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) 10961 { 10962 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); 10963 const char *spec; 10964 char *pattern; 10965 int n; 10966 10967 *link = NULL; 10968 10969 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */ 10970 if (strcmp(prog->sec_name, "kprobe.multi") == 0 || 10971 strcmp(prog->sec_name, "kretprobe.multi") == 0) 10972 return 0; 10973 10974 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/"); 10975 if (opts.retprobe) 10976 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1; 10977 else 10978 spec = prog->sec_name + sizeof("kprobe.multi/") - 1; 10979 10980 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern); 10981 if (n < 1) { 10982 pr_warn("kprobe multi pattern is invalid: %s\n", pattern); 10983 return -EINVAL; 10984 } 10985 10986 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts); 10987 free(pattern); 10988 return libbpf_get_error(*link); 10989 } 10990 10991 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) 10992 { 10993 char *probe_type = NULL, *binary_path = NULL, *func_name = NULL; 10994 LIBBPF_OPTS(bpf_uprobe_multi_opts, opts); 10995 int n, ret = -EINVAL; 10996 10997 *link = NULL; 10998 10999 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%ms", 11000 &probe_type, &binary_path, &func_name); 11001 switch (n) { 11002 case 1: 11003 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ 11004 ret = 0; 11005 break; 11006 case 3: 11007 opts.retprobe = strcmp(probe_type, "uretprobe.multi") == 0; 11008 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts); 11009 ret = libbpf_get_error(*link); 11010 break; 11011 default: 11012 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, 11013 prog->sec_name); 11014 break; 11015 } 11016 free(probe_type); 11017 free(binary_path); 11018 free(func_name); 11019 return ret; 11020 } 11021 11022 static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz, 11023 const char *binary_path, uint64_t offset) 11024 { 11025 int i; 11026 11027 snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset); 11028 11029 /* sanitize binary_path in the probe name */ 11030 for (i = 0; buf[i]; i++) { 11031 if (!isalnum(buf[i])) 11032 buf[i] = '_'; 11033 } 11034 } 11035 11036 static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe, 11037 const char *binary_path, size_t offset) 11038 { 11039 return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx", 11040 retprobe ? 'r' : 'p', 11041 retprobe ? "uretprobes" : "uprobes", 11042 probe_name, binary_path, offset); 11043 } 11044 11045 static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe) 11046 { 11047 return append_to_file(tracefs_uprobe_events(), "-:%s/%s", 11048 retprobe ? "uretprobes" : "uprobes", probe_name); 11049 } 11050 11051 static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe) 11052 { 11053 char file[512]; 11054 11055 snprintf(file, sizeof(file), "%s/events/%s/%s/id", 11056 tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name); 11057 11058 return parse_uint_from_file(file, "%d\n"); 11059 } 11060 11061 static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, 11062 const char *binary_path, size_t offset, int pid) 11063 { 11064 const size_t attr_sz = sizeof(struct perf_event_attr); 11065 struct perf_event_attr attr; 11066 int type, pfd, err; 11067 11068 err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset); 11069 if (err < 0) { 11070 pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n", 11071 binary_path, (size_t)offset, err); 11072 return err; 11073 } 11074 type = determine_uprobe_perf_type_legacy(probe_name, retprobe); 11075 if (type < 0) { 11076 err = type; 11077 pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n", 11078 binary_path, offset, err); 11079 goto err_clean_legacy; 11080 } 11081 11082 memset(&attr, 0, attr_sz); 11083 attr.size = attr_sz; 11084 attr.config = type; 11085 attr.type = PERF_TYPE_TRACEPOINT; 11086 11087 pfd = syscall(__NR_perf_event_open, &attr, 11088 pid < 0 ? -1 : pid, /* pid */ 11089 pid == -1 ? 0 : -1, /* cpu */ 11090 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 11091 if (pfd < 0) { 11092 err = -errno; 11093 pr_warn("legacy uprobe perf_event_open() failed: %d\n", err); 11094 goto err_clean_legacy; 11095 } 11096 return pfd; 11097 11098 err_clean_legacy: 11099 /* Clear the newly added legacy uprobe_event */ 11100 remove_uprobe_event_legacy(probe_name, retprobe); 11101 return err; 11102 } 11103 11104 /* Find offset of function name in archive specified by path. Currently 11105 * supported are .zip files that do not compress their contents, as used on 11106 * Android in the form of APKs, for example. "file_name" is the name of the ELF 11107 * file inside the archive. "func_name" matches symbol name or name@@LIB for 11108 * library functions. 11109 * 11110 * An overview of the APK format specifically provided here: 11111 * https://en.wikipedia.org/w/index.php?title=Apk_(file_format)&oldid=1139099120#Package_contents 11112 */ 11113 static long elf_find_func_offset_from_archive(const char *archive_path, const char *file_name, 11114 const char *func_name) 11115 { 11116 struct zip_archive *archive; 11117 struct zip_entry entry; 11118 long ret; 11119 Elf *elf; 11120 11121 archive = zip_archive_open(archive_path); 11122 if (IS_ERR(archive)) { 11123 ret = PTR_ERR(archive); 11124 pr_warn("zip: failed to open %s: %ld\n", archive_path, ret); 11125 return ret; 11126 } 11127 11128 ret = zip_archive_find_entry(archive, file_name, &entry); 11129 if (ret) { 11130 pr_warn("zip: could not find archive member %s in %s: %ld\n", file_name, 11131 archive_path, ret); 11132 goto out; 11133 } 11134 pr_debug("zip: found entry for %s in %s at 0x%lx\n", file_name, archive_path, 11135 (unsigned long)entry.data_offset); 11136 11137 if (entry.compression) { 11138 pr_warn("zip: entry %s of %s is compressed and cannot be handled\n", file_name, 11139 archive_path); 11140 ret = -LIBBPF_ERRNO__FORMAT; 11141 goto out; 11142 } 11143 11144 elf = elf_memory((void *)entry.data, entry.data_length); 11145 if (!elf) { 11146 pr_warn("elf: could not read elf file %s from %s: %s\n", file_name, archive_path, 11147 elf_errmsg(-1)); 11148 ret = -LIBBPF_ERRNO__LIBELF; 11149 goto out; 11150 } 11151 11152 ret = elf_find_func_offset(elf, file_name, func_name); 11153 if (ret > 0) { 11154 pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n", 11155 func_name, file_name, archive_path, entry.data_offset, ret, 11156 ret + entry.data_offset); 11157 ret += entry.data_offset; 11158 } 11159 elf_end(elf); 11160 11161 out: 11162 zip_archive_close(archive); 11163 return ret; 11164 } 11165 11166 static const char *arch_specific_lib_paths(void) 11167 { 11168 /* 11169 * Based on https://packages.debian.org/sid/libc6. 11170 * 11171 * Assume that the traced program is built for the same architecture 11172 * as libbpf, which should cover the vast majority of cases. 11173 */ 11174 #if defined(__x86_64__) 11175 return "/lib/x86_64-linux-gnu"; 11176 #elif defined(__i386__) 11177 return "/lib/i386-linux-gnu"; 11178 #elif defined(__s390x__) 11179 return "/lib/s390x-linux-gnu"; 11180 #elif defined(__s390__) 11181 return "/lib/s390-linux-gnu"; 11182 #elif defined(__arm__) && defined(__SOFTFP__) 11183 return "/lib/arm-linux-gnueabi"; 11184 #elif defined(__arm__) && !defined(__SOFTFP__) 11185 return "/lib/arm-linux-gnueabihf"; 11186 #elif defined(__aarch64__) 11187 return "/lib/aarch64-linux-gnu"; 11188 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64 11189 return "/lib/mips64el-linux-gnuabi64"; 11190 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32 11191 return "/lib/mipsel-linux-gnu"; 11192 #elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 11193 return "/lib/powerpc64le-linux-gnu"; 11194 #elif defined(__sparc__) && defined(__arch64__) 11195 return "/lib/sparc64-linux-gnu"; 11196 #elif defined(__riscv) && __riscv_xlen == 64 11197 return "/lib/riscv64-linux-gnu"; 11198 #else 11199 return NULL; 11200 #endif 11201 } 11202 11203 /* Get full path to program/shared library. */ 11204 static int resolve_full_path(const char *file, char *result, size_t result_sz) 11205 { 11206 const char *search_paths[3] = {}; 11207 int i, perm; 11208 11209 if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { 11210 search_paths[0] = getenv("LD_LIBRARY_PATH"); 11211 search_paths[1] = "/usr/lib64:/usr/lib"; 11212 search_paths[2] = arch_specific_lib_paths(); 11213 perm = R_OK; 11214 } else { 11215 search_paths[0] = getenv("PATH"); 11216 search_paths[1] = "/usr/bin:/usr/sbin"; 11217 perm = R_OK | X_OK; 11218 } 11219 11220 for (i = 0; i < ARRAY_SIZE(search_paths); i++) { 11221 const char *s; 11222 11223 if (!search_paths[i]) 11224 continue; 11225 for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) { 11226 char *next_path; 11227 int seg_len; 11228 11229 if (s[0] == ':') 11230 s++; 11231 next_path = strchr(s, ':'); 11232 seg_len = next_path ? next_path - s : strlen(s); 11233 if (!seg_len) 11234 continue; 11235 snprintf(result, result_sz, "%.*s/%s", seg_len, s, file); 11236 /* ensure it has required permissions */ 11237 if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0) 11238 continue; 11239 pr_debug("resolved '%s' to '%s'\n", file, result); 11240 return 0; 11241 } 11242 } 11243 return -ENOENT; 11244 } 11245 11246 struct bpf_link * 11247 bpf_program__attach_uprobe_multi(const struct bpf_program *prog, 11248 pid_t pid, 11249 const char *path, 11250 const char *func_pattern, 11251 const struct bpf_uprobe_multi_opts *opts) 11252 { 11253 const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL; 11254 LIBBPF_OPTS(bpf_link_create_opts, lopts); 11255 unsigned long *resolved_offsets = NULL; 11256 int err = 0, link_fd, prog_fd; 11257 struct bpf_link *link = NULL; 11258 char errmsg[STRERR_BUFSIZE]; 11259 char full_path[PATH_MAX]; 11260 const __u64 *cookies; 11261 const char **syms; 11262 size_t cnt; 11263 11264 if (!OPTS_VALID(opts, bpf_uprobe_multi_opts)) 11265 return libbpf_err_ptr(-EINVAL); 11266 11267 syms = OPTS_GET(opts, syms, NULL); 11268 offsets = OPTS_GET(opts, offsets, NULL); 11269 ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL); 11270 cookies = OPTS_GET(opts, cookies, NULL); 11271 cnt = OPTS_GET(opts, cnt, 0); 11272 11273 /* 11274 * User can specify 2 mutually exclusive set of inputs: 11275 * 11276 * 1) use only path/func_pattern/pid arguments 11277 * 11278 * 2) use path/pid with allowed combinations of: 11279 * syms/offsets/ref_ctr_offsets/cookies/cnt 11280 * 11281 * - syms and offsets are mutually exclusive 11282 * - ref_ctr_offsets and cookies are optional 11283 * 11284 * Any other usage results in error. 11285 */ 11286 11287 if (!path) 11288 return libbpf_err_ptr(-EINVAL); 11289 if (!func_pattern && cnt == 0) 11290 return libbpf_err_ptr(-EINVAL); 11291 11292 if (func_pattern) { 11293 if (syms || offsets || ref_ctr_offsets || cookies || cnt) 11294 return libbpf_err_ptr(-EINVAL); 11295 } else { 11296 if (!!syms == !!offsets) 11297 return libbpf_err_ptr(-EINVAL); 11298 } 11299 11300 if (func_pattern) { 11301 if (!strchr(path, '/')) { 11302 err = resolve_full_path(path, full_path, sizeof(full_path)); 11303 if (err) { 11304 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", 11305 prog->name, path, err); 11306 return libbpf_err_ptr(err); 11307 } 11308 path = full_path; 11309 } 11310 11311 err = elf_resolve_pattern_offsets(path, func_pattern, 11312 &resolved_offsets, &cnt); 11313 if (err < 0) 11314 return libbpf_err_ptr(err); 11315 offsets = resolved_offsets; 11316 } else if (syms) { 11317 err = elf_resolve_syms_offsets(path, cnt, syms, &resolved_offsets); 11318 if (err < 0) 11319 return libbpf_err_ptr(err); 11320 offsets = resolved_offsets; 11321 } 11322 11323 lopts.uprobe_multi.path = path; 11324 lopts.uprobe_multi.offsets = offsets; 11325 lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets; 11326 lopts.uprobe_multi.cookies = cookies; 11327 lopts.uprobe_multi.cnt = cnt; 11328 lopts.uprobe_multi.flags = OPTS_GET(opts, retprobe, false) ? BPF_F_UPROBE_MULTI_RETURN : 0; 11329 11330 if (pid == 0) 11331 pid = getpid(); 11332 if (pid > 0) 11333 lopts.uprobe_multi.pid = pid; 11334 11335 link = calloc(1, sizeof(*link)); 11336 if (!link) { 11337 err = -ENOMEM; 11338 goto error; 11339 } 11340 link->detach = &bpf_link__detach_fd; 11341 11342 prog_fd = bpf_program__fd(prog); 11343 link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts); 11344 if (link_fd < 0) { 11345 err = -errno; 11346 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n", 11347 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11348 goto error; 11349 } 11350 link->fd = link_fd; 11351 free(resolved_offsets); 11352 return link; 11353 11354 error: 11355 free(resolved_offsets); 11356 free(link); 11357 return libbpf_err_ptr(err); 11358 } 11359 11360 LIBBPF_API struct bpf_link * 11361 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, 11362 const char *binary_path, size_t func_offset, 11363 const struct bpf_uprobe_opts *opts) 11364 { 11365 const char *archive_path = NULL, *archive_sep = NULL; 11366 char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL; 11367 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 11368 enum probe_attach_mode attach_mode; 11369 char full_path[PATH_MAX]; 11370 struct bpf_link *link; 11371 size_t ref_ctr_off; 11372 int pfd, err; 11373 bool retprobe, legacy; 11374 const char *func_name; 11375 11376 if (!OPTS_VALID(opts, bpf_uprobe_opts)) 11377 return libbpf_err_ptr(-EINVAL); 11378 11379 attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT); 11380 retprobe = OPTS_GET(opts, retprobe, false); 11381 ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0); 11382 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 11383 11384 if (!binary_path) 11385 return libbpf_err_ptr(-EINVAL); 11386 11387 /* Check if "binary_path" refers to an archive. */ 11388 archive_sep = strstr(binary_path, "!/"); 11389 if (archive_sep) { 11390 full_path[0] = '\0'; 11391 libbpf_strlcpy(full_path, binary_path, 11392 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1))); 11393 archive_path = full_path; 11394 binary_path = archive_sep + 2; 11395 } else if (!strchr(binary_path, '/')) { 11396 err = resolve_full_path(binary_path, full_path, sizeof(full_path)); 11397 if (err) { 11398 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", 11399 prog->name, binary_path, err); 11400 return libbpf_err_ptr(err); 11401 } 11402 binary_path = full_path; 11403 } 11404 func_name = OPTS_GET(opts, func_name, NULL); 11405 if (func_name) { 11406 long sym_off; 11407 11408 if (archive_path) { 11409 sym_off = elf_find_func_offset_from_archive(archive_path, binary_path, 11410 func_name); 11411 binary_path = archive_path; 11412 } else { 11413 sym_off = elf_find_func_offset_from_file(binary_path, func_name); 11414 } 11415 if (sym_off < 0) 11416 return libbpf_err_ptr(sym_off); 11417 func_offset += sym_off; 11418 } 11419 11420 legacy = determine_uprobe_perf_type() < 0; 11421 switch (attach_mode) { 11422 case PROBE_ATTACH_MODE_LEGACY: 11423 legacy = true; 11424 pe_opts.force_ioctl_attach = true; 11425 break; 11426 case PROBE_ATTACH_MODE_PERF: 11427 if (legacy) 11428 return libbpf_err_ptr(-ENOTSUP); 11429 pe_opts.force_ioctl_attach = true; 11430 break; 11431 case PROBE_ATTACH_MODE_LINK: 11432 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) 11433 return libbpf_err_ptr(-ENOTSUP); 11434 break; 11435 case PROBE_ATTACH_MODE_DEFAULT: 11436 break; 11437 default: 11438 return libbpf_err_ptr(-EINVAL); 11439 } 11440 11441 if (!legacy) { 11442 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path, 11443 func_offset, pid, ref_ctr_off); 11444 } else { 11445 char probe_name[PATH_MAX + 64]; 11446 11447 if (ref_ctr_off) 11448 return libbpf_err_ptr(-EINVAL); 11449 11450 gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name), 11451 binary_path, func_offset); 11452 11453 legacy_probe = strdup(probe_name); 11454 if (!legacy_probe) 11455 return libbpf_err_ptr(-ENOMEM); 11456 11457 pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe, 11458 binary_path, func_offset, pid); 11459 } 11460 if (pfd < 0) { 11461 err = -errno; 11462 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 11463 prog->name, retprobe ? "uretprobe" : "uprobe", 11464 binary_path, func_offset, 11465 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11466 goto err_out; 11467 } 11468 11469 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 11470 err = libbpf_get_error(link); 11471 if (err) { 11472 close(pfd); 11473 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n", 11474 prog->name, retprobe ? "uretprobe" : "uprobe", 11475 binary_path, func_offset, 11476 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11477 goto err_clean_legacy; 11478 } 11479 if (legacy) { 11480 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 11481 11482 perf_link->legacy_probe_name = legacy_probe; 11483 perf_link->legacy_is_kprobe = false; 11484 perf_link->legacy_is_retprobe = retprobe; 11485 } 11486 return link; 11487 11488 err_clean_legacy: 11489 if (legacy) 11490 remove_uprobe_event_legacy(legacy_probe, retprobe); 11491 err_out: 11492 free(legacy_probe); 11493 return libbpf_err_ptr(err); 11494 } 11495 11496 /* Format of u[ret]probe section definition supporting auto-attach: 11497 * u[ret]probe/binary:function[+offset] 11498 * 11499 * binary can be an absolute/relative path or a filename; the latter is resolved to a 11500 * full binary path via bpf_program__attach_uprobe_opts. 11501 * 11502 * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be 11503 * specified (and auto-attach is not possible) or the above format is specified for 11504 * auto-attach. 11505 */ 11506 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11507 { 11508 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts); 11509 char *probe_type = NULL, *binary_path = NULL, *func_name = NULL; 11510 int n, ret = -EINVAL; 11511 long offset = 0; 11512 11513 *link = NULL; 11514 11515 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[a-zA-Z0-9_.]+%li", 11516 &probe_type, &binary_path, &func_name, &offset); 11517 switch (n) { 11518 case 1: 11519 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ 11520 ret = 0; 11521 break; 11522 case 2: 11523 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n", 11524 prog->name, prog->sec_name); 11525 break; 11526 case 3: 11527 case 4: 11528 opts.retprobe = strcmp(probe_type, "uretprobe") == 0 || 11529 strcmp(probe_type, "uretprobe.s") == 0; 11530 if (opts.retprobe && offset != 0) { 11531 pr_warn("prog '%s': uretprobes do not support offset specification\n", 11532 prog->name); 11533 break; 11534 } 11535 opts.func_name = func_name; 11536 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts); 11537 ret = libbpf_get_error(*link); 11538 break; 11539 default: 11540 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, 11541 prog->sec_name); 11542 break; 11543 } 11544 free(probe_type); 11545 free(binary_path); 11546 free(func_name); 11547 11548 return ret; 11549 } 11550 11551 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog, 11552 bool retprobe, pid_t pid, 11553 const char *binary_path, 11554 size_t func_offset) 11555 { 11556 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe); 11557 11558 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts); 11559 } 11560 11561 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog, 11562 pid_t pid, const char *binary_path, 11563 const char *usdt_provider, const char *usdt_name, 11564 const struct bpf_usdt_opts *opts) 11565 { 11566 char resolved_path[512]; 11567 struct bpf_object *obj = prog->obj; 11568 struct bpf_link *link; 11569 __u64 usdt_cookie; 11570 int err; 11571 11572 if (!OPTS_VALID(opts, bpf_uprobe_opts)) 11573 return libbpf_err_ptr(-EINVAL); 11574 11575 if (bpf_program__fd(prog) < 0) { 11576 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n", 11577 prog->name); 11578 return libbpf_err_ptr(-EINVAL); 11579 } 11580 11581 if (!binary_path) 11582 return libbpf_err_ptr(-EINVAL); 11583 11584 if (!strchr(binary_path, '/')) { 11585 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path)); 11586 if (err) { 11587 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", 11588 prog->name, binary_path, err); 11589 return libbpf_err_ptr(err); 11590 } 11591 binary_path = resolved_path; 11592 } 11593 11594 /* USDT manager is instantiated lazily on first USDT attach. It will 11595 * be destroyed together with BPF object in bpf_object__close(). 11596 */ 11597 if (IS_ERR(obj->usdt_man)) 11598 return libbpf_ptr(obj->usdt_man); 11599 if (!obj->usdt_man) { 11600 obj->usdt_man = usdt_manager_new(obj); 11601 if (IS_ERR(obj->usdt_man)) 11602 return libbpf_ptr(obj->usdt_man); 11603 } 11604 11605 usdt_cookie = OPTS_GET(opts, usdt_cookie, 0); 11606 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path, 11607 usdt_provider, usdt_name, usdt_cookie); 11608 err = libbpf_get_error(link); 11609 if (err) 11610 return libbpf_err_ptr(err); 11611 return link; 11612 } 11613 11614 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11615 { 11616 char *path = NULL, *provider = NULL, *name = NULL; 11617 const char *sec_name; 11618 int n, err; 11619 11620 sec_name = bpf_program__section_name(prog); 11621 if (strcmp(sec_name, "usdt") == 0) { 11622 /* no auto-attach for just SEC("usdt") */ 11623 *link = NULL; 11624 return 0; 11625 } 11626 11627 n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name); 11628 if (n != 3) { 11629 pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n", 11630 sec_name); 11631 err = -EINVAL; 11632 } else { 11633 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path, 11634 provider, name, NULL); 11635 err = libbpf_get_error(*link); 11636 } 11637 free(path); 11638 free(provider); 11639 free(name); 11640 return err; 11641 } 11642 11643 static int determine_tracepoint_id(const char *tp_category, 11644 const char *tp_name) 11645 { 11646 char file[PATH_MAX]; 11647 int ret; 11648 11649 ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id", 11650 tracefs_path(), tp_category, tp_name); 11651 if (ret < 0) 11652 return -errno; 11653 if (ret >= sizeof(file)) { 11654 pr_debug("tracepoint %s/%s path is too long\n", 11655 tp_category, tp_name); 11656 return -E2BIG; 11657 } 11658 return parse_uint_from_file(file, "%d\n"); 11659 } 11660 11661 static int perf_event_open_tracepoint(const char *tp_category, 11662 const char *tp_name) 11663 { 11664 const size_t attr_sz = sizeof(struct perf_event_attr); 11665 struct perf_event_attr attr; 11666 char errmsg[STRERR_BUFSIZE]; 11667 int tp_id, pfd, err; 11668 11669 tp_id = determine_tracepoint_id(tp_category, tp_name); 11670 if (tp_id < 0) { 11671 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 11672 tp_category, tp_name, 11673 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 11674 return tp_id; 11675 } 11676 11677 memset(&attr, 0, attr_sz); 11678 attr.type = PERF_TYPE_TRACEPOINT; 11679 attr.size = attr_sz; 11680 attr.config = tp_id; 11681 11682 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 11683 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 11684 if (pfd < 0) { 11685 err = -errno; 11686 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", 11687 tp_category, tp_name, 11688 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11689 return err; 11690 } 11691 return pfd; 11692 } 11693 11694 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, 11695 const char *tp_category, 11696 const char *tp_name, 11697 const struct bpf_tracepoint_opts *opts) 11698 { 11699 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 11700 char errmsg[STRERR_BUFSIZE]; 11701 struct bpf_link *link; 11702 int pfd, err; 11703 11704 if (!OPTS_VALID(opts, bpf_tracepoint_opts)) 11705 return libbpf_err_ptr(-EINVAL); 11706 11707 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 11708 11709 pfd = perf_event_open_tracepoint(tp_category, tp_name); 11710 if (pfd < 0) { 11711 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 11712 prog->name, tp_category, tp_name, 11713 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 11714 return libbpf_err_ptr(pfd); 11715 } 11716 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 11717 err = libbpf_get_error(link); 11718 if (err) { 11719 close(pfd); 11720 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n", 11721 prog->name, tp_category, tp_name, 11722 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11723 return libbpf_err_ptr(err); 11724 } 11725 return link; 11726 } 11727 11728 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog, 11729 const char *tp_category, 11730 const char *tp_name) 11731 { 11732 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL); 11733 } 11734 11735 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11736 { 11737 char *sec_name, *tp_cat, *tp_name; 11738 11739 *link = NULL; 11740 11741 /* no auto-attach for SEC("tp") or SEC("tracepoint") */ 11742 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0) 11743 return 0; 11744 11745 sec_name = strdup(prog->sec_name); 11746 if (!sec_name) 11747 return -ENOMEM; 11748 11749 /* extract "tp/<category>/<name>" or "tracepoint/<category>/<name>" */ 11750 if (str_has_pfx(prog->sec_name, "tp/")) 11751 tp_cat = sec_name + sizeof("tp/") - 1; 11752 else 11753 tp_cat = sec_name + sizeof("tracepoint/") - 1; 11754 tp_name = strchr(tp_cat, '/'); 11755 if (!tp_name) { 11756 free(sec_name); 11757 return -EINVAL; 11758 } 11759 *tp_name = '\0'; 11760 tp_name++; 11761 11762 *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); 11763 free(sec_name); 11764 return libbpf_get_error(*link); 11765 } 11766 11767 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog, 11768 const char *tp_name) 11769 { 11770 char errmsg[STRERR_BUFSIZE]; 11771 struct bpf_link *link; 11772 int prog_fd, pfd; 11773 11774 prog_fd = bpf_program__fd(prog); 11775 if (prog_fd < 0) { 11776 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 11777 return libbpf_err_ptr(-EINVAL); 11778 } 11779 11780 link = calloc(1, sizeof(*link)); 11781 if (!link) 11782 return libbpf_err_ptr(-ENOMEM); 11783 link->detach = &bpf_link__detach_fd; 11784 11785 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 11786 if (pfd < 0) { 11787 pfd = -errno; 11788 free(link); 11789 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n", 11790 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 11791 return libbpf_err_ptr(pfd); 11792 } 11793 link->fd = pfd; 11794 return link; 11795 } 11796 11797 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11798 { 11799 static const char *const prefixes[] = { 11800 "raw_tp", 11801 "raw_tracepoint", 11802 "raw_tp.w", 11803 "raw_tracepoint.w", 11804 }; 11805 size_t i; 11806 const char *tp_name = NULL; 11807 11808 *link = NULL; 11809 11810 for (i = 0; i < ARRAY_SIZE(prefixes); i++) { 11811 size_t pfx_len; 11812 11813 if (!str_has_pfx(prog->sec_name, prefixes[i])) 11814 continue; 11815 11816 pfx_len = strlen(prefixes[i]); 11817 /* no auto-attach case of, e.g., SEC("raw_tp") */ 11818 if (prog->sec_name[pfx_len] == '\0') 11819 return 0; 11820 11821 if (prog->sec_name[pfx_len] != '/') 11822 continue; 11823 11824 tp_name = prog->sec_name + pfx_len + 1; 11825 break; 11826 } 11827 11828 if (!tp_name) { 11829 pr_warn("prog '%s': invalid section name '%s'\n", 11830 prog->name, prog->sec_name); 11831 return -EINVAL; 11832 } 11833 11834 *link = bpf_program__attach_raw_tracepoint(prog, tp_name); 11835 return libbpf_get_error(*link); 11836 } 11837 11838 /* Common logic for all BPF program types that attach to a btf_id */ 11839 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog, 11840 const struct bpf_trace_opts *opts) 11841 { 11842 LIBBPF_OPTS(bpf_link_create_opts, link_opts); 11843 char errmsg[STRERR_BUFSIZE]; 11844 struct bpf_link *link; 11845 int prog_fd, pfd; 11846 11847 if (!OPTS_VALID(opts, bpf_trace_opts)) 11848 return libbpf_err_ptr(-EINVAL); 11849 11850 prog_fd = bpf_program__fd(prog); 11851 if (prog_fd < 0) { 11852 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 11853 return libbpf_err_ptr(-EINVAL); 11854 } 11855 11856 link = calloc(1, sizeof(*link)); 11857 if (!link) 11858 return libbpf_err_ptr(-ENOMEM); 11859 link->detach = &bpf_link__detach_fd; 11860 11861 /* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */ 11862 link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0); 11863 pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts); 11864 if (pfd < 0) { 11865 pfd = -errno; 11866 free(link); 11867 pr_warn("prog '%s': failed to attach: %s\n", 11868 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 11869 return libbpf_err_ptr(pfd); 11870 } 11871 link->fd = pfd; 11872 return link; 11873 } 11874 11875 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) 11876 { 11877 return bpf_program__attach_btf_id(prog, NULL); 11878 } 11879 11880 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog, 11881 const struct bpf_trace_opts *opts) 11882 { 11883 return bpf_program__attach_btf_id(prog, opts); 11884 } 11885 11886 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog) 11887 { 11888 return bpf_program__attach_btf_id(prog, NULL); 11889 } 11890 11891 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11892 { 11893 *link = bpf_program__attach_trace(prog); 11894 return libbpf_get_error(*link); 11895 } 11896 11897 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11898 { 11899 *link = bpf_program__attach_lsm(prog); 11900 return libbpf_get_error(*link); 11901 } 11902 11903 static struct bpf_link * 11904 bpf_program_attach_fd(const struct bpf_program *prog, 11905 int target_fd, const char *target_name, 11906 const struct bpf_link_create_opts *opts) 11907 { 11908 enum bpf_attach_type attach_type; 11909 char errmsg[STRERR_BUFSIZE]; 11910 struct bpf_link *link; 11911 int prog_fd, link_fd; 11912 11913 prog_fd = bpf_program__fd(prog); 11914 if (prog_fd < 0) { 11915 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 11916 return libbpf_err_ptr(-EINVAL); 11917 } 11918 11919 link = calloc(1, sizeof(*link)); 11920 if (!link) 11921 return libbpf_err_ptr(-ENOMEM); 11922 link->detach = &bpf_link__detach_fd; 11923 11924 attach_type = bpf_program__expected_attach_type(prog); 11925 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, opts); 11926 if (link_fd < 0) { 11927 link_fd = -errno; 11928 free(link); 11929 pr_warn("prog '%s': failed to attach to %s: %s\n", 11930 prog->name, target_name, 11931 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 11932 return libbpf_err_ptr(link_fd); 11933 } 11934 link->fd = link_fd; 11935 return link; 11936 } 11937 11938 struct bpf_link * 11939 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd) 11940 { 11941 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL); 11942 } 11943 11944 struct bpf_link * 11945 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd) 11946 { 11947 return bpf_program_attach_fd(prog, netns_fd, "netns", NULL); 11948 } 11949 11950 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex) 11951 { 11952 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 11953 return bpf_program_attach_fd(prog, ifindex, "xdp", NULL); 11954 } 11955 11956 struct bpf_link * 11957 bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex, 11958 const struct bpf_tcx_opts *opts) 11959 { 11960 LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 11961 __u32 relative_id; 11962 int relative_fd; 11963 11964 if (!OPTS_VALID(opts, bpf_tcx_opts)) 11965 return libbpf_err_ptr(-EINVAL); 11966 11967 relative_id = OPTS_GET(opts, relative_id, 0); 11968 relative_fd = OPTS_GET(opts, relative_fd, 0); 11969 11970 /* validate we don't have unexpected combinations of non-zero fields */ 11971 if (!ifindex) { 11972 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n", 11973 prog->name); 11974 return libbpf_err_ptr(-EINVAL); 11975 } 11976 if (relative_fd && relative_id) { 11977 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n", 11978 prog->name); 11979 return libbpf_err_ptr(-EINVAL); 11980 } 11981 11982 link_create_opts.tcx.expected_revision = OPTS_GET(opts, expected_revision, 0); 11983 link_create_opts.tcx.relative_fd = relative_fd; 11984 link_create_opts.tcx.relative_id = relative_id; 11985 link_create_opts.flags = OPTS_GET(opts, flags, 0); 11986 11987 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 11988 return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts); 11989 } 11990 11991 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, 11992 int target_fd, 11993 const char *attach_func_name) 11994 { 11995 int btf_id; 11996 11997 if (!!target_fd != !!attach_func_name) { 11998 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n", 11999 prog->name); 12000 return libbpf_err_ptr(-EINVAL); 12001 } 12002 12003 if (prog->type != BPF_PROG_TYPE_EXT) { 12004 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace", 12005 prog->name); 12006 return libbpf_err_ptr(-EINVAL); 12007 } 12008 12009 if (target_fd) { 12010 LIBBPF_OPTS(bpf_link_create_opts, target_opts); 12011 12012 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd); 12013 if (btf_id < 0) 12014 return libbpf_err_ptr(btf_id); 12015 12016 target_opts.target_btf_id = btf_id; 12017 12018 return bpf_program_attach_fd(prog, target_fd, "freplace", 12019 &target_opts); 12020 } else { 12021 /* no target, so use raw_tracepoint_open for compatibility 12022 * with old kernels 12023 */ 12024 return bpf_program__attach_trace(prog); 12025 } 12026 } 12027 12028 struct bpf_link * 12029 bpf_program__attach_iter(const struct bpf_program *prog, 12030 const struct bpf_iter_attach_opts *opts) 12031 { 12032 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 12033 char errmsg[STRERR_BUFSIZE]; 12034 struct bpf_link *link; 12035 int prog_fd, link_fd; 12036 __u32 target_fd = 0; 12037 12038 if (!OPTS_VALID(opts, bpf_iter_attach_opts)) 12039 return libbpf_err_ptr(-EINVAL); 12040 12041 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0); 12042 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0); 12043 12044 prog_fd = bpf_program__fd(prog); 12045 if (prog_fd < 0) { 12046 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12047 return libbpf_err_ptr(-EINVAL); 12048 } 12049 12050 link = calloc(1, sizeof(*link)); 12051 if (!link) 12052 return libbpf_err_ptr(-ENOMEM); 12053 link->detach = &bpf_link__detach_fd; 12054 12055 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER, 12056 &link_create_opts); 12057 if (link_fd < 0) { 12058 link_fd = -errno; 12059 free(link); 12060 pr_warn("prog '%s': failed to attach to iterator: %s\n", 12061 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 12062 return libbpf_err_ptr(link_fd); 12063 } 12064 link->fd = link_fd; 12065 return link; 12066 } 12067 12068 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12069 { 12070 *link = bpf_program__attach_iter(prog, NULL); 12071 return libbpf_get_error(*link); 12072 } 12073 12074 struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog, 12075 const struct bpf_netfilter_opts *opts) 12076 { 12077 LIBBPF_OPTS(bpf_link_create_opts, lopts); 12078 struct bpf_link *link; 12079 int prog_fd, link_fd; 12080 12081 if (!OPTS_VALID(opts, bpf_netfilter_opts)) 12082 return libbpf_err_ptr(-EINVAL); 12083 12084 prog_fd = bpf_program__fd(prog); 12085 if (prog_fd < 0) { 12086 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12087 return libbpf_err_ptr(-EINVAL); 12088 } 12089 12090 link = calloc(1, sizeof(*link)); 12091 if (!link) 12092 return libbpf_err_ptr(-ENOMEM); 12093 12094 link->detach = &bpf_link__detach_fd; 12095 12096 lopts.netfilter.pf = OPTS_GET(opts, pf, 0); 12097 lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0); 12098 lopts.netfilter.priority = OPTS_GET(opts, priority, 0); 12099 lopts.netfilter.flags = OPTS_GET(opts, flags, 0); 12100 12101 link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts); 12102 if (link_fd < 0) { 12103 char errmsg[STRERR_BUFSIZE]; 12104 12105 link_fd = -errno; 12106 free(link); 12107 pr_warn("prog '%s': failed to attach to netfilter: %s\n", 12108 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 12109 return libbpf_err_ptr(link_fd); 12110 } 12111 link->fd = link_fd; 12112 12113 return link; 12114 } 12115 12116 struct bpf_link *bpf_program__attach(const struct bpf_program *prog) 12117 { 12118 struct bpf_link *link = NULL; 12119 int err; 12120 12121 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) 12122 return libbpf_err_ptr(-EOPNOTSUPP); 12123 12124 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link); 12125 if (err) 12126 return libbpf_err_ptr(err); 12127 12128 /* When calling bpf_program__attach() explicitly, auto-attach support 12129 * is expected to work, so NULL returned link is considered an error. 12130 * This is different for skeleton's attach, see comment in 12131 * bpf_object__attach_skeleton(). 12132 */ 12133 if (!link) 12134 return libbpf_err_ptr(-EOPNOTSUPP); 12135 12136 return link; 12137 } 12138 12139 struct bpf_link_struct_ops { 12140 struct bpf_link link; 12141 int map_fd; 12142 }; 12143 12144 static int bpf_link__detach_struct_ops(struct bpf_link *link) 12145 { 12146 struct bpf_link_struct_ops *st_link; 12147 __u32 zero = 0; 12148 12149 st_link = container_of(link, struct bpf_link_struct_ops, link); 12150 12151 if (st_link->map_fd < 0) 12152 /* w/o a real link */ 12153 return bpf_map_delete_elem(link->fd, &zero); 12154 12155 return close(link->fd); 12156 } 12157 12158 struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map) 12159 { 12160 struct bpf_link_struct_ops *link; 12161 __u32 zero = 0; 12162 int err, fd; 12163 12164 if (!bpf_map__is_struct_ops(map) || map->fd == -1) 12165 return libbpf_err_ptr(-EINVAL); 12166 12167 link = calloc(1, sizeof(*link)); 12168 if (!link) 12169 return libbpf_err_ptr(-EINVAL); 12170 12171 /* kern_vdata should be prepared during the loading phase. */ 12172 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); 12173 /* It can be EBUSY if the map has been used to create or 12174 * update a link before. We don't allow updating the value of 12175 * a struct_ops once it is set. That ensures that the value 12176 * never changed. So, it is safe to skip EBUSY. 12177 */ 12178 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) { 12179 free(link); 12180 return libbpf_err_ptr(err); 12181 } 12182 12183 link->link.detach = bpf_link__detach_struct_ops; 12184 12185 if (!(map->def.map_flags & BPF_F_LINK)) { 12186 /* w/o a real link */ 12187 link->link.fd = map->fd; 12188 link->map_fd = -1; 12189 return &link->link; 12190 } 12191 12192 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL); 12193 if (fd < 0) { 12194 free(link); 12195 return libbpf_err_ptr(fd); 12196 } 12197 12198 link->link.fd = fd; 12199 link->map_fd = map->fd; 12200 12201 return &link->link; 12202 } 12203 12204 /* 12205 * Swap the back struct_ops of a link with a new struct_ops map. 12206 */ 12207 int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map) 12208 { 12209 struct bpf_link_struct_ops *st_ops_link; 12210 __u32 zero = 0; 12211 int err; 12212 12213 if (!bpf_map__is_struct_ops(map) || map->fd < 0) 12214 return -EINVAL; 12215 12216 st_ops_link = container_of(link, struct bpf_link_struct_ops, link); 12217 /* Ensure the type of a link is correct */ 12218 if (st_ops_link->map_fd < 0) 12219 return -EINVAL; 12220 12221 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); 12222 /* It can be EBUSY if the map has been used to create or 12223 * update a link before. We don't allow updating the value of 12224 * a struct_ops once it is set. That ensures that the value 12225 * never changed. So, it is safe to skip EBUSY. 12226 */ 12227 if (err && err != -EBUSY) 12228 return err; 12229 12230 err = bpf_link_update(link->fd, map->fd, NULL); 12231 if (err < 0) 12232 return err; 12233 12234 st_ops_link->map_fd = map->fd; 12235 12236 return 0; 12237 } 12238 12239 typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr, 12240 void *private_data); 12241 12242 static enum bpf_perf_event_ret 12243 perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 12244 void **copy_mem, size_t *copy_size, 12245 bpf_perf_event_print_t fn, void *private_data) 12246 { 12247 struct perf_event_mmap_page *header = mmap_mem; 12248 __u64 data_head = ring_buffer_read_head(header); 12249 __u64 data_tail = header->data_tail; 12250 void *base = ((__u8 *)header) + page_size; 12251 int ret = LIBBPF_PERF_EVENT_CONT; 12252 struct perf_event_header *ehdr; 12253 size_t ehdr_size; 12254 12255 while (data_head != data_tail) { 12256 ehdr = base + (data_tail & (mmap_size - 1)); 12257 ehdr_size = ehdr->size; 12258 12259 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 12260 void *copy_start = ehdr; 12261 size_t len_first = base + mmap_size - copy_start; 12262 size_t len_secnd = ehdr_size - len_first; 12263 12264 if (*copy_size < ehdr_size) { 12265 free(*copy_mem); 12266 *copy_mem = malloc(ehdr_size); 12267 if (!*copy_mem) { 12268 *copy_size = 0; 12269 ret = LIBBPF_PERF_EVENT_ERROR; 12270 break; 12271 } 12272 *copy_size = ehdr_size; 12273 } 12274 12275 memcpy(*copy_mem, copy_start, len_first); 12276 memcpy(*copy_mem + len_first, base, len_secnd); 12277 ehdr = *copy_mem; 12278 } 12279 12280 ret = fn(ehdr, private_data); 12281 data_tail += ehdr_size; 12282 if (ret != LIBBPF_PERF_EVENT_CONT) 12283 break; 12284 } 12285 12286 ring_buffer_write_tail(header, data_tail); 12287 return libbpf_err(ret); 12288 } 12289 12290 struct perf_buffer; 12291 12292 struct perf_buffer_params { 12293 struct perf_event_attr *attr; 12294 /* if event_cb is specified, it takes precendence */ 12295 perf_buffer_event_fn event_cb; 12296 /* sample_cb and lost_cb are higher-level common-case callbacks */ 12297 perf_buffer_sample_fn sample_cb; 12298 perf_buffer_lost_fn lost_cb; 12299 void *ctx; 12300 int cpu_cnt; 12301 int *cpus; 12302 int *map_keys; 12303 }; 12304 12305 struct perf_cpu_buf { 12306 struct perf_buffer *pb; 12307 void *base; /* mmap()'ed memory */ 12308 void *buf; /* for reconstructing segmented data */ 12309 size_t buf_size; 12310 int fd; 12311 int cpu; 12312 int map_key; 12313 }; 12314 12315 struct perf_buffer { 12316 perf_buffer_event_fn event_cb; 12317 perf_buffer_sample_fn sample_cb; 12318 perf_buffer_lost_fn lost_cb; 12319 void *ctx; /* passed into callbacks */ 12320 12321 size_t page_size; 12322 size_t mmap_size; 12323 struct perf_cpu_buf **cpu_bufs; 12324 struct epoll_event *events; 12325 int cpu_cnt; /* number of allocated CPU buffers */ 12326 int epoll_fd; /* perf event FD */ 12327 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 12328 }; 12329 12330 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 12331 struct perf_cpu_buf *cpu_buf) 12332 { 12333 if (!cpu_buf) 12334 return; 12335 if (cpu_buf->base && 12336 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 12337 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 12338 if (cpu_buf->fd >= 0) { 12339 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 12340 close(cpu_buf->fd); 12341 } 12342 free(cpu_buf->buf); 12343 free(cpu_buf); 12344 } 12345 12346 void perf_buffer__free(struct perf_buffer *pb) 12347 { 12348 int i; 12349 12350 if (IS_ERR_OR_NULL(pb)) 12351 return; 12352 if (pb->cpu_bufs) { 12353 for (i = 0; i < pb->cpu_cnt; i++) { 12354 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 12355 12356 if (!cpu_buf) 12357 continue; 12358 12359 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 12360 perf_buffer__free_cpu_buf(pb, cpu_buf); 12361 } 12362 free(pb->cpu_bufs); 12363 } 12364 if (pb->epoll_fd >= 0) 12365 close(pb->epoll_fd); 12366 free(pb->events); 12367 free(pb); 12368 } 12369 12370 static struct perf_cpu_buf * 12371 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 12372 int cpu, int map_key) 12373 { 12374 struct perf_cpu_buf *cpu_buf; 12375 char msg[STRERR_BUFSIZE]; 12376 int err; 12377 12378 cpu_buf = calloc(1, sizeof(*cpu_buf)); 12379 if (!cpu_buf) 12380 return ERR_PTR(-ENOMEM); 12381 12382 cpu_buf->pb = pb; 12383 cpu_buf->cpu = cpu; 12384 cpu_buf->map_key = map_key; 12385 12386 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 12387 -1, PERF_FLAG_FD_CLOEXEC); 12388 if (cpu_buf->fd < 0) { 12389 err = -errno; 12390 pr_warn("failed to open perf buffer event on cpu #%d: %s\n", 12391 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 12392 goto error; 12393 } 12394 12395 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 12396 PROT_READ | PROT_WRITE, MAP_SHARED, 12397 cpu_buf->fd, 0); 12398 if (cpu_buf->base == MAP_FAILED) { 12399 cpu_buf->base = NULL; 12400 err = -errno; 12401 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", 12402 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 12403 goto error; 12404 } 12405 12406 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 12407 err = -errno; 12408 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", 12409 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 12410 goto error; 12411 } 12412 12413 return cpu_buf; 12414 12415 error: 12416 perf_buffer__free_cpu_buf(pb, cpu_buf); 12417 return (struct perf_cpu_buf *)ERR_PTR(err); 12418 } 12419 12420 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 12421 struct perf_buffer_params *p); 12422 12423 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 12424 perf_buffer_sample_fn sample_cb, 12425 perf_buffer_lost_fn lost_cb, 12426 void *ctx, 12427 const struct perf_buffer_opts *opts) 12428 { 12429 const size_t attr_sz = sizeof(struct perf_event_attr); 12430 struct perf_buffer_params p = {}; 12431 struct perf_event_attr attr; 12432 __u32 sample_period; 12433 12434 if (!OPTS_VALID(opts, perf_buffer_opts)) 12435 return libbpf_err_ptr(-EINVAL); 12436 12437 sample_period = OPTS_GET(opts, sample_period, 1); 12438 if (!sample_period) 12439 sample_period = 1; 12440 12441 memset(&attr, 0, attr_sz); 12442 attr.size = attr_sz; 12443 attr.config = PERF_COUNT_SW_BPF_OUTPUT; 12444 attr.type = PERF_TYPE_SOFTWARE; 12445 attr.sample_type = PERF_SAMPLE_RAW; 12446 attr.sample_period = sample_period; 12447 attr.wakeup_events = sample_period; 12448 12449 p.attr = &attr; 12450 p.sample_cb = sample_cb; 12451 p.lost_cb = lost_cb; 12452 p.ctx = ctx; 12453 12454 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 12455 } 12456 12457 struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt, 12458 struct perf_event_attr *attr, 12459 perf_buffer_event_fn event_cb, void *ctx, 12460 const struct perf_buffer_raw_opts *opts) 12461 { 12462 struct perf_buffer_params p = {}; 12463 12464 if (!attr) 12465 return libbpf_err_ptr(-EINVAL); 12466 12467 if (!OPTS_VALID(opts, perf_buffer_raw_opts)) 12468 return libbpf_err_ptr(-EINVAL); 12469 12470 p.attr = attr; 12471 p.event_cb = event_cb; 12472 p.ctx = ctx; 12473 p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0); 12474 p.cpus = OPTS_GET(opts, cpus, NULL); 12475 p.map_keys = OPTS_GET(opts, map_keys, NULL); 12476 12477 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 12478 } 12479 12480 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 12481 struct perf_buffer_params *p) 12482 { 12483 const char *online_cpus_file = "/sys/devices/system/cpu/online"; 12484 struct bpf_map_info map; 12485 char msg[STRERR_BUFSIZE]; 12486 struct perf_buffer *pb; 12487 bool *online = NULL; 12488 __u32 map_info_len; 12489 int err, i, j, n; 12490 12491 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) { 12492 pr_warn("page count should be power of two, but is %zu\n", 12493 page_cnt); 12494 return ERR_PTR(-EINVAL); 12495 } 12496 12497 /* best-effort sanity checks */ 12498 memset(&map, 0, sizeof(map)); 12499 map_info_len = sizeof(map); 12500 err = bpf_map_get_info_by_fd(map_fd, &map, &map_info_len); 12501 if (err) { 12502 err = -errno; 12503 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return 12504 * -EBADFD, -EFAULT, or -E2BIG on real error 12505 */ 12506 if (err != -EINVAL) { 12507 pr_warn("failed to get map info for map FD %d: %s\n", 12508 map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); 12509 return ERR_PTR(err); 12510 } 12511 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n", 12512 map_fd); 12513 } else { 12514 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 12515 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 12516 map.name); 12517 return ERR_PTR(-EINVAL); 12518 } 12519 } 12520 12521 pb = calloc(1, sizeof(*pb)); 12522 if (!pb) 12523 return ERR_PTR(-ENOMEM); 12524 12525 pb->event_cb = p->event_cb; 12526 pb->sample_cb = p->sample_cb; 12527 pb->lost_cb = p->lost_cb; 12528 pb->ctx = p->ctx; 12529 12530 pb->page_size = getpagesize(); 12531 pb->mmap_size = pb->page_size * page_cnt; 12532 pb->map_fd = map_fd; 12533 12534 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 12535 if (pb->epoll_fd < 0) { 12536 err = -errno; 12537 pr_warn("failed to create epoll instance: %s\n", 12538 libbpf_strerror_r(err, msg, sizeof(msg))); 12539 goto error; 12540 } 12541 12542 if (p->cpu_cnt > 0) { 12543 pb->cpu_cnt = p->cpu_cnt; 12544 } else { 12545 pb->cpu_cnt = libbpf_num_possible_cpus(); 12546 if (pb->cpu_cnt < 0) { 12547 err = pb->cpu_cnt; 12548 goto error; 12549 } 12550 if (map.max_entries && map.max_entries < pb->cpu_cnt) 12551 pb->cpu_cnt = map.max_entries; 12552 } 12553 12554 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 12555 if (!pb->events) { 12556 err = -ENOMEM; 12557 pr_warn("failed to allocate events: out of memory\n"); 12558 goto error; 12559 } 12560 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 12561 if (!pb->cpu_bufs) { 12562 err = -ENOMEM; 12563 pr_warn("failed to allocate buffers: out of memory\n"); 12564 goto error; 12565 } 12566 12567 err = parse_cpu_mask_file(online_cpus_file, &online, &n); 12568 if (err) { 12569 pr_warn("failed to get online CPU mask: %d\n", err); 12570 goto error; 12571 } 12572 12573 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { 12574 struct perf_cpu_buf *cpu_buf; 12575 int cpu, map_key; 12576 12577 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 12578 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 12579 12580 /* in case user didn't explicitly requested particular CPUs to 12581 * be attached to, skip offline/not present CPUs 12582 */ 12583 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) 12584 continue; 12585 12586 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 12587 if (IS_ERR(cpu_buf)) { 12588 err = PTR_ERR(cpu_buf); 12589 goto error; 12590 } 12591 12592 pb->cpu_bufs[j] = cpu_buf; 12593 12594 err = bpf_map_update_elem(pb->map_fd, &map_key, 12595 &cpu_buf->fd, 0); 12596 if (err) { 12597 err = -errno; 12598 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 12599 cpu, map_key, cpu_buf->fd, 12600 libbpf_strerror_r(err, msg, sizeof(msg))); 12601 goto error; 12602 } 12603 12604 pb->events[j].events = EPOLLIN; 12605 pb->events[j].data.ptr = cpu_buf; 12606 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 12607 &pb->events[j]) < 0) { 12608 err = -errno; 12609 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 12610 cpu, cpu_buf->fd, 12611 libbpf_strerror_r(err, msg, sizeof(msg))); 12612 goto error; 12613 } 12614 j++; 12615 } 12616 pb->cpu_cnt = j; 12617 free(online); 12618 12619 return pb; 12620 12621 error: 12622 free(online); 12623 if (pb) 12624 perf_buffer__free(pb); 12625 return ERR_PTR(err); 12626 } 12627 12628 struct perf_sample_raw { 12629 struct perf_event_header header; 12630 uint32_t size; 12631 char data[]; 12632 }; 12633 12634 struct perf_sample_lost { 12635 struct perf_event_header header; 12636 uint64_t id; 12637 uint64_t lost; 12638 uint64_t sample_id; 12639 }; 12640 12641 static enum bpf_perf_event_ret 12642 perf_buffer__process_record(struct perf_event_header *e, void *ctx) 12643 { 12644 struct perf_cpu_buf *cpu_buf = ctx; 12645 struct perf_buffer *pb = cpu_buf->pb; 12646 void *data = e; 12647 12648 /* user wants full control over parsing perf event */ 12649 if (pb->event_cb) 12650 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 12651 12652 switch (e->type) { 12653 case PERF_RECORD_SAMPLE: { 12654 struct perf_sample_raw *s = data; 12655 12656 if (pb->sample_cb) 12657 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 12658 break; 12659 } 12660 case PERF_RECORD_LOST: { 12661 struct perf_sample_lost *s = data; 12662 12663 if (pb->lost_cb) 12664 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 12665 break; 12666 } 12667 default: 12668 pr_warn("unknown perf sample type %d\n", e->type); 12669 return LIBBPF_PERF_EVENT_ERROR; 12670 } 12671 return LIBBPF_PERF_EVENT_CONT; 12672 } 12673 12674 static int perf_buffer__process_records(struct perf_buffer *pb, 12675 struct perf_cpu_buf *cpu_buf) 12676 { 12677 enum bpf_perf_event_ret ret; 12678 12679 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size, 12680 pb->page_size, &cpu_buf->buf, 12681 &cpu_buf->buf_size, 12682 perf_buffer__process_record, cpu_buf); 12683 if (ret != LIBBPF_PERF_EVENT_CONT) 12684 return ret; 12685 return 0; 12686 } 12687 12688 int perf_buffer__epoll_fd(const struct perf_buffer *pb) 12689 { 12690 return pb->epoll_fd; 12691 } 12692 12693 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 12694 { 12695 int i, cnt, err; 12696 12697 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 12698 if (cnt < 0) 12699 return -errno; 12700 12701 for (i = 0; i < cnt; i++) { 12702 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 12703 12704 err = perf_buffer__process_records(pb, cpu_buf); 12705 if (err) { 12706 pr_warn("error while processing records: %d\n", err); 12707 return libbpf_err(err); 12708 } 12709 } 12710 return cnt; 12711 } 12712 12713 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer 12714 * manager. 12715 */ 12716 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb) 12717 { 12718 return pb->cpu_cnt; 12719 } 12720 12721 /* 12722 * Return perf_event FD of a ring buffer in *buf_idx* slot of 12723 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using 12724 * select()/poll()/epoll() Linux syscalls. 12725 */ 12726 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx) 12727 { 12728 struct perf_cpu_buf *cpu_buf; 12729 12730 if (buf_idx >= pb->cpu_cnt) 12731 return libbpf_err(-EINVAL); 12732 12733 cpu_buf = pb->cpu_bufs[buf_idx]; 12734 if (!cpu_buf) 12735 return libbpf_err(-ENOENT); 12736 12737 return cpu_buf->fd; 12738 } 12739 12740 int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size) 12741 { 12742 struct perf_cpu_buf *cpu_buf; 12743 12744 if (buf_idx >= pb->cpu_cnt) 12745 return libbpf_err(-EINVAL); 12746 12747 cpu_buf = pb->cpu_bufs[buf_idx]; 12748 if (!cpu_buf) 12749 return libbpf_err(-ENOENT); 12750 12751 *buf = cpu_buf->base; 12752 *buf_size = pb->mmap_size; 12753 return 0; 12754 } 12755 12756 /* 12757 * Consume data from perf ring buffer corresponding to slot *buf_idx* in 12758 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to 12759 * consume, do nothing and return success. 12760 * Returns: 12761 * - 0 on success; 12762 * - <0 on failure. 12763 */ 12764 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx) 12765 { 12766 struct perf_cpu_buf *cpu_buf; 12767 12768 if (buf_idx >= pb->cpu_cnt) 12769 return libbpf_err(-EINVAL); 12770 12771 cpu_buf = pb->cpu_bufs[buf_idx]; 12772 if (!cpu_buf) 12773 return libbpf_err(-ENOENT); 12774 12775 return perf_buffer__process_records(pb, cpu_buf); 12776 } 12777 12778 int perf_buffer__consume(struct perf_buffer *pb) 12779 { 12780 int i, err; 12781 12782 for (i = 0; i < pb->cpu_cnt; i++) { 12783 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 12784 12785 if (!cpu_buf) 12786 continue; 12787 12788 err = perf_buffer__process_records(pb, cpu_buf); 12789 if (err) { 12790 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err); 12791 return libbpf_err(err); 12792 } 12793 } 12794 return 0; 12795 } 12796 12797 int bpf_program__set_attach_target(struct bpf_program *prog, 12798 int attach_prog_fd, 12799 const char *attach_func_name) 12800 { 12801 int btf_obj_fd = 0, btf_id = 0, err; 12802 12803 if (!prog || attach_prog_fd < 0) 12804 return libbpf_err(-EINVAL); 12805 12806 if (prog->obj->loaded) 12807 return libbpf_err(-EINVAL); 12808 12809 if (attach_prog_fd && !attach_func_name) { 12810 /* remember attach_prog_fd and let bpf_program__load() find 12811 * BTF ID during the program load 12812 */ 12813 prog->attach_prog_fd = attach_prog_fd; 12814 return 0; 12815 } 12816 12817 if (attach_prog_fd) { 12818 btf_id = libbpf_find_prog_btf_id(attach_func_name, 12819 attach_prog_fd); 12820 if (btf_id < 0) 12821 return libbpf_err(btf_id); 12822 } else { 12823 if (!attach_func_name) 12824 return libbpf_err(-EINVAL); 12825 12826 /* load btf_vmlinux, if not yet */ 12827 err = bpf_object__load_vmlinux_btf(prog->obj, true); 12828 if (err) 12829 return libbpf_err(err); 12830 err = find_kernel_btf_id(prog->obj, attach_func_name, 12831 prog->expected_attach_type, 12832 &btf_obj_fd, &btf_id); 12833 if (err) 12834 return libbpf_err(err); 12835 } 12836 12837 prog->attach_btf_id = btf_id; 12838 prog->attach_btf_obj_fd = btf_obj_fd; 12839 prog->attach_prog_fd = attach_prog_fd; 12840 return 0; 12841 } 12842 12843 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) 12844 { 12845 int err = 0, n, len, start, end = -1; 12846 bool *tmp; 12847 12848 *mask = NULL; 12849 *mask_sz = 0; 12850 12851 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 12852 while (*s) { 12853 if (*s == ',' || *s == '\n') { 12854 s++; 12855 continue; 12856 } 12857 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); 12858 if (n <= 0 || n > 2) { 12859 pr_warn("Failed to get CPU range %s: %d\n", s, n); 12860 err = -EINVAL; 12861 goto cleanup; 12862 } else if (n == 1) { 12863 end = start; 12864 } 12865 if (start < 0 || start > end) { 12866 pr_warn("Invalid CPU range [%d,%d] in %s\n", 12867 start, end, s); 12868 err = -EINVAL; 12869 goto cleanup; 12870 } 12871 tmp = realloc(*mask, end + 1); 12872 if (!tmp) { 12873 err = -ENOMEM; 12874 goto cleanup; 12875 } 12876 *mask = tmp; 12877 memset(tmp + *mask_sz, 0, start - *mask_sz); 12878 memset(tmp + start, 1, end - start + 1); 12879 *mask_sz = end + 1; 12880 s += len; 12881 } 12882 if (!*mask_sz) { 12883 pr_warn("Empty CPU range\n"); 12884 return -EINVAL; 12885 } 12886 return 0; 12887 cleanup: 12888 free(*mask); 12889 *mask = NULL; 12890 return err; 12891 } 12892 12893 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) 12894 { 12895 int fd, err = 0, len; 12896 char buf[128]; 12897 12898 fd = open(fcpu, O_RDONLY | O_CLOEXEC); 12899 if (fd < 0) { 12900 err = -errno; 12901 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); 12902 return err; 12903 } 12904 len = read(fd, buf, sizeof(buf)); 12905 close(fd); 12906 if (len <= 0) { 12907 err = len ? -errno : -EINVAL; 12908 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); 12909 return err; 12910 } 12911 if (len >= sizeof(buf)) { 12912 pr_warn("CPU mask is too big in file %s\n", fcpu); 12913 return -E2BIG; 12914 } 12915 buf[len] = '\0'; 12916 12917 return parse_cpu_mask_str(buf, mask, mask_sz); 12918 } 12919 12920 int libbpf_num_possible_cpus(void) 12921 { 12922 static const char *fcpu = "/sys/devices/system/cpu/possible"; 12923 static int cpus; 12924 int err, n, i, tmp_cpus; 12925 bool *mask; 12926 12927 tmp_cpus = READ_ONCE(cpus); 12928 if (tmp_cpus > 0) 12929 return tmp_cpus; 12930 12931 err = parse_cpu_mask_file(fcpu, &mask, &n); 12932 if (err) 12933 return libbpf_err(err); 12934 12935 tmp_cpus = 0; 12936 for (i = 0; i < n; i++) { 12937 if (mask[i]) 12938 tmp_cpus++; 12939 } 12940 free(mask); 12941 12942 WRITE_ONCE(cpus, tmp_cpus); 12943 return tmp_cpus; 12944 } 12945 12946 static int populate_skeleton_maps(const struct bpf_object *obj, 12947 struct bpf_map_skeleton *maps, 12948 size_t map_cnt) 12949 { 12950 int i; 12951 12952 for (i = 0; i < map_cnt; i++) { 12953 struct bpf_map **map = maps[i].map; 12954 const char *name = maps[i].name; 12955 void **mmaped = maps[i].mmaped; 12956 12957 *map = bpf_object__find_map_by_name(obj, name); 12958 if (!*map) { 12959 pr_warn("failed to find skeleton map '%s'\n", name); 12960 return -ESRCH; 12961 } 12962 12963 /* externs shouldn't be pre-setup from user code */ 12964 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) 12965 *mmaped = (*map)->mmaped; 12966 } 12967 return 0; 12968 } 12969 12970 static int populate_skeleton_progs(const struct bpf_object *obj, 12971 struct bpf_prog_skeleton *progs, 12972 size_t prog_cnt) 12973 { 12974 int i; 12975 12976 for (i = 0; i < prog_cnt; i++) { 12977 struct bpf_program **prog = progs[i].prog; 12978 const char *name = progs[i].name; 12979 12980 *prog = bpf_object__find_program_by_name(obj, name); 12981 if (!*prog) { 12982 pr_warn("failed to find skeleton program '%s'\n", name); 12983 return -ESRCH; 12984 } 12985 } 12986 return 0; 12987 } 12988 12989 int bpf_object__open_skeleton(struct bpf_object_skeleton *s, 12990 const struct bpf_object_open_opts *opts) 12991 { 12992 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts, 12993 .object_name = s->name, 12994 ); 12995 struct bpf_object *obj; 12996 int err; 12997 12998 /* Attempt to preserve opts->object_name, unless overriden by user 12999 * explicitly. Overwriting object name for skeletons is discouraged, 13000 * as it breaks global data maps, because they contain object name 13001 * prefix as their own map name prefix. When skeleton is generated, 13002 * bpftool is making an assumption that this name will stay the same. 13003 */ 13004 if (opts) { 13005 memcpy(&skel_opts, opts, sizeof(*opts)); 13006 if (!opts->object_name) 13007 skel_opts.object_name = s->name; 13008 } 13009 13010 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); 13011 err = libbpf_get_error(obj); 13012 if (err) { 13013 pr_warn("failed to initialize skeleton BPF object '%s': %d\n", 13014 s->name, err); 13015 return libbpf_err(err); 13016 } 13017 13018 *s->obj = obj; 13019 err = populate_skeleton_maps(obj, s->maps, s->map_cnt); 13020 if (err) { 13021 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err); 13022 return libbpf_err(err); 13023 } 13024 13025 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt); 13026 if (err) { 13027 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err); 13028 return libbpf_err(err); 13029 } 13030 13031 return 0; 13032 } 13033 13034 int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s) 13035 { 13036 int err, len, var_idx, i; 13037 const char *var_name; 13038 const struct bpf_map *map; 13039 struct btf *btf; 13040 __u32 map_type_id; 13041 const struct btf_type *map_type, *var_type; 13042 const struct bpf_var_skeleton *var_skel; 13043 struct btf_var_secinfo *var; 13044 13045 if (!s->obj) 13046 return libbpf_err(-EINVAL); 13047 13048 btf = bpf_object__btf(s->obj); 13049 if (!btf) { 13050 pr_warn("subskeletons require BTF at runtime (object %s)\n", 13051 bpf_object__name(s->obj)); 13052 return libbpf_err(-errno); 13053 } 13054 13055 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt); 13056 if (err) { 13057 pr_warn("failed to populate subskeleton maps: %d\n", err); 13058 return libbpf_err(err); 13059 } 13060 13061 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt); 13062 if (err) { 13063 pr_warn("failed to populate subskeleton maps: %d\n", err); 13064 return libbpf_err(err); 13065 } 13066 13067 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) { 13068 var_skel = &s->vars[var_idx]; 13069 map = *var_skel->map; 13070 map_type_id = bpf_map__btf_value_type_id(map); 13071 map_type = btf__type_by_id(btf, map_type_id); 13072 13073 if (!btf_is_datasec(map_type)) { 13074 pr_warn("type for map '%1$s' is not a datasec: %2$s", 13075 bpf_map__name(map), 13076 __btf_kind_str(btf_kind(map_type))); 13077 return libbpf_err(-EINVAL); 13078 } 13079 13080 len = btf_vlen(map_type); 13081 var = btf_var_secinfos(map_type); 13082 for (i = 0; i < len; i++, var++) { 13083 var_type = btf__type_by_id(btf, var->type); 13084 var_name = btf__name_by_offset(btf, var_type->name_off); 13085 if (strcmp(var_name, var_skel->name) == 0) { 13086 *var_skel->addr = map->mmaped + var->offset; 13087 break; 13088 } 13089 } 13090 } 13091 return 0; 13092 } 13093 13094 void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s) 13095 { 13096 if (!s) 13097 return; 13098 free(s->maps); 13099 free(s->progs); 13100 free(s->vars); 13101 free(s); 13102 } 13103 13104 int bpf_object__load_skeleton(struct bpf_object_skeleton *s) 13105 { 13106 int i, err; 13107 13108 err = bpf_object__load(*s->obj); 13109 if (err) { 13110 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); 13111 return libbpf_err(err); 13112 } 13113 13114 for (i = 0; i < s->map_cnt; i++) { 13115 struct bpf_map *map = *s->maps[i].map; 13116 size_t mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 13117 int prot, map_fd = bpf_map__fd(map); 13118 void **mmaped = s->maps[i].mmaped; 13119 13120 if (!mmaped) 13121 continue; 13122 13123 if (!(map->def.map_flags & BPF_F_MMAPABLE)) { 13124 *mmaped = NULL; 13125 continue; 13126 } 13127 13128 if (map->def.map_flags & BPF_F_RDONLY_PROG) 13129 prot = PROT_READ; 13130 else 13131 prot = PROT_READ | PROT_WRITE; 13132 13133 /* Remap anonymous mmap()-ed "map initialization image" as 13134 * a BPF map-backed mmap()-ed memory, but preserving the same 13135 * memory address. This will cause kernel to change process' 13136 * page table to point to a different piece of kernel memory, 13137 * but from userspace point of view memory address (and its 13138 * contents, being identical at this point) will stay the 13139 * same. This mapping will be released by bpf_object__close() 13140 * as per normal clean up procedure, so we don't need to worry 13141 * about it from skeleton's clean up perspective. 13142 */ 13143 *mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0); 13144 if (*mmaped == MAP_FAILED) { 13145 err = -errno; 13146 *mmaped = NULL; 13147 pr_warn("failed to re-mmap() map '%s': %d\n", 13148 bpf_map__name(map), err); 13149 return libbpf_err(err); 13150 } 13151 } 13152 13153 return 0; 13154 } 13155 13156 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) 13157 { 13158 int i, err; 13159 13160 for (i = 0; i < s->prog_cnt; i++) { 13161 struct bpf_program *prog = *s->progs[i].prog; 13162 struct bpf_link **link = s->progs[i].link; 13163 13164 if (!prog->autoload || !prog->autoattach) 13165 continue; 13166 13167 /* auto-attaching not supported for this program */ 13168 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) 13169 continue; 13170 13171 /* if user already set the link manually, don't attempt auto-attach */ 13172 if (*link) 13173 continue; 13174 13175 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link); 13176 if (err) { 13177 pr_warn("prog '%s': failed to auto-attach: %d\n", 13178 bpf_program__name(prog), err); 13179 return libbpf_err(err); 13180 } 13181 13182 /* It's possible that for some SEC() definitions auto-attach 13183 * is supported in some cases (e.g., if definition completely 13184 * specifies target information), but is not in other cases. 13185 * SEC("uprobe") is one such case. If user specified target 13186 * binary and function name, such BPF program can be 13187 * auto-attached. But if not, it shouldn't trigger skeleton's 13188 * attach to fail. It should just be skipped. 13189 * attach_fn signals such case with returning 0 (no error) and 13190 * setting link to NULL. 13191 */ 13192 } 13193 13194 return 0; 13195 } 13196 13197 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) 13198 { 13199 int i; 13200 13201 for (i = 0; i < s->prog_cnt; i++) { 13202 struct bpf_link **link = s->progs[i].link; 13203 13204 bpf_link__destroy(*link); 13205 *link = NULL; 13206 } 13207 } 13208 13209 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) 13210 { 13211 if (!s) 13212 return; 13213 13214 if (s->progs) 13215 bpf_object__detach_skeleton(s); 13216 if (s->obj) 13217 bpf_object__close(*s->obj); 13218 free(s->maps); 13219 free(s->progs); 13220 free(s); 13221 } 13222