1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <limits.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <endian.h> 25 #include <fcntl.h> 26 #include <errno.h> 27 #include <ctype.h> 28 #include <asm/unistd.h> 29 #include <linux/err.h> 30 #include <linux/kernel.h> 31 #include <linux/bpf.h> 32 #include <linux/btf.h> 33 #include <linux/filter.h> 34 #include <linux/limits.h> 35 #include <linux/perf_event.h> 36 #include <linux/bpf_perf_event.h> 37 #include <linux/ring_buffer.h> 38 #include <sys/epoll.h> 39 #include <sys/ioctl.h> 40 #include <sys/mman.h> 41 #include <sys/stat.h> 42 #include <sys/types.h> 43 #include <sys/vfs.h> 44 #include <sys/utsname.h> 45 #include <sys/resource.h> 46 #include <libelf.h> 47 #include <gelf.h> 48 #include <zlib.h> 49 50 #include "libbpf.h" 51 #include "bpf.h" 52 #include "btf.h" 53 #include "str_error.h" 54 #include "libbpf_internal.h" 55 #include "hashmap.h" 56 #include "bpf_gen_internal.h" 57 #include "zip.h" 58 59 #ifndef BPF_FS_MAGIC 60 #define BPF_FS_MAGIC 0xcafe4a11 61 #endif 62 63 #define BPF_FS_DEFAULT_PATH "/sys/fs/bpf" 64 65 #define BPF_INSN_SZ (sizeof(struct bpf_insn)) 66 67 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 68 * compilation if user enables corresponding warning. Disable it explicitly. 69 */ 70 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 71 72 #define __printf(a, b) __attribute__((format(printf, a, b))) 73 74 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); 75 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog); 76 static int map_set_def_max_entries(struct bpf_map *map); 77 78 static const char * const attach_type_name[] = { 79 [BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress", 80 [BPF_CGROUP_INET_EGRESS] = "cgroup_inet_egress", 81 [BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create", 82 [BPF_CGROUP_INET_SOCK_RELEASE] = "cgroup_inet_sock_release", 83 [BPF_CGROUP_SOCK_OPS] = "cgroup_sock_ops", 84 [BPF_CGROUP_DEVICE] = "cgroup_device", 85 [BPF_CGROUP_INET4_BIND] = "cgroup_inet4_bind", 86 [BPF_CGROUP_INET6_BIND] = "cgroup_inet6_bind", 87 [BPF_CGROUP_INET4_CONNECT] = "cgroup_inet4_connect", 88 [BPF_CGROUP_INET6_CONNECT] = "cgroup_inet6_connect", 89 [BPF_CGROUP_UNIX_CONNECT] = "cgroup_unix_connect", 90 [BPF_CGROUP_INET4_POST_BIND] = "cgroup_inet4_post_bind", 91 [BPF_CGROUP_INET6_POST_BIND] = "cgroup_inet6_post_bind", 92 [BPF_CGROUP_INET4_GETPEERNAME] = "cgroup_inet4_getpeername", 93 [BPF_CGROUP_INET6_GETPEERNAME] = "cgroup_inet6_getpeername", 94 [BPF_CGROUP_UNIX_GETPEERNAME] = "cgroup_unix_getpeername", 95 [BPF_CGROUP_INET4_GETSOCKNAME] = "cgroup_inet4_getsockname", 96 [BPF_CGROUP_INET6_GETSOCKNAME] = "cgroup_inet6_getsockname", 97 [BPF_CGROUP_UNIX_GETSOCKNAME] = "cgroup_unix_getsockname", 98 [BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg", 99 [BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg", 100 [BPF_CGROUP_UNIX_SENDMSG] = "cgroup_unix_sendmsg", 101 [BPF_CGROUP_SYSCTL] = "cgroup_sysctl", 102 [BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg", 103 [BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg", 104 [BPF_CGROUP_UNIX_RECVMSG] = "cgroup_unix_recvmsg", 105 [BPF_CGROUP_GETSOCKOPT] = "cgroup_getsockopt", 106 [BPF_CGROUP_SETSOCKOPT] = "cgroup_setsockopt", 107 [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser", 108 [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict", 109 [BPF_SK_SKB_VERDICT] = "sk_skb_verdict", 110 [BPF_SK_MSG_VERDICT] = "sk_msg_verdict", 111 [BPF_LIRC_MODE2] = "lirc_mode2", 112 [BPF_FLOW_DISSECTOR] = "flow_dissector", 113 [BPF_TRACE_RAW_TP] = "trace_raw_tp", 114 [BPF_TRACE_FENTRY] = "trace_fentry", 115 [BPF_TRACE_FEXIT] = "trace_fexit", 116 [BPF_MODIFY_RETURN] = "modify_return", 117 [BPF_LSM_MAC] = "lsm_mac", 118 [BPF_LSM_CGROUP] = "lsm_cgroup", 119 [BPF_SK_LOOKUP] = "sk_lookup", 120 [BPF_TRACE_ITER] = "trace_iter", 121 [BPF_XDP_DEVMAP] = "xdp_devmap", 122 [BPF_XDP_CPUMAP] = "xdp_cpumap", 123 [BPF_XDP] = "xdp", 124 [BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select", 125 [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_reuseport_select_or_migrate", 126 [BPF_PERF_EVENT] = "perf_event", 127 [BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi", 128 [BPF_STRUCT_OPS] = "struct_ops", 129 [BPF_NETFILTER] = "netfilter", 130 [BPF_TCX_INGRESS] = "tcx_ingress", 131 [BPF_TCX_EGRESS] = "tcx_egress", 132 [BPF_TRACE_UPROBE_MULTI] = "trace_uprobe_multi", 133 [BPF_NETKIT_PRIMARY] = "netkit_primary", 134 [BPF_NETKIT_PEER] = "netkit_peer", 135 }; 136 137 static const char * const link_type_name[] = { 138 [BPF_LINK_TYPE_UNSPEC] = "unspec", 139 [BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", 140 [BPF_LINK_TYPE_TRACING] = "tracing", 141 [BPF_LINK_TYPE_CGROUP] = "cgroup", 142 [BPF_LINK_TYPE_ITER] = "iter", 143 [BPF_LINK_TYPE_NETNS] = "netns", 144 [BPF_LINK_TYPE_XDP] = "xdp", 145 [BPF_LINK_TYPE_PERF_EVENT] = "perf_event", 146 [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi", 147 [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops", 148 [BPF_LINK_TYPE_NETFILTER] = "netfilter", 149 [BPF_LINK_TYPE_TCX] = "tcx", 150 [BPF_LINK_TYPE_UPROBE_MULTI] = "uprobe_multi", 151 [BPF_LINK_TYPE_NETKIT] = "netkit", 152 }; 153 154 static const char * const map_type_name[] = { 155 [BPF_MAP_TYPE_UNSPEC] = "unspec", 156 [BPF_MAP_TYPE_HASH] = "hash", 157 [BPF_MAP_TYPE_ARRAY] = "array", 158 [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array", 159 [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array", 160 [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash", 161 [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array", 162 [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace", 163 [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array", 164 [BPF_MAP_TYPE_LRU_HASH] = "lru_hash", 165 [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash", 166 [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie", 167 [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps", 168 [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps", 169 [BPF_MAP_TYPE_DEVMAP] = "devmap", 170 [BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash", 171 [BPF_MAP_TYPE_SOCKMAP] = "sockmap", 172 [BPF_MAP_TYPE_CPUMAP] = "cpumap", 173 [BPF_MAP_TYPE_XSKMAP] = "xskmap", 174 [BPF_MAP_TYPE_SOCKHASH] = "sockhash", 175 [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage", 176 [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray", 177 [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage", 178 [BPF_MAP_TYPE_QUEUE] = "queue", 179 [BPF_MAP_TYPE_STACK] = "stack", 180 [BPF_MAP_TYPE_SK_STORAGE] = "sk_storage", 181 [BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops", 182 [BPF_MAP_TYPE_RINGBUF] = "ringbuf", 183 [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage", 184 [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage", 185 [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter", 186 [BPF_MAP_TYPE_USER_RINGBUF] = "user_ringbuf", 187 [BPF_MAP_TYPE_CGRP_STORAGE] = "cgrp_storage", 188 }; 189 190 static const char * const prog_type_name[] = { 191 [BPF_PROG_TYPE_UNSPEC] = "unspec", 192 [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", 193 [BPF_PROG_TYPE_KPROBE] = "kprobe", 194 [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls", 195 [BPF_PROG_TYPE_SCHED_ACT] = "sched_act", 196 [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint", 197 [BPF_PROG_TYPE_XDP] = "xdp", 198 [BPF_PROG_TYPE_PERF_EVENT] = "perf_event", 199 [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb", 200 [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock", 201 [BPF_PROG_TYPE_LWT_IN] = "lwt_in", 202 [BPF_PROG_TYPE_LWT_OUT] = "lwt_out", 203 [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", 204 [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", 205 [BPF_PROG_TYPE_SK_SKB] = "sk_skb", 206 [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device", 207 [BPF_PROG_TYPE_SK_MSG] = "sk_msg", 208 [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", 209 [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr", 210 [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local", 211 [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2", 212 [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport", 213 [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector", 214 [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl", 215 [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable", 216 [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt", 217 [BPF_PROG_TYPE_TRACING] = "tracing", 218 [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops", 219 [BPF_PROG_TYPE_EXT] = "ext", 220 [BPF_PROG_TYPE_LSM] = "lsm", 221 [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup", 222 [BPF_PROG_TYPE_SYSCALL] = "syscall", 223 [BPF_PROG_TYPE_NETFILTER] = "netfilter", 224 }; 225 226 static int __base_pr(enum libbpf_print_level level, const char *format, 227 va_list args) 228 { 229 if (level == LIBBPF_DEBUG) 230 return 0; 231 232 return vfprintf(stderr, format, args); 233 } 234 235 static libbpf_print_fn_t __libbpf_pr = __base_pr; 236 237 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) 238 { 239 libbpf_print_fn_t old_print_fn; 240 241 old_print_fn = __atomic_exchange_n(&__libbpf_pr, fn, __ATOMIC_RELAXED); 242 243 return old_print_fn; 244 } 245 246 __printf(2, 3) 247 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 248 { 249 va_list args; 250 int old_errno; 251 libbpf_print_fn_t print_fn; 252 253 print_fn = __atomic_load_n(&__libbpf_pr, __ATOMIC_RELAXED); 254 if (!print_fn) 255 return; 256 257 old_errno = errno; 258 259 va_start(args, format); 260 __libbpf_pr(level, format, args); 261 va_end(args); 262 263 errno = old_errno; 264 } 265 266 static void pr_perm_msg(int err) 267 { 268 struct rlimit limit; 269 char buf[100]; 270 271 if (err != -EPERM || geteuid() != 0) 272 return; 273 274 err = getrlimit(RLIMIT_MEMLOCK, &limit); 275 if (err) 276 return; 277 278 if (limit.rlim_cur == RLIM_INFINITY) 279 return; 280 281 if (limit.rlim_cur < 1024) 282 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); 283 else if (limit.rlim_cur < 1024*1024) 284 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); 285 else 286 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); 287 288 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", 289 buf); 290 } 291 292 #define STRERR_BUFSIZE 128 293 294 /* Copied from tools/perf/util/util.h */ 295 #ifndef zfree 296 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 297 #endif 298 299 #ifndef zclose 300 # define zclose(fd) ({ \ 301 int ___err = 0; \ 302 if ((fd) >= 0) \ 303 ___err = close((fd)); \ 304 fd = -1; \ 305 ___err; }) 306 #endif 307 308 static inline __u64 ptr_to_u64(const void *ptr) 309 { 310 return (__u64) (unsigned long) ptr; 311 } 312 313 int libbpf_set_strict_mode(enum libbpf_strict_mode mode) 314 { 315 /* as of v1.0 libbpf_set_strict_mode() is a no-op */ 316 return 0; 317 } 318 319 __u32 libbpf_major_version(void) 320 { 321 return LIBBPF_MAJOR_VERSION; 322 } 323 324 __u32 libbpf_minor_version(void) 325 { 326 return LIBBPF_MINOR_VERSION; 327 } 328 329 const char *libbpf_version_string(void) 330 { 331 #define __S(X) #X 332 #define _S(X) __S(X) 333 return "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION); 334 #undef _S 335 #undef __S 336 } 337 338 enum reloc_type { 339 RELO_LD64, 340 RELO_CALL, 341 RELO_DATA, 342 RELO_EXTERN_LD64, 343 RELO_EXTERN_CALL, 344 RELO_SUBPROG_ADDR, 345 RELO_CORE, 346 }; 347 348 struct reloc_desc { 349 enum reloc_type type; 350 int insn_idx; 351 union { 352 const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */ 353 struct { 354 int map_idx; 355 int sym_off; 356 int ext_idx; 357 }; 358 }; 359 }; 360 361 /* stored as sec_def->cookie for all libbpf-supported SEC()s */ 362 enum sec_def_flags { 363 SEC_NONE = 0, 364 /* expected_attach_type is optional, if kernel doesn't support that */ 365 SEC_EXP_ATTACH_OPT = 1, 366 /* legacy, only used by libbpf_get_type_names() and 367 * libbpf_attach_type_by_name(), not used by libbpf itself at all. 368 * This used to be associated with cgroup (and few other) BPF programs 369 * that were attachable through BPF_PROG_ATTACH command. Pretty 370 * meaningless nowadays, though. 371 */ 372 SEC_ATTACHABLE = 2, 373 SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT, 374 /* attachment target is specified through BTF ID in either kernel or 375 * other BPF program's BTF object 376 */ 377 SEC_ATTACH_BTF = 4, 378 /* BPF program type allows sleeping/blocking in kernel */ 379 SEC_SLEEPABLE = 8, 380 /* BPF program support non-linear XDP buffer */ 381 SEC_XDP_FRAGS = 16, 382 /* Setup proper attach type for usdt probes. */ 383 SEC_USDT = 32, 384 }; 385 386 struct bpf_sec_def { 387 char *sec; 388 enum bpf_prog_type prog_type; 389 enum bpf_attach_type expected_attach_type; 390 long cookie; 391 int handler_id; 392 393 libbpf_prog_setup_fn_t prog_setup_fn; 394 libbpf_prog_prepare_load_fn_t prog_prepare_load_fn; 395 libbpf_prog_attach_fn_t prog_attach_fn; 396 }; 397 398 /* 399 * bpf_prog should be a better name but it has been used in 400 * linux/filter.h. 401 */ 402 struct bpf_program { 403 char *name; 404 char *sec_name; 405 size_t sec_idx; 406 const struct bpf_sec_def *sec_def; 407 /* this program's instruction offset (in number of instructions) 408 * within its containing ELF section 409 */ 410 size_t sec_insn_off; 411 /* number of original instructions in ELF section belonging to this 412 * program, not taking into account subprogram instructions possible 413 * appended later during relocation 414 */ 415 size_t sec_insn_cnt; 416 /* Offset (in number of instructions) of the start of instruction 417 * belonging to this BPF program within its containing main BPF 418 * program. For the entry-point (main) BPF program, this is always 419 * zero. For a sub-program, this gets reset before each of main BPF 420 * programs are processed and relocated and is used to determined 421 * whether sub-program was already appended to the main program, and 422 * if yes, at which instruction offset. 423 */ 424 size_t sub_insn_off; 425 426 /* instructions that belong to BPF program; insns[0] is located at 427 * sec_insn_off instruction within its ELF section in ELF file, so 428 * when mapping ELF file instruction index to the local instruction, 429 * one needs to subtract sec_insn_off; and vice versa. 430 */ 431 struct bpf_insn *insns; 432 /* actual number of instruction in this BPF program's image; for 433 * entry-point BPF programs this includes the size of main program 434 * itself plus all the used sub-programs, appended at the end 435 */ 436 size_t insns_cnt; 437 438 struct reloc_desc *reloc_desc; 439 int nr_reloc; 440 441 /* BPF verifier log settings */ 442 char *log_buf; 443 size_t log_size; 444 __u32 log_level; 445 446 struct bpf_object *obj; 447 448 int fd; 449 bool autoload; 450 bool autoattach; 451 bool sym_global; 452 bool mark_btf_static; 453 enum bpf_prog_type type; 454 enum bpf_attach_type expected_attach_type; 455 int exception_cb_idx; 456 457 int prog_ifindex; 458 __u32 attach_btf_obj_fd; 459 __u32 attach_btf_id; 460 __u32 attach_prog_fd; 461 462 void *func_info; 463 __u32 func_info_rec_size; 464 __u32 func_info_cnt; 465 466 void *line_info; 467 __u32 line_info_rec_size; 468 __u32 line_info_cnt; 469 __u32 prog_flags; 470 }; 471 472 struct bpf_struct_ops { 473 const char *tname; 474 const struct btf_type *type; 475 struct bpf_program **progs; 476 __u32 *kern_func_off; 477 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ 478 void *data; 479 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in 480 * btf_vmlinux's format. 481 * struct bpf_struct_ops_tcp_congestion_ops { 482 * [... some other kernel fields ...] 483 * struct tcp_congestion_ops data; 484 * } 485 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) 486 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" 487 * from "data". 488 */ 489 void *kern_vdata; 490 __u32 type_id; 491 }; 492 493 #define DATA_SEC ".data" 494 #define BSS_SEC ".bss" 495 #define RODATA_SEC ".rodata" 496 #define KCONFIG_SEC ".kconfig" 497 #define KSYMS_SEC ".ksyms" 498 #define STRUCT_OPS_SEC ".struct_ops" 499 #define STRUCT_OPS_LINK_SEC ".struct_ops.link" 500 501 enum libbpf_map_type { 502 LIBBPF_MAP_UNSPEC, 503 LIBBPF_MAP_DATA, 504 LIBBPF_MAP_BSS, 505 LIBBPF_MAP_RODATA, 506 LIBBPF_MAP_KCONFIG, 507 }; 508 509 struct bpf_map_def { 510 unsigned int type; 511 unsigned int key_size; 512 unsigned int value_size; 513 unsigned int max_entries; 514 unsigned int map_flags; 515 }; 516 517 struct bpf_map { 518 struct bpf_object *obj; 519 char *name; 520 /* real_name is defined for special internal maps (.rodata*, 521 * .data*, .bss, .kconfig) and preserves their original ELF section 522 * name. This is important to be able to find corresponding BTF 523 * DATASEC information. 524 */ 525 char *real_name; 526 int fd; 527 int sec_idx; 528 size_t sec_offset; 529 int map_ifindex; 530 int inner_map_fd; 531 struct bpf_map_def def; 532 __u32 numa_node; 533 __u32 btf_var_idx; 534 int mod_btf_fd; 535 __u32 btf_key_type_id; 536 __u32 btf_value_type_id; 537 __u32 btf_vmlinux_value_type_id; 538 enum libbpf_map_type libbpf_type; 539 void *mmaped; 540 struct bpf_struct_ops *st_ops; 541 struct bpf_map *inner_map; 542 void **init_slots; 543 int init_slots_sz; 544 char *pin_path; 545 bool pinned; 546 bool reused; 547 bool autocreate; 548 __u64 map_extra; 549 }; 550 551 enum extern_type { 552 EXT_UNKNOWN, 553 EXT_KCFG, 554 EXT_KSYM, 555 }; 556 557 enum kcfg_type { 558 KCFG_UNKNOWN, 559 KCFG_CHAR, 560 KCFG_BOOL, 561 KCFG_INT, 562 KCFG_TRISTATE, 563 KCFG_CHAR_ARR, 564 }; 565 566 struct extern_desc { 567 enum extern_type type; 568 int sym_idx; 569 int btf_id; 570 int sec_btf_id; 571 const char *name; 572 char *essent_name; 573 bool is_set; 574 bool is_weak; 575 union { 576 struct { 577 enum kcfg_type type; 578 int sz; 579 int align; 580 int data_off; 581 bool is_signed; 582 } kcfg; 583 struct { 584 unsigned long long addr; 585 586 /* target btf_id of the corresponding kernel var. */ 587 int kernel_btf_obj_fd; 588 int kernel_btf_id; 589 590 /* local btf_id of the ksym extern's type. */ 591 __u32 type_id; 592 /* BTF fd index to be patched in for insn->off, this is 593 * 0 for vmlinux BTF, index in obj->fd_array for module 594 * BTF 595 */ 596 __s16 btf_fd_idx; 597 } ksym; 598 }; 599 }; 600 601 struct module_btf { 602 struct btf *btf; 603 char *name; 604 __u32 id; 605 int fd; 606 int fd_array_idx; 607 }; 608 609 enum sec_type { 610 SEC_UNUSED = 0, 611 SEC_RELO, 612 SEC_BSS, 613 SEC_DATA, 614 SEC_RODATA, 615 }; 616 617 struct elf_sec_desc { 618 enum sec_type sec_type; 619 Elf64_Shdr *shdr; 620 Elf_Data *data; 621 }; 622 623 struct elf_state { 624 int fd; 625 const void *obj_buf; 626 size_t obj_buf_sz; 627 Elf *elf; 628 Elf64_Ehdr *ehdr; 629 Elf_Data *symbols; 630 Elf_Data *st_ops_data; 631 Elf_Data *st_ops_link_data; 632 size_t shstrndx; /* section index for section name strings */ 633 size_t strtabidx; 634 struct elf_sec_desc *secs; 635 size_t sec_cnt; 636 int btf_maps_shndx; 637 __u32 btf_maps_sec_btf_id; 638 int text_shndx; 639 int symbols_shndx; 640 int st_ops_shndx; 641 int st_ops_link_shndx; 642 }; 643 644 struct usdt_manager; 645 646 struct bpf_object { 647 char name[BPF_OBJ_NAME_LEN]; 648 char license[64]; 649 __u32 kern_version; 650 651 struct bpf_program *programs; 652 size_t nr_programs; 653 struct bpf_map *maps; 654 size_t nr_maps; 655 size_t maps_cap; 656 657 char *kconfig; 658 struct extern_desc *externs; 659 int nr_extern; 660 int kconfig_map_idx; 661 662 bool loaded; 663 bool has_subcalls; 664 bool has_rodata; 665 666 struct bpf_gen *gen_loader; 667 668 /* Information when doing ELF related work. Only valid if efile.elf is not NULL */ 669 struct elf_state efile; 670 671 struct btf *btf; 672 struct btf_ext *btf_ext; 673 674 /* Parse and load BTF vmlinux if any of the programs in the object need 675 * it at load time. 676 */ 677 struct btf *btf_vmlinux; 678 /* Path to the custom BTF to be used for BPF CO-RE relocations as an 679 * override for vmlinux BTF. 680 */ 681 char *btf_custom_path; 682 /* vmlinux BTF override for CO-RE relocations */ 683 struct btf *btf_vmlinux_override; 684 /* Lazily initialized kernel module BTFs */ 685 struct module_btf *btf_modules; 686 bool btf_modules_loaded; 687 size_t btf_module_cnt; 688 size_t btf_module_cap; 689 690 /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */ 691 char *log_buf; 692 size_t log_size; 693 __u32 log_level; 694 695 int *fd_array; 696 size_t fd_array_cap; 697 size_t fd_array_cnt; 698 699 struct usdt_manager *usdt_man; 700 701 struct kern_feature_cache *feat_cache; 702 char *token_path; 703 int token_fd; 704 705 char path[]; 706 }; 707 708 static const char *elf_sym_str(const struct bpf_object *obj, size_t off); 709 static const char *elf_sec_str(const struct bpf_object *obj, size_t off); 710 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx); 711 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name); 712 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn); 713 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn); 714 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn); 715 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx); 716 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx); 717 718 void bpf_program__unload(struct bpf_program *prog) 719 { 720 if (!prog) 721 return; 722 723 zclose(prog->fd); 724 725 zfree(&prog->func_info); 726 zfree(&prog->line_info); 727 } 728 729 static void bpf_program__exit(struct bpf_program *prog) 730 { 731 if (!prog) 732 return; 733 734 bpf_program__unload(prog); 735 zfree(&prog->name); 736 zfree(&prog->sec_name); 737 zfree(&prog->insns); 738 zfree(&prog->reloc_desc); 739 740 prog->nr_reloc = 0; 741 prog->insns_cnt = 0; 742 prog->sec_idx = -1; 743 } 744 745 static bool insn_is_subprog_call(const struct bpf_insn *insn) 746 { 747 return BPF_CLASS(insn->code) == BPF_JMP && 748 BPF_OP(insn->code) == BPF_CALL && 749 BPF_SRC(insn->code) == BPF_K && 750 insn->src_reg == BPF_PSEUDO_CALL && 751 insn->dst_reg == 0 && 752 insn->off == 0; 753 } 754 755 static bool is_call_insn(const struct bpf_insn *insn) 756 { 757 return insn->code == (BPF_JMP | BPF_CALL); 758 } 759 760 static bool insn_is_pseudo_func(struct bpf_insn *insn) 761 { 762 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; 763 } 764 765 static int 766 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, 767 const char *name, size_t sec_idx, const char *sec_name, 768 size_t sec_off, void *insn_data, size_t insn_data_sz) 769 { 770 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) { 771 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n", 772 sec_name, name, sec_off, insn_data_sz); 773 return -EINVAL; 774 } 775 776 memset(prog, 0, sizeof(*prog)); 777 prog->obj = obj; 778 779 prog->sec_idx = sec_idx; 780 prog->sec_insn_off = sec_off / BPF_INSN_SZ; 781 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; 782 /* insns_cnt can later be increased by appending used subprograms */ 783 prog->insns_cnt = prog->sec_insn_cnt; 784 785 prog->type = BPF_PROG_TYPE_UNSPEC; 786 prog->fd = -1; 787 prog->exception_cb_idx = -1; 788 789 /* libbpf's convention for SEC("?abc...") is that it's just like 790 * SEC("abc...") but the corresponding bpf_program starts out with 791 * autoload set to false. 792 */ 793 if (sec_name[0] == '?') { 794 prog->autoload = false; 795 /* from now on forget there was ? in section name */ 796 sec_name++; 797 } else { 798 prog->autoload = true; 799 } 800 801 prog->autoattach = true; 802 803 /* inherit object's log_level */ 804 prog->log_level = obj->log_level; 805 806 prog->sec_name = strdup(sec_name); 807 if (!prog->sec_name) 808 goto errout; 809 810 prog->name = strdup(name); 811 if (!prog->name) 812 goto errout; 813 814 prog->insns = malloc(insn_data_sz); 815 if (!prog->insns) 816 goto errout; 817 memcpy(prog->insns, insn_data, insn_data_sz); 818 819 return 0; 820 errout: 821 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name); 822 bpf_program__exit(prog); 823 return -ENOMEM; 824 } 825 826 static int 827 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, 828 const char *sec_name, int sec_idx) 829 { 830 Elf_Data *symbols = obj->efile.symbols; 831 struct bpf_program *prog, *progs; 832 void *data = sec_data->d_buf; 833 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; 834 int nr_progs, err, i; 835 const char *name; 836 Elf64_Sym *sym; 837 838 progs = obj->programs; 839 nr_progs = obj->nr_programs; 840 nr_syms = symbols->d_size / sizeof(Elf64_Sym); 841 842 for (i = 0; i < nr_syms; i++) { 843 sym = elf_sym_by_idx(obj, i); 844 845 if (sym->st_shndx != sec_idx) 846 continue; 847 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) 848 continue; 849 850 prog_sz = sym->st_size; 851 sec_off = sym->st_value; 852 853 name = elf_sym_str(obj, sym->st_name); 854 if (!name) { 855 pr_warn("sec '%s': failed to get symbol name for offset %zu\n", 856 sec_name, sec_off); 857 return -LIBBPF_ERRNO__FORMAT; 858 } 859 860 if (sec_off + prog_sz > sec_sz) { 861 pr_warn("sec '%s': program at offset %zu crosses section boundary\n", 862 sec_name, sec_off); 863 return -LIBBPF_ERRNO__FORMAT; 864 } 865 866 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { 867 pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name); 868 return -ENOTSUP; 869 } 870 871 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n", 872 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz); 873 874 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs)); 875 if (!progs) { 876 /* 877 * In this case the original obj->programs 878 * is still valid, so don't need special treat for 879 * bpf_close_object(). 880 */ 881 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n", 882 sec_name, name); 883 return -ENOMEM; 884 } 885 obj->programs = progs; 886 887 prog = &progs[nr_progs]; 888 889 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, 890 sec_off, data + sec_off, prog_sz); 891 if (err) 892 return err; 893 894 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL) 895 prog->sym_global = true; 896 897 /* if function is a global/weak symbol, but has restricted 898 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC 899 * as static to enable more permissive BPF verification mode 900 * with more outside context available to BPF verifier 901 */ 902 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN 903 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) 904 prog->mark_btf_static = true; 905 906 nr_progs++; 907 obj->nr_programs = nr_progs; 908 } 909 910 return 0; 911 } 912 913 static const struct btf_member * 914 find_member_by_offset(const struct btf_type *t, __u32 bit_offset) 915 { 916 struct btf_member *m; 917 int i; 918 919 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 920 if (btf_member_bit_offset(t, i) == bit_offset) 921 return m; 922 } 923 924 return NULL; 925 } 926 927 static const struct btf_member * 928 find_member_by_name(const struct btf *btf, const struct btf_type *t, 929 const char *name) 930 { 931 struct btf_member *m; 932 int i; 933 934 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 935 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) 936 return m; 937 } 938 939 return NULL; 940 } 941 942 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, 943 __u16 kind, struct btf **res_btf, 944 struct module_btf **res_mod_btf); 945 946 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" 947 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 948 const char *name, __u32 kind); 949 950 static int 951 find_struct_ops_kern_types(struct bpf_object *obj, const char *tname, 952 struct module_btf **mod_btf, 953 const struct btf_type **type, __u32 *type_id, 954 const struct btf_type **vtype, __u32 *vtype_id, 955 const struct btf_member **data_member) 956 { 957 const struct btf_type *kern_type, *kern_vtype; 958 const struct btf_member *kern_data_member; 959 struct btf *btf; 960 __s32 kern_vtype_id, kern_type_id; 961 __u32 i; 962 963 kern_type_id = find_ksym_btf_id(obj, tname, BTF_KIND_STRUCT, 964 &btf, mod_btf); 965 if (kern_type_id < 0) { 966 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", 967 tname); 968 return kern_type_id; 969 } 970 kern_type = btf__type_by_id(btf, kern_type_id); 971 972 /* Find the corresponding "map_value" type that will be used 973 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example, 974 * find "struct bpf_struct_ops_tcp_congestion_ops" from the 975 * btf_vmlinux. 976 */ 977 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX, 978 tname, BTF_KIND_STRUCT); 979 if (kern_vtype_id < 0) { 980 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n", 981 STRUCT_OPS_VALUE_PREFIX, tname); 982 return kern_vtype_id; 983 } 984 kern_vtype = btf__type_by_id(btf, kern_vtype_id); 985 986 /* Find "struct tcp_congestion_ops" from 987 * struct bpf_struct_ops_tcp_congestion_ops { 988 * [ ... ] 989 * struct tcp_congestion_ops data; 990 * } 991 */ 992 kern_data_member = btf_members(kern_vtype); 993 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { 994 if (kern_data_member->type == kern_type_id) 995 break; 996 } 997 if (i == btf_vlen(kern_vtype)) { 998 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n", 999 tname, STRUCT_OPS_VALUE_PREFIX, tname); 1000 return -EINVAL; 1001 } 1002 1003 *type = kern_type; 1004 *type_id = kern_type_id; 1005 *vtype = kern_vtype; 1006 *vtype_id = kern_vtype_id; 1007 *data_member = kern_data_member; 1008 1009 return 0; 1010 } 1011 1012 static bool bpf_map__is_struct_ops(const struct bpf_map *map) 1013 { 1014 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; 1015 } 1016 1017 static bool is_valid_st_ops_program(struct bpf_object *obj, 1018 const struct bpf_program *prog) 1019 { 1020 int i; 1021 1022 for (i = 0; i < obj->nr_programs; i++) { 1023 if (&obj->programs[i] == prog) 1024 return prog->type == BPF_PROG_TYPE_STRUCT_OPS; 1025 } 1026 1027 return false; 1028 } 1029 1030 /* Init the map's fields that depend on kern_btf */ 1031 static int bpf_map__init_kern_struct_ops(struct bpf_map *map) 1032 { 1033 const struct btf_member *member, *kern_member, *kern_data_member; 1034 const struct btf_type *type, *kern_type, *kern_vtype; 1035 __u32 i, kern_type_id, kern_vtype_id, kern_data_off; 1036 struct bpf_object *obj = map->obj; 1037 const struct btf *btf = obj->btf; 1038 struct bpf_struct_ops *st_ops; 1039 const struct btf *kern_btf; 1040 struct module_btf *mod_btf; 1041 void *data, *kern_data; 1042 const char *tname; 1043 int err; 1044 1045 st_ops = map->st_ops; 1046 type = st_ops->type; 1047 tname = st_ops->tname; 1048 err = find_struct_ops_kern_types(obj, tname, &mod_btf, 1049 &kern_type, &kern_type_id, 1050 &kern_vtype, &kern_vtype_id, 1051 &kern_data_member); 1052 if (err) 1053 return err; 1054 1055 kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux; 1056 1057 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", 1058 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); 1059 1060 map->mod_btf_fd = mod_btf ? mod_btf->fd : -1; 1061 map->def.value_size = kern_vtype->size; 1062 map->btf_vmlinux_value_type_id = kern_vtype_id; 1063 1064 st_ops->kern_vdata = calloc(1, kern_vtype->size); 1065 if (!st_ops->kern_vdata) 1066 return -ENOMEM; 1067 1068 data = st_ops->data; 1069 kern_data_off = kern_data_member->offset / 8; 1070 kern_data = st_ops->kern_vdata + kern_data_off; 1071 1072 member = btf_members(type); 1073 for (i = 0; i < btf_vlen(type); i++, member++) { 1074 const struct btf_type *mtype, *kern_mtype; 1075 __u32 mtype_id, kern_mtype_id; 1076 void *mdata, *kern_mdata; 1077 __s64 msize, kern_msize; 1078 __u32 moff, kern_moff; 1079 __u32 kern_member_idx; 1080 const char *mname; 1081 1082 mname = btf__name_by_offset(btf, member->name_off); 1083 kern_member = find_member_by_name(kern_btf, kern_type, mname); 1084 if (!kern_member) { 1085 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", 1086 map->name, mname); 1087 return -ENOTSUP; 1088 } 1089 1090 kern_member_idx = kern_member - btf_members(kern_type); 1091 if (btf_member_bitfield_size(type, i) || 1092 btf_member_bitfield_size(kern_type, kern_member_idx)) { 1093 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", 1094 map->name, mname); 1095 return -ENOTSUP; 1096 } 1097 1098 moff = member->offset / 8; 1099 kern_moff = kern_member->offset / 8; 1100 1101 mdata = data + moff; 1102 kern_mdata = kern_data + kern_moff; 1103 1104 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); 1105 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, 1106 &kern_mtype_id); 1107 if (BTF_INFO_KIND(mtype->info) != 1108 BTF_INFO_KIND(kern_mtype->info)) { 1109 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", 1110 map->name, mname, BTF_INFO_KIND(mtype->info), 1111 BTF_INFO_KIND(kern_mtype->info)); 1112 return -ENOTSUP; 1113 } 1114 1115 if (btf_is_ptr(mtype)) { 1116 struct bpf_program *prog; 1117 1118 /* Update the value from the shadow type */ 1119 prog = *(void **)mdata; 1120 st_ops->progs[i] = prog; 1121 if (!prog) 1122 continue; 1123 if (!is_valid_st_ops_program(obj, prog)) { 1124 pr_warn("struct_ops init_kern %s: member %s is not a struct_ops program\n", 1125 map->name, mname); 1126 return -ENOTSUP; 1127 } 1128 1129 kern_mtype = skip_mods_and_typedefs(kern_btf, 1130 kern_mtype->type, 1131 &kern_mtype_id); 1132 1133 /* mtype->type must be a func_proto which was 1134 * guaranteed in bpf_object__collect_st_ops_relos(), 1135 * so only check kern_mtype for func_proto here. 1136 */ 1137 if (!btf_is_func_proto(kern_mtype)) { 1138 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n", 1139 map->name, mname); 1140 return -ENOTSUP; 1141 } 1142 1143 if (mod_btf) 1144 prog->attach_btf_obj_fd = mod_btf->fd; 1145 prog->attach_btf_id = kern_type_id; 1146 prog->expected_attach_type = kern_member_idx; 1147 1148 st_ops->kern_func_off[i] = kern_data_off + kern_moff; 1149 1150 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", 1151 map->name, mname, prog->name, moff, 1152 kern_moff); 1153 1154 continue; 1155 } 1156 1157 msize = btf__resolve_size(btf, mtype_id); 1158 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); 1159 if (msize < 0 || kern_msize < 0 || msize != kern_msize) { 1160 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", 1161 map->name, mname, (ssize_t)msize, 1162 (ssize_t)kern_msize); 1163 return -ENOTSUP; 1164 } 1165 1166 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", 1167 map->name, mname, (unsigned int)msize, 1168 moff, kern_moff); 1169 memcpy(kern_mdata, mdata, msize); 1170 } 1171 1172 return 0; 1173 } 1174 1175 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) 1176 { 1177 struct bpf_map *map; 1178 size_t i; 1179 int err; 1180 1181 for (i = 0; i < obj->nr_maps; i++) { 1182 map = &obj->maps[i]; 1183 1184 if (!bpf_map__is_struct_ops(map)) 1185 continue; 1186 1187 err = bpf_map__init_kern_struct_ops(map); 1188 if (err) 1189 return err; 1190 } 1191 1192 return 0; 1193 } 1194 1195 static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name, 1196 int shndx, Elf_Data *data, __u32 map_flags) 1197 { 1198 const struct btf_type *type, *datasec; 1199 const struct btf_var_secinfo *vsi; 1200 struct bpf_struct_ops *st_ops; 1201 const char *tname, *var_name; 1202 __s32 type_id, datasec_id; 1203 const struct btf *btf; 1204 struct bpf_map *map; 1205 __u32 i; 1206 1207 if (shndx == -1) 1208 return 0; 1209 1210 btf = obj->btf; 1211 datasec_id = btf__find_by_name_kind(btf, sec_name, 1212 BTF_KIND_DATASEC); 1213 if (datasec_id < 0) { 1214 pr_warn("struct_ops init: DATASEC %s not found\n", 1215 sec_name); 1216 return -EINVAL; 1217 } 1218 1219 datasec = btf__type_by_id(btf, datasec_id); 1220 vsi = btf_var_secinfos(datasec); 1221 for (i = 0; i < btf_vlen(datasec); i++, vsi++) { 1222 type = btf__type_by_id(obj->btf, vsi->type); 1223 var_name = btf__name_by_offset(obj->btf, type->name_off); 1224 1225 type_id = btf__resolve_type(obj->btf, vsi->type); 1226 if (type_id < 0) { 1227 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", 1228 vsi->type, sec_name); 1229 return -EINVAL; 1230 } 1231 1232 type = btf__type_by_id(obj->btf, type_id); 1233 tname = btf__name_by_offset(obj->btf, type->name_off); 1234 if (!tname[0]) { 1235 pr_warn("struct_ops init: anonymous type is not supported\n"); 1236 return -ENOTSUP; 1237 } 1238 if (!btf_is_struct(type)) { 1239 pr_warn("struct_ops init: %s is not a struct\n", tname); 1240 return -EINVAL; 1241 } 1242 1243 map = bpf_object__add_map(obj); 1244 if (IS_ERR(map)) 1245 return PTR_ERR(map); 1246 1247 map->sec_idx = shndx; 1248 map->sec_offset = vsi->offset; 1249 map->name = strdup(var_name); 1250 if (!map->name) 1251 return -ENOMEM; 1252 map->btf_value_type_id = type_id; 1253 1254 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; 1255 map->def.key_size = sizeof(int); 1256 map->def.value_size = type->size; 1257 map->def.max_entries = 1; 1258 map->def.map_flags = map_flags; 1259 1260 map->st_ops = calloc(1, sizeof(*map->st_ops)); 1261 if (!map->st_ops) 1262 return -ENOMEM; 1263 st_ops = map->st_ops; 1264 st_ops->data = malloc(type->size); 1265 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); 1266 st_ops->kern_func_off = malloc(btf_vlen(type) * 1267 sizeof(*st_ops->kern_func_off)); 1268 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) 1269 return -ENOMEM; 1270 1271 if (vsi->offset + type->size > data->d_size) { 1272 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", 1273 var_name, sec_name); 1274 return -EINVAL; 1275 } 1276 1277 memcpy(st_ops->data, 1278 data->d_buf + vsi->offset, 1279 type->size); 1280 st_ops->tname = tname; 1281 st_ops->type = type; 1282 st_ops->type_id = type_id; 1283 1284 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", 1285 tname, type_id, var_name, vsi->offset); 1286 } 1287 1288 return 0; 1289 } 1290 1291 static int bpf_object_init_struct_ops(struct bpf_object *obj) 1292 { 1293 int err; 1294 1295 err = init_struct_ops_maps(obj, STRUCT_OPS_SEC, obj->efile.st_ops_shndx, 1296 obj->efile.st_ops_data, 0); 1297 err = err ?: init_struct_ops_maps(obj, STRUCT_OPS_LINK_SEC, 1298 obj->efile.st_ops_link_shndx, 1299 obj->efile.st_ops_link_data, 1300 BPF_F_LINK); 1301 return err; 1302 } 1303 1304 static struct bpf_object *bpf_object__new(const char *path, 1305 const void *obj_buf, 1306 size_t obj_buf_sz, 1307 const char *obj_name) 1308 { 1309 struct bpf_object *obj; 1310 char *end; 1311 1312 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 1313 if (!obj) { 1314 pr_warn("alloc memory failed for %s\n", path); 1315 return ERR_PTR(-ENOMEM); 1316 } 1317 1318 strcpy(obj->path, path); 1319 if (obj_name) { 1320 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); 1321 } else { 1322 /* Using basename() GNU version which doesn't modify arg. */ 1323 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); 1324 end = strchr(obj->name, '.'); 1325 if (end) 1326 *end = 0; 1327 } 1328 1329 obj->efile.fd = -1; 1330 /* 1331 * Caller of this function should also call 1332 * bpf_object__elf_finish() after data collection to return 1333 * obj_buf to user. If not, we should duplicate the buffer to 1334 * avoid user freeing them before elf finish. 1335 */ 1336 obj->efile.obj_buf = obj_buf; 1337 obj->efile.obj_buf_sz = obj_buf_sz; 1338 obj->efile.btf_maps_shndx = -1; 1339 obj->efile.st_ops_shndx = -1; 1340 obj->efile.st_ops_link_shndx = -1; 1341 obj->kconfig_map_idx = -1; 1342 1343 obj->kern_version = get_kernel_version(); 1344 obj->loaded = false; 1345 1346 return obj; 1347 } 1348 1349 static void bpf_object__elf_finish(struct bpf_object *obj) 1350 { 1351 if (!obj->efile.elf) 1352 return; 1353 1354 elf_end(obj->efile.elf); 1355 obj->efile.elf = NULL; 1356 obj->efile.symbols = NULL; 1357 obj->efile.st_ops_data = NULL; 1358 obj->efile.st_ops_link_data = NULL; 1359 1360 zfree(&obj->efile.secs); 1361 obj->efile.sec_cnt = 0; 1362 zclose(obj->efile.fd); 1363 obj->efile.obj_buf = NULL; 1364 obj->efile.obj_buf_sz = 0; 1365 } 1366 1367 static int bpf_object__elf_init(struct bpf_object *obj) 1368 { 1369 Elf64_Ehdr *ehdr; 1370 int err = 0; 1371 Elf *elf; 1372 1373 if (obj->efile.elf) { 1374 pr_warn("elf: init internal error\n"); 1375 return -LIBBPF_ERRNO__LIBELF; 1376 } 1377 1378 if (obj->efile.obj_buf_sz > 0) { 1379 /* obj_buf should have been validated by bpf_object__open_mem(). */ 1380 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz); 1381 } else { 1382 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC); 1383 if (obj->efile.fd < 0) { 1384 char errmsg[STRERR_BUFSIZE], *cp; 1385 1386 err = -errno; 1387 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 1388 pr_warn("elf: failed to open %s: %s\n", obj->path, cp); 1389 return err; 1390 } 1391 1392 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); 1393 } 1394 1395 if (!elf) { 1396 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); 1397 err = -LIBBPF_ERRNO__LIBELF; 1398 goto errout; 1399 } 1400 1401 obj->efile.elf = elf; 1402 1403 if (elf_kind(elf) != ELF_K_ELF) { 1404 err = -LIBBPF_ERRNO__FORMAT; 1405 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path); 1406 goto errout; 1407 } 1408 1409 if (gelf_getclass(elf) != ELFCLASS64) { 1410 err = -LIBBPF_ERRNO__FORMAT; 1411 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path); 1412 goto errout; 1413 } 1414 1415 obj->efile.ehdr = ehdr = elf64_getehdr(elf); 1416 if (!obj->efile.ehdr) { 1417 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); 1418 err = -LIBBPF_ERRNO__FORMAT; 1419 goto errout; 1420 } 1421 1422 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) { 1423 pr_warn("elf: failed to get section names section index for %s: %s\n", 1424 obj->path, elf_errmsg(-1)); 1425 err = -LIBBPF_ERRNO__FORMAT; 1426 goto errout; 1427 } 1428 1429 /* ELF is corrupted/truncated, avoid calling elf_strptr. */ 1430 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) { 1431 pr_warn("elf: failed to get section names strings from %s: %s\n", 1432 obj->path, elf_errmsg(-1)); 1433 err = -LIBBPF_ERRNO__FORMAT; 1434 goto errout; 1435 } 1436 1437 /* Old LLVM set e_machine to EM_NONE */ 1438 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) { 1439 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); 1440 err = -LIBBPF_ERRNO__FORMAT; 1441 goto errout; 1442 } 1443 1444 return 0; 1445 errout: 1446 bpf_object__elf_finish(obj); 1447 return err; 1448 } 1449 1450 static int bpf_object__check_endianness(struct bpf_object *obj) 1451 { 1452 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1453 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB) 1454 return 0; 1455 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 1456 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB) 1457 return 0; 1458 #else 1459 # error "Unrecognized __BYTE_ORDER__" 1460 #endif 1461 pr_warn("elf: endianness mismatch in %s.\n", obj->path); 1462 return -LIBBPF_ERRNO__ENDIAN; 1463 } 1464 1465 static int 1466 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 1467 { 1468 if (!data) { 1469 pr_warn("invalid license section in %s\n", obj->path); 1470 return -LIBBPF_ERRNO__FORMAT; 1471 } 1472 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't 1473 * go over allowed ELF data section buffer 1474 */ 1475 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license))); 1476 pr_debug("license of %s is %s\n", obj->path, obj->license); 1477 return 0; 1478 } 1479 1480 static int 1481 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 1482 { 1483 __u32 kver; 1484 1485 if (!data || size != sizeof(kver)) { 1486 pr_warn("invalid kver section in %s\n", obj->path); 1487 return -LIBBPF_ERRNO__FORMAT; 1488 } 1489 memcpy(&kver, data, sizeof(kver)); 1490 obj->kern_version = kver; 1491 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 1492 return 0; 1493 } 1494 1495 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 1496 { 1497 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 1498 type == BPF_MAP_TYPE_HASH_OF_MAPS) 1499 return true; 1500 return false; 1501 } 1502 1503 static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size) 1504 { 1505 Elf_Data *data; 1506 Elf_Scn *scn; 1507 1508 if (!name) 1509 return -EINVAL; 1510 1511 scn = elf_sec_by_name(obj, name); 1512 data = elf_sec_data(obj, scn); 1513 if (data) { 1514 *size = data->d_size; 1515 return 0; /* found it */ 1516 } 1517 1518 return -ENOENT; 1519 } 1520 1521 static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name) 1522 { 1523 Elf_Data *symbols = obj->efile.symbols; 1524 const char *sname; 1525 size_t si; 1526 1527 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) { 1528 Elf64_Sym *sym = elf_sym_by_idx(obj, si); 1529 1530 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT) 1531 continue; 1532 1533 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL && 1534 ELF64_ST_BIND(sym->st_info) != STB_WEAK) 1535 continue; 1536 1537 sname = elf_sym_str(obj, sym->st_name); 1538 if (!sname) { 1539 pr_warn("failed to get sym name string for var %s\n", name); 1540 return ERR_PTR(-EIO); 1541 } 1542 if (strcmp(name, sname) == 0) 1543 return sym; 1544 } 1545 1546 return ERR_PTR(-ENOENT); 1547 } 1548 1549 /* Some versions of Android don't provide memfd_create() in their libc 1550 * implementation, so avoid complications and just go straight to Linux 1551 * syscall. 1552 */ 1553 static int sys_memfd_create(const char *name, unsigned flags) 1554 { 1555 return syscall(__NR_memfd_create, name, flags); 1556 } 1557 1558 static int create_placeholder_fd(void) 1559 { 1560 int fd; 1561 1562 fd = ensure_good_fd(sys_memfd_create("libbpf-placeholder-fd", MFD_CLOEXEC)); 1563 if (fd < 0) 1564 return -errno; 1565 return fd; 1566 } 1567 1568 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 1569 { 1570 struct bpf_map *map; 1571 int err; 1572 1573 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap, 1574 sizeof(*obj->maps), obj->nr_maps + 1); 1575 if (err) 1576 return ERR_PTR(err); 1577 1578 map = &obj->maps[obj->nr_maps++]; 1579 map->obj = obj; 1580 /* Preallocate map FD without actually creating BPF map just yet. 1581 * These map FD "placeholders" will be reused later without changing 1582 * FD value when map is actually created in the kernel. 1583 * 1584 * This is useful to be able to perform BPF program relocations 1585 * without having to create BPF maps before that step. This allows us 1586 * to finalize and load BTF very late in BPF object's loading phase, 1587 * right before BPF maps have to be created and BPF programs have to 1588 * be loaded. By having these map FD placeholders we can perform all 1589 * the sanitizations, relocations, and any other adjustments before we 1590 * start creating actual BPF kernel objects (BTF, maps, progs). 1591 */ 1592 map->fd = create_placeholder_fd(); 1593 if (map->fd < 0) 1594 return ERR_PTR(map->fd); 1595 map->inner_map_fd = -1; 1596 map->autocreate = true; 1597 1598 return map; 1599 } 1600 1601 static size_t bpf_map_mmap_sz(unsigned int value_sz, unsigned int max_entries) 1602 { 1603 const long page_sz = sysconf(_SC_PAGE_SIZE); 1604 size_t map_sz; 1605 1606 map_sz = (size_t)roundup(value_sz, 8) * max_entries; 1607 map_sz = roundup(map_sz, page_sz); 1608 return map_sz; 1609 } 1610 1611 static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz) 1612 { 1613 void *mmaped; 1614 1615 if (!map->mmaped) 1616 return -EINVAL; 1617 1618 if (old_sz == new_sz) 1619 return 0; 1620 1621 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1622 if (mmaped == MAP_FAILED) 1623 return -errno; 1624 1625 memcpy(mmaped, map->mmaped, min(old_sz, new_sz)); 1626 munmap(map->mmaped, old_sz); 1627 map->mmaped = mmaped; 1628 return 0; 1629 } 1630 1631 static char *internal_map_name(struct bpf_object *obj, const char *real_name) 1632 { 1633 char map_name[BPF_OBJ_NAME_LEN], *p; 1634 int pfx_len, sfx_len = max((size_t)7, strlen(real_name)); 1635 1636 /* This is one of the more confusing parts of libbpf for various 1637 * reasons, some of which are historical. The original idea for naming 1638 * internal names was to include as much of BPF object name prefix as 1639 * possible, so that it can be distinguished from similar internal 1640 * maps of a different BPF object. 1641 * As an example, let's say we have bpf_object named 'my_object_name' 1642 * and internal map corresponding to '.rodata' ELF section. The final 1643 * map name advertised to user and to the kernel will be 1644 * 'my_objec.rodata', taking first 8 characters of object name and 1645 * entire 7 characters of '.rodata'. 1646 * Somewhat confusingly, if internal map ELF section name is shorter 1647 * than 7 characters, e.g., '.bss', we still reserve 7 characters 1648 * for the suffix, even though we only have 4 actual characters, and 1649 * resulting map will be called 'my_objec.bss', not even using all 15 1650 * characters allowed by the kernel. Oh well, at least the truncated 1651 * object name is somewhat consistent in this case. But if the map 1652 * name is '.kconfig', we'll still have entirety of '.kconfig' added 1653 * (8 chars) and thus will be left with only first 7 characters of the 1654 * object name ('my_obje'). Happy guessing, user, that the final map 1655 * name will be "my_obje.kconfig". 1656 * Now, with libbpf starting to support arbitrarily named .rodata.* 1657 * and .data.* data sections, it's possible that ELF section name is 1658 * longer than allowed 15 chars, so we now need to be careful to take 1659 * only up to 15 first characters of ELF name, taking no BPF object 1660 * name characters at all. So '.rodata.abracadabra' will result in 1661 * '.rodata.abracad' kernel and user-visible name. 1662 * We need to keep this convoluted logic intact for .data, .bss and 1663 * .rodata maps, but for new custom .data.custom and .rodata.custom 1664 * maps we use their ELF names as is, not prepending bpf_object name 1665 * in front. We still need to truncate them to 15 characters for the 1666 * kernel. Full name can be recovered for such maps by using DATASEC 1667 * BTF type associated with such map's value type, though. 1668 */ 1669 if (sfx_len >= BPF_OBJ_NAME_LEN) 1670 sfx_len = BPF_OBJ_NAME_LEN - 1; 1671 1672 /* if there are two or more dots in map name, it's a custom dot map */ 1673 if (strchr(real_name + 1, '.') != NULL) 1674 pfx_len = 0; 1675 else 1676 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name)); 1677 1678 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, 1679 sfx_len, real_name); 1680 1681 /* sanitise map name to characters allowed by kernel */ 1682 for (p = map_name; *p && p < map_name + sizeof(map_name); p++) 1683 if (!isalnum(*p) && *p != '_' && *p != '.') 1684 *p = '_'; 1685 1686 return strdup(map_name); 1687 } 1688 1689 static int 1690 map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map); 1691 1692 /* Internal BPF map is mmap()'able only if at least one of corresponding 1693 * DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL 1694 * variable and it's not marked as __hidden (which turns it into, effectively, 1695 * a STATIC variable). 1696 */ 1697 static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map) 1698 { 1699 const struct btf_type *t, *vt; 1700 struct btf_var_secinfo *vsi; 1701 int i, n; 1702 1703 if (!map->btf_value_type_id) 1704 return false; 1705 1706 t = btf__type_by_id(obj->btf, map->btf_value_type_id); 1707 if (!btf_is_datasec(t)) 1708 return false; 1709 1710 vsi = btf_var_secinfos(t); 1711 for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) { 1712 vt = btf__type_by_id(obj->btf, vsi->type); 1713 if (!btf_is_var(vt)) 1714 continue; 1715 1716 if (btf_var(vt)->linkage != BTF_VAR_STATIC) 1717 return true; 1718 } 1719 1720 return false; 1721 } 1722 1723 static int 1724 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 1725 const char *real_name, int sec_idx, void *data, size_t data_sz) 1726 { 1727 struct bpf_map_def *def; 1728 struct bpf_map *map; 1729 size_t mmap_sz; 1730 int err; 1731 1732 map = bpf_object__add_map(obj); 1733 if (IS_ERR(map)) 1734 return PTR_ERR(map); 1735 1736 map->libbpf_type = type; 1737 map->sec_idx = sec_idx; 1738 map->sec_offset = 0; 1739 map->real_name = strdup(real_name); 1740 map->name = internal_map_name(obj, real_name); 1741 if (!map->real_name || !map->name) { 1742 zfree(&map->real_name); 1743 zfree(&map->name); 1744 return -ENOMEM; 1745 } 1746 1747 def = &map->def; 1748 def->type = BPF_MAP_TYPE_ARRAY; 1749 def->key_size = sizeof(int); 1750 def->value_size = data_sz; 1751 def->max_entries = 1; 1752 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG 1753 ? BPF_F_RDONLY_PROG : 0; 1754 1755 /* failures are fine because of maps like .rodata.str1.1 */ 1756 (void) map_fill_btf_type_info(obj, map); 1757 1758 if (map_is_mmapable(obj, map)) 1759 def->map_flags |= BPF_F_MMAPABLE; 1760 1761 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", 1762 map->name, map->sec_idx, map->sec_offset, def->map_flags); 1763 1764 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 1765 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 1766 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1767 if (map->mmaped == MAP_FAILED) { 1768 err = -errno; 1769 map->mmaped = NULL; 1770 pr_warn("failed to alloc map '%s' content buffer: %d\n", 1771 map->name, err); 1772 zfree(&map->real_name); 1773 zfree(&map->name); 1774 return err; 1775 } 1776 1777 if (data) 1778 memcpy(map->mmaped, data, data_sz); 1779 1780 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 1781 return 0; 1782 } 1783 1784 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 1785 { 1786 struct elf_sec_desc *sec_desc; 1787 const char *sec_name; 1788 int err = 0, sec_idx; 1789 1790 /* 1791 * Populate obj->maps with libbpf internal maps. 1792 */ 1793 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) { 1794 sec_desc = &obj->efile.secs[sec_idx]; 1795 1796 /* Skip recognized sections with size 0. */ 1797 if (!sec_desc->data || sec_desc->data->d_size == 0) 1798 continue; 1799 1800 switch (sec_desc->sec_type) { 1801 case SEC_DATA: 1802 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 1803 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 1804 sec_name, sec_idx, 1805 sec_desc->data->d_buf, 1806 sec_desc->data->d_size); 1807 break; 1808 case SEC_RODATA: 1809 obj->has_rodata = true; 1810 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 1811 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 1812 sec_name, sec_idx, 1813 sec_desc->data->d_buf, 1814 sec_desc->data->d_size); 1815 break; 1816 case SEC_BSS: 1817 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 1818 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 1819 sec_name, sec_idx, 1820 NULL, 1821 sec_desc->data->d_size); 1822 break; 1823 default: 1824 /* skip */ 1825 break; 1826 } 1827 if (err) 1828 return err; 1829 } 1830 return 0; 1831 } 1832 1833 1834 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, 1835 const void *name) 1836 { 1837 int i; 1838 1839 for (i = 0; i < obj->nr_extern; i++) { 1840 if (strcmp(obj->externs[i].name, name) == 0) 1841 return &obj->externs[i]; 1842 } 1843 return NULL; 1844 } 1845 1846 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, 1847 char value) 1848 { 1849 switch (ext->kcfg.type) { 1850 case KCFG_BOOL: 1851 if (value == 'm') { 1852 pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n", 1853 ext->name, value); 1854 return -EINVAL; 1855 } 1856 *(bool *)ext_val = value == 'y' ? true : false; 1857 break; 1858 case KCFG_TRISTATE: 1859 if (value == 'y') 1860 *(enum libbpf_tristate *)ext_val = TRI_YES; 1861 else if (value == 'm') 1862 *(enum libbpf_tristate *)ext_val = TRI_MODULE; 1863 else /* value == 'n' */ 1864 *(enum libbpf_tristate *)ext_val = TRI_NO; 1865 break; 1866 case KCFG_CHAR: 1867 *(char *)ext_val = value; 1868 break; 1869 case KCFG_UNKNOWN: 1870 case KCFG_INT: 1871 case KCFG_CHAR_ARR: 1872 default: 1873 pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n", 1874 ext->name, value); 1875 return -EINVAL; 1876 } 1877 ext->is_set = true; 1878 return 0; 1879 } 1880 1881 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, 1882 const char *value) 1883 { 1884 size_t len; 1885 1886 if (ext->kcfg.type != KCFG_CHAR_ARR) { 1887 pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n", 1888 ext->name, value); 1889 return -EINVAL; 1890 } 1891 1892 len = strlen(value); 1893 if (value[len - 1] != '"') { 1894 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n", 1895 ext->name, value); 1896 return -EINVAL; 1897 } 1898 1899 /* strip quotes */ 1900 len -= 2; 1901 if (len >= ext->kcfg.sz) { 1902 pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n", 1903 ext->name, value, len, ext->kcfg.sz - 1); 1904 len = ext->kcfg.sz - 1; 1905 } 1906 memcpy(ext_val, value + 1, len); 1907 ext_val[len] = '\0'; 1908 ext->is_set = true; 1909 return 0; 1910 } 1911 1912 static int parse_u64(const char *value, __u64 *res) 1913 { 1914 char *value_end; 1915 int err; 1916 1917 errno = 0; 1918 *res = strtoull(value, &value_end, 0); 1919 if (errno) { 1920 err = -errno; 1921 pr_warn("failed to parse '%s' as integer: %d\n", value, err); 1922 return err; 1923 } 1924 if (*value_end) { 1925 pr_warn("failed to parse '%s' as integer completely\n", value); 1926 return -EINVAL; 1927 } 1928 return 0; 1929 } 1930 1931 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) 1932 { 1933 int bit_sz = ext->kcfg.sz * 8; 1934 1935 if (ext->kcfg.sz == 8) 1936 return true; 1937 1938 /* Validate that value stored in u64 fits in integer of `ext->sz` 1939 * bytes size without any loss of information. If the target integer 1940 * is signed, we rely on the following limits of integer type of 1941 * Y bits and subsequent transformation: 1942 * 1943 * -2^(Y-1) <= X <= 2^(Y-1) - 1 1944 * 0 <= X + 2^(Y-1) <= 2^Y - 1 1945 * 0 <= X + 2^(Y-1) < 2^Y 1946 * 1947 * For unsigned target integer, check that all the (64 - Y) bits are 1948 * zero. 1949 */ 1950 if (ext->kcfg.is_signed) 1951 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); 1952 else 1953 return (v >> bit_sz) == 0; 1954 } 1955 1956 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, 1957 __u64 value) 1958 { 1959 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR && 1960 ext->kcfg.type != KCFG_BOOL) { 1961 pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n", 1962 ext->name, (unsigned long long)value); 1963 return -EINVAL; 1964 } 1965 if (ext->kcfg.type == KCFG_BOOL && value > 1) { 1966 pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n", 1967 ext->name, (unsigned long long)value); 1968 return -EINVAL; 1969 1970 } 1971 if (!is_kcfg_value_in_range(ext, value)) { 1972 pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n", 1973 ext->name, (unsigned long long)value, ext->kcfg.sz); 1974 return -ERANGE; 1975 } 1976 switch (ext->kcfg.sz) { 1977 case 1: 1978 *(__u8 *)ext_val = value; 1979 break; 1980 case 2: 1981 *(__u16 *)ext_val = value; 1982 break; 1983 case 4: 1984 *(__u32 *)ext_val = value; 1985 break; 1986 case 8: 1987 *(__u64 *)ext_val = value; 1988 break; 1989 default: 1990 return -EINVAL; 1991 } 1992 ext->is_set = true; 1993 return 0; 1994 } 1995 1996 static int bpf_object__process_kconfig_line(struct bpf_object *obj, 1997 char *buf, void *data) 1998 { 1999 struct extern_desc *ext; 2000 char *sep, *value; 2001 int len, err = 0; 2002 void *ext_val; 2003 __u64 num; 2004 2005 if (!str_has_pfx(buf, "CONFIG_")) 2006 return 0; 2007 2008 sep = strchr(buf, '='); 2009 if (!sep) { 2010 pr_warn("failed to parse '%s': no separator\n", buf); 2011 return -EINVAL; 2012 } 2013 2014 /* Trim ending '\n' */ 2015 len = strlen(buf); 2016 if (buf[len - 1] == '\n') 2017 buf[len - 1] = '\0'; 2018 /* Split on '=' and ensure that a value is present. */ 2019 *sep = '\0'; 2020 if (!sep[1]) { 2021 *sep = '='; 2022 pr_warn("failed to parse '%s': no value\n", buf); 2023 return -EINVAL; 2024 } 2025 2026 ext = find_extern_by_name(obj, buf); 2027 if (!ext || ext->is_set) 2028 return 0; 2029 2030 ext_val = data + ext->kcfg.data_off; 2031 value = sep + 1; 2032 2033 switch (*value) { 2034 case 'y': case 'n': case 'm': 2035 err = set_kcfg_value_tri(ext, ext_val, *value); 2036 break; 2037 case '"': 2038 err = set_kcfg_value_str(ext, ext_val, value); 2039 break; 2040 default: 2041 /* assume integer */ 2042 err = parse_u64(value, &num); 2043 if (err) { 2044 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value); 2045 return err; 2046 } 2047 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { 2048 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value); 2049 return -EINVAL; 2050 } 2051 err = set_kcfg_value_num(ext, ext_val, num); 2052 break; 2053 } 2054 if (err) 2055 return err; 2056 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value); 2057 return 0; 2058 } 2059 2060 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) 2061 { 2062 char buf[PATH_MAX]; 2063 struct utsname uts; 2064 int len, err = 0; 2065 gzFile file; 2066 2067 uname(&uts); 2068 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); 2069 if (len < 0) 2070 return -EINVAL; 2071 else if (len >= PATH_MAX) 2072 return -ENAMETOOLONG; 2073 2074 /* gzopen also accepts uncompressed files. */ 2075 file = gzopen(buf, "re"); 2076 if (!file) 2077 file = gzopen("/proc/config.gz", "re"); 2078 2079 if (!file) { 2080 pr_warn("failed to open system Kconfig\n"); 2081 return -ENOENT; 2082 } 2083 2084 while (gzgets(file, buf, sizeof(buf))) { 2085 err = bpf_object__process_kconfig_line(obj, buf, data); 2086 if (err) { 2087 pr_warn("error parsing system Kconfig line '%s': %d\n", 2088 buf, err); 2089 goto out; 2090 } 2091 } 2092 2093 out: 2094 gzclose(file); 2095 return err; 2096 } 2097 2098 static int bpf_object__read_kconfig_mem(struct bpf_object *obj, 2099 const char *config, void *data) 2100 { 2101 char buf[PATH_MAX]; 2102 int err = 0; 2103 FILE *file; 2104 2105 file = fmemopen((void *)config, strlen(config), "r"); 2106 if (!file) { 2107 err = -errno; 2108 pr_warn("failed to open in-memory Kconfig: %d\n", err); 2109 return err; 2110 } 2111 2112 while (fgets(buf, sizeof(buf), file)) { 2113 err = bpf_object__process_kconfig_line(obj, buf, data); 2114 if (err) { 2115 pr_warn("error parsing in-memory Kconfig line '%s': %d\n", 2116 buf, err); 2117 break; 2118 } 2119 } 2120 2121 fclose(file); 2122 return err; 2123 } 2124 2125 static int bpf_object__init_kconfig_map(struct bpf_object *obj) 2126 { 2127 struct extern_desc *last_ext = NULL, *ext; 2128 size_t map_sz; 2129 int i, err; 2130 2131 for (i = 0; i < obj->nr_extern; i++) { 2132 ext = &obj->externs[i]; 2133 if (ext->type == EXT_KCFG) 2134 last_ext = ext; 2135 } 2136 2137 if (!last_ext) 2138 return 0; 2139 2140 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; 2141 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, 2142 ".kconfig", obj->efile.symbols_shndx, 2143 NULL, map_sz); 2144 if (err) 2145 return err; 2146 2147 obj->kconfig_map_idx = obj->nr_maps - 1; 2148 2149 return 0; 2150 } 2151 2152 const struct btf_type * 2153 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) 2154 { 2155 const struct btf_type *t = btf__type_by_id(btf, id); 2156 2157 if (res_id) 2158 *res_id = id; 2159 2160 while (btf_is_mod(t) || btf_is_typedef(t)) { 2161 if (res_id) 2162 *res_id = t->type; 2163 t = btf__type_by_id(btf, t->type); 2164 } 2165 2166 return t; 2167 } 2168 2169 static const struct btf_type * 2170 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) 2171 { 2172 const struct btf_type *t; 2173 2174 t = skip_mods_and_typedefs(btf, id, NULL); 2175 if (!btf_is_ptr(t)) 2176 return NULL; 2177 2178 t = skip_mods_and_typedefs(btf, t->type, res_id); 2179 2180 return btf_is_func_proto(t) ? t : NULL; 2181 } 2182 2183 static const char *__btf_kind_str(__u16 kind) 2184 { 2185 switch (kind) { 2186 case BTF_KIND_UNKN: return "void"; 2187 case BTF_KIND_INT: return "int"; 2188 case BTF_KIND_PTR: return "ptr"; 2189 case BTF_KIND_ARRAY: return "array"; 2190 case BTF_KIND_STRUCT: return "struct"; 2191 case BTF_KIND_UNION: return "union"; 2192 case BTF_KIND_ENUM: return "enum"; 2193 case BTF_KIND_FWD: return "fwd"; 2194 case BTF_KIND_TYPEDEF: return "typedef"; 2195 case BTF_KIND_VOLATILE: return "volatile"; 2196 case BTF_KIND_CONST: return "const"; 2197 case BTF_KIND_RESTRICT: return "restrict"; 2198 case BTF_KIND_FUNC: return "func"; 2199 case BTF_KIND_FUNC_PROTO: return "func_proto"; 2200 case BTF_KIND_VAR: return "var"; 2201 case BTF_KIND_DATASEC: return "datasec"; 2202 case BTF_KIND_FLOAT: return "float"; 2203 case BTF_KIND_DECL_TAG: return "decl_tag"; 2204 case BTF_KIND_TYPE_TAG: return "type_tag"; 2205 case BTF_KIND_ENUM64: return "enum64"; 2206 default: return "unknown"; 2207 } 2208 } 2209 2210 const char *btf_kind_str(const struct btf_type *t) 2211 { 2212 return __btf_kind_str(btf_kind(t)); 2213 } 2214 2215 /* 2216 * Fetch integer attribute of BTF map definition. Such attributes are 2217 * represented using a pointer to an array, in which dimensionality of array 2218 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 2219 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 2220 * type definition, while using only sizeof(void *) space in ELF data section. 2221 */ 2222 static bool get_map_field_int(const char *map_name, const struct btf *btf, 2223 const struct btf_member *m, __u32 *res) 2224 { 2225 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); 2226 const char *name = btf__name_by_offset(btf, m->name_off); 2227 const struct btf_array *arr_info; 2228 const struct btf_type *arr_t; 2229 2230 if (!btf_is_ptr(t)) { 2231 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n", 2232 map_name, name, btf_kind_str(t)); 2233 return false; 2234 } 2235 2236 arr_t = btf__type_by_id(btf, t->type); 2237 if (!arr_t) { 2238 pr_warn("map '%s': attr '%s': type [%u] not found.\n", 2239 map_name, name, t->type); 2240 return false; 2241 } 2242 if (!btf_is_array(arr_t)) { 2243 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n", 2244 map_name, name, btf_kind_str(arr_t)); 2245 return false; 2246 } 2247 arr_info = btf_array(arr_t); 2248 *res = arr_info->nelems; 2249 return true; 2250 } 2251 2252 static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name) 2253 { 2254 int len; 2255 2256 len = snprintf(buf, buf_sz, "%s/%s", path, name); 2257 if (len < 0) 2258 return -EINVAL; 2259 if (len >= buf_sz) 2260 return -ENAMETOOLONG; 2261 2262 return 0; 2263 } 2264 2265 static int build_map_pin_path(struct bpf_map *map, const char *path) 2266 { 2267 char buf[PATH_MAX]; 2268 int err; 2269 2270 if (!path) 2271 path = BPF_FS_DEFAULT_PATH; 2272 2273 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 2274 if (err) 2275 return err; 2276 2277 return bpf_map__set_pin_path(map, buf); 2278 } 2279 2280 /* should match definition in bpf_helpers.h */ 2281 enum libbpf_pin_type { 2282 LIBBPF_PIN_NONE, 2283 /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ 2284 LIBBPF_PIN_BY_NAME, 2285 }; 2286 2287 int parse_btf_map_def(const char *map_name, struct btf *btf, 2288 const struct btf_type *def_t, bool strict, 2289 struct btf_map_def *map_def, struct btf_map_def *inner_def) 2290 { 2291 const struct btf_type *t; 2292 const struct btf_member *m; 2293 bool is_inner = inner_def == NULL; 2294 int vlen, i; 2295 2296 vlen = btf_vlen(def_t); 2297 m = btf_members(def_t); 2298 for (i = 0; i < vlen; i++, m++) { 2299 const char *name = btf__name_by_offset(btf, m->name_off); 2300 2301 if (!name) { 2302 pr_warn("map '%s': invalid field #%d.\n", map_name, i); 2303 return -EINVAL; 2304 } 2305 if (strcmp(name, "type") == 0) { 2306 if (!get_map_field_int(map_name, btf, m, &map_def->map_type)) 2307 return -EINVAL; 2308 map_def->parts |= MAP_DEF_MAP_TYPE; 2309 } else if (strcmp(name, "max_entries") == 0) { 2310 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries)) 2311 return -EINVAL; 2312 map_def->parts |= MAP_DEF_MAX_ENTRIES; 2313 } else if (strcmp(name, "map_flags") == 0) { 2314 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags)) 2315 return -EINVAL; 2316 map_def->parts |= MAP_DEF_MAP_FLAGS; 2317 } else if (strcmp(name, "numa_node") == 0) { 2318 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node)) 2319 return -EINVAL; 2320 map_def->parts |= MAP_DEF_NUMA_NODE; 2321 } else if (strcmp(name, "key_size") == 0) { 2322 __u32 sz; 2323 2324 if (!get_map_field_int(map_name, btf, m, &sz)) 2325 return -EINVAL; 2326 if (map_def->key_size && map_def->key_size != sz) { 2327 pr_warn("map '%s': conflicting key size %u != %u.\n", 2328 map_name, map_def->key_size, sz); 2329 return -EINVAL; 2330 } 2331 map_def->key_size = sz; 2332 map_def->parts |= MAP_DEF_KEY_SIZE; 2333 } else if (strcmp(name, "key") == 0) { 2334 __s64 sz; 2335 2336 t = btf__type_by_id(btf, m->type); 2337 if (!t) { 2338 pr_warn("map '%s': key type [%d] not found.\n", 2339 map_name, m->type); 2340 return -EINVAL; 2341 } 2342 if (!btf_is_ptr(t)) { 2343 pr_warn("map '%s': key spec is not PTR: %s.\n", 2344 map_name, btf_kind_str(t)); 2345 return -EINVAL; 2346 } 2347 sz = btf__resolve_size(btf, t->type); 2348 if (sz < 0) { 2349 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", 2350 map_name, t->type, (ssize_t)sz); 2351 return sz; 2352 } 2353 if (map_def->key_size && map_def->key_size != sz) { 2354 pr_warn("map '%s': conflicting key size %u != %zd.\n", 2355 map_name, map_def->key_size, (ssize_t)sz); 2356 return -EINVAL; 2357 } 2358 map_def->key_size = sz; 2359 map_def->key_type_id = t->type; 2360 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE; 2361 } else if (strcmp(name, "value_size") == 0) { 2362 __u32 sz; 2363 2364 if (!get_map_field_int(map_name, btf, m, &sz)) 2365 return -EINVAL; 2366 if (map_def->value_size && map_def->value_size != sz) { 2367 pr_warn("map '%s': conflicting value size %u != %u.\n", 2368 map_name, map_def->value_size, sz); 2369 return -EINVAL; 2370 } 2371 map_def->value_size = sz; 2372 map_def->parts |= MAP_DEF_VALUE_SIZE; 2373 } else if (strcmp(name, "value") == 0) { 2374 __s64 sz; 2375 2376 t = btf__type_by_id(btf, m->type); 2377 if (!t) { 2378 pr_warn("map '%s': value type [%d] not found.\n", 2379 map_name, m->type); 2380 return -EINVAL; 2381 } 2382 if (!btf_is_ptr(t)) { 2383 pr_warn("map '%s': value spec is not PTR: %s.\n", 2384 map_name, btf_kind_str(t)); 2385 return -EINVAL; 2386 } 2387 sz = btf__resolve_size(btf, t->type); 2388 if (sz < 0) { 2389 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", 2390 map_name, t->type, (ssize_t)sz); 2391 return sz; 2392 } 2393 if (map_def->value_size && map_def->value_size != sz) { 2394 pr_warn("map '%s': conflicting value size %u != %zd.\n", 2395 map_name, map_def->value_size, (ssize_t)sz); 2396 return -EINVAL; 2397 } 2398 map_def->value_size = sz; 2399 map_def->value_type_id = t->type; 2400 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE; 2401 } 2402 else if (strcmp(name, "values") == 0) { 2403 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type); 2404 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY; 2405 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value"; 2406 char inner_map_name[128]; 2407 int err; 2408 2409 if (is_inner) { 2410 pr_warn("map '%s': multi-level inner maps not supported.\n", 2411 map_name); 2412 return -ENOTSUP; 2413 } 2414 if (i != vlen - 1) { 2415 pr_warn("map '%s': '%s' member should be last.\n", 2416 map_name, name); 2417 return -EINVAL; 2418 } 2419 if (!is_map_in_map && !is_prog_array) { 2420 pr_warn("map '%s': should be map-in-map or prog-array.\n", 2421 map_name); 2422 return -ENOTSUP; 2423 } 2424 if (map_def->value_size && map_def->value_size != 4) { 2425 pr_warn("map '%s': conflicting value size %u != 4.\n", 2426 map_name, map_def->value_size); 2427 return -EINVAL; 2428 } 2429 map_def->value_size = 4; 2430 t = btf__type_by_id(btf, m->type); 2431 if (!t) { 2432 pr_warn("map '%s': %s type [%d] not found.\n", 2433 map_name, desc, m->type); 2434 return -EINVAL; 2435 } 2436 if (!btf_is_array(t) || btf_array(t)->nelems) { 2437 pr_warn("map '%s': %s spec is not a zero-sized array.\n", 2438 map_name, desc); 2439 return -EINVAL; 2440 } 2441 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL); 2442 if (!btf_is_ptr(t)) { 2443 pr_warn("map '%s': %s def is of unexpected kind %s.\n", 2444 map_name, desc, btf_kind_str(t)); 2445 return -EINVAL; 2446 } 2447 t = skip_mods_and_typedefs(btf, t->type, NULL); 2448 if (is_prog_array) { 2449 if (!btf_is_func_proto(t)) { 2450 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n", 2451 map_name, btf_kind_str(t)); 2452 return -EINVAL; 2453 } 2454 continue; 2455 } 2456 if (!btf_is_struct(t)) { 2457 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2458 map_name, btf_kind_str(t)); 2459 return -EINVAL; 2460 } 2461 2462 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name); 2463 err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL); 2464 if (err) 2465 return err; 2466 2467 map_def->parts |= MAP_DEF_INNER_MAP; 2468 } else if (strcmp(name, "pinning") == 0) { 2469 __u32 val; 2470 2471 if (is_inner) { 2472 pr_warn("map '%s': inner def can't be pinned.\n", map_name); 2473 return -EINVAL; 2474 } 2475 if (!get_map_field_int(map_name, btf, m, &val)) 2476 return -EINVAL; 2477 if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) { 2478 pr_warn("map '%s': invalid pinning value %u.\n", 2479 map_name, val); 2480 return -EINVAL; 2481 } 2482 map_def->pinning = val; 2483 map_def->parts |= MAP_DEF_PINNING; 2484 } else if (strcmp(name, "map_extra") == 0) { 2485 __u32 map_extra; 2486 2487 if (!get_map_field_int(map_name, btf, m, &map_extra)) 2488 return -EINVAL; 2489 map_def->map_extra = map_extra; 2490 map_def->parts |= MAP_DEF_MAP_EXTRA; 2491 } else { 2492 if (strict) { 2493 pr_warn("map '%s': unknown field '%s'.\n", map_name, name); 2494 return -ENOTSUP; 2495 } 2496 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name); 2497 } 2498 } 2499 2500 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) { 2501 pr_warn("map '%s': map type isn't specified.\n", map_name); 2502 return -EINVAL; 2503 } 2504 2505 return 0; 2506 } 2507 2508 static size_t adjust_ringbuf_sz(size_t sz) 2509 { 2510 __u32 page_sz = sysconf(_SC_PAGE_SIZE); 2511 __u32 mul; 2512 2513 /* if user forgot to set any size, make sure they see error */ 2514 if (sz == 0) 2515 return 0; 2516 /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be 2517 * a power-of-2 multiple of kernel's page size. If user diligently 2518 * satisified these conditions, pass the size through. 2519 */ 2520 if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz)) 2521 return sz; 2522 2523 /* Otherwise find closest (page_sz * power_of_2) product bigger than 2524 * user-set size to satisfy both user size request and kernel 2525 * requirements and substitute correct max_entries for map creation. 2526 */ 2527 for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) { 2528 if (mul * page_sz > sz) 2529 return mul * page_sz; 2530 } 2531 2532 /* if it's impossible to satisfy the conditions (i.e., user size is 2533 * very close to UINT_MAX but is not a power-of-2 multiple of 2534 * page_size) then just return original size and let kernel reject it 2535 */ 2536 return sz; 2537 } 2538 2539 static bool map_is_ringbuf(const struct bpf_map *map) 2540 { 2541 return map->def.type == BPF_MAP_TYPE_RINGBUF || 2542 map->def.type == BPF_MAP_TYPE_USER_RINGBUF; 2543 } 2544 2545 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) 2546 { 2547 map->def.type = def->map_type; 2548 map->def.key_size = def->key_size; 2549 map->def.value_size = def->value_size; 2550 map->def.max_entries = def->max_entries; 2551 map->def.map_flags = def->map_flags; 2552 map->map_extra = def->map_extra; 2553 2554 map->numa_node = def->numa_node; 2555 map->btf_key_type_id = def->key_type_id; 2556 map->btf_value_type_id = def->value_type_id; 2557 2558 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ 2559 if (map_is_ringbuf(map)) 2560 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); 2561 2562 if (def->parts & MAP_DEF_MAP_TYPE) 2563 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); 2564 2565 if (def->parts & MAP_DEF_KEY_TYPE) 2566 pr_debug("map '%s': found key [%u], sz = %u.\n", 2567 map->name, def->key_type_id, def->key_size); 2568 else if (def->parts & MAP_DEF_KEY_SIZE) 2569 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size); 2570 2571 if (def->parts & MAP_DEF_VALUE_TYPE) 2572 pr_debug("map '%s': found value [%u], sz = %u.\n", 2573 map->name, def->value_type_id, def->value_size); 2574 else if (def->parts & MAP_DEF_VALUE_SIZE) 2575 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size); 2576 2577 if (def->parts & MAP_DEF_MAX_ENTRIES) 2578 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries); 2579 if (def->parts & MAP_DEF_MAP_FLAGS) 2580 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags); 2581 if (def->parts & MAP_DEF_MAP_EXTRA) 2582 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name, 2583 (unsigned long long)def->map_extra); 2584 if (def->parts & MAP_DEF_PINNING) 2585 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning); 2586 if (def->parts & MAP_DEF_NUMA_NODE) 2587 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node); 2588 2589 if (def->parts & MAP_DEF_INNER_MAP) 2590 pr_debug("map '%s': found inner map definition.\n", map->name); 2591 } 2592 2593 static const char *btf_var_linkage_str(__u32 linkage) 2594 { 2595 switch (linkage) { 2596 case BTF_VAR_STATIC: return "static"; 2597 case BTF_VAR_GLOBAL_ALLOCATED: return "global"; 2598 case BTF_VAR_GLOBAL_EXTERN: return "extern"; 2599 default: return "unknown"; 2600 } 2601 } 2602 2603 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 2604 const struct btf_type *sec, 2605 int var_idx, int sec_idx, 2606 const Elf_Data *data, bool strict, 2607 const char *pin_root_path) 2608 { 2609 struct btf_map_def map_def = {}, inner_def = {}; 2610 const struct btf_type *var, *def; 2611 const struct btf_var_secinfo *vi; 2612 const struct btf_var *var_extra; 2613 const char *map_name; 2614 struct bpf_map *map; 2615 int err; 2616 2617 vi = btf_var_secinfos(sec) + var_idx; 2618 var = btf__type_by_id(obj->btf, vi->type); 2619 var_extra = btf_var(var); 2620 map_name = btf__name_by_offset(obj->btf, var->name_off); 2621 2622 if (map_name == NULL || map_name[0] == '\0') { 2623 pr_warn("map #%d: empty name.\n", var_idx); 2624 return -EINVAL; 2625 } 2626 if ((__u64)vi->offset + vi->size > data->d_size) { 2627 pr_warn("map '%s' BTF data is corrupted.\n", map_name); 2628 return -EINVAL; 2629 } 2630 if (!btf_is_var(var)) { 2631 pr_warn("map '%s': unexpected var kind %s.\n", 2632 map_name, btf_kind_str(var)); 2633 return -EINVAL; 2634 } 2635 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) { 2636 pr_warn("map '%s': unsupported map linkage %s.\n", 2637 map_name, btf_var_linkage_str(var_extra->linkage)); 2638 return -EOPNOTSUPP; 2639 } 2640 2641 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 2642 if (!btf_is_struct(def)) { 2643 pr_warn("map '%s': unexpected def kind %s.\n", 2644 map_name, btf_kind_str(var)); 2645 return -EINVAL; 2646 } 2647 if (def->size > vi->size) { 2648 pr_warn("map '%s': invalid def size.\n", map_name); 2649 return -EINVAL; 2650 } 2651 2652 map = bpf_object__add_map(obj); 2653 if (IS_ERR(map)) 2654 return PTR_ERR(map); 2655 map->name = strdup(map_name); 2656 if (!map->name) { 2657 pr_warn("map '%s': failed to alloc map name.\n", map_name); 2658 return -ENOMEM; 2659 } 2660 map->libbpf_type = LIBBPF_MAP_UNSPEC; 2661 map->def.type = BPF_MAP_TYPE_UNSPEC; 2662 map->sec_idx = sec_idx; 2663 map->sec_offset = vi->offset; 2664 map->btf_var_idx = var_idx; 2665 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 2666 map_name, map->sec_idx, map->sec_offset); 2667 2668 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); 2669 if (err) 2670 return err; 2671 2672 fill_map_from_def(map, &map_def); 2673 2674 if (map_def.pinning == LIBBPF_PIN_BY_NAME) { 2675 err = build_map_pin_path(map, pin_root_path); 2676 if (err) { 2677 pr_warn("map '%s': couldn't build pin path.\n", map->name); 2678 return err; 2679 } 2680 } 2681 2682 if (map_def.parts & MAP_DEF_INNER_MAP) { 2683 map->inner_map = calloc(1, sizeof(*map->inner_map)); 2684 if (!map->inner_map) 2685 return -ENOMEM; 2686 map->inner_map->fd = create_placeholder_fd(); 2687 if (map->inner_map->fd < 0) 2688 return map->inner_map->fd; 2689 map->inner_map->sec_idx = sec_idx; 2690 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1); 2691 if (!map->inner_map->name) 2692 return -ENOMEM; 2693 sprintf(map->inner_map->name, "%s.inner", map_name); 2694 2695 fill_map_from_def(map->inner_map, &inner_def); 2696 } 2697 2698 err = map_fill_btf_type_info(obj, map); 2699 if (err) 2700 return err; 2701 2702 return 0; 2703 } 2704 2705 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, 2706 const char *pin_root_path) 2707 { 2708 const struct btf_type *sec = NULL; 2709 int nr_types, i, vlen, err; 2710 const struct btf_type *t; 2711 const char *name; 2712 Elf_Data *data; 2713 Elf_Scn *scn; 2714 2715 if (obj->efile.btf_maps_shndx < 0) 2716 return 0; 2717 2718 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); 2719 data = elf_sec_data(obj, scn); 2720 if (!scn || !data) { 2721 pr_warn("elf: failed to get %s map definitions for %s\n", 2722 MAPS_ELF_SEC, obj->path); 2723 return -EINVAL; 2724 } 2725 2726 nr_types = btf__type_cnt(obj->btf); 2727 for (i = 1; i < nr_types; i++) { 2728 t = btf__type_by_id(obj->btf, i); 2729 if (!btf_is_datasec(t)) 2730 continue; 2731 name = btf__name_by_offset(obj->btf, t->name_off); 2732 if (strcmp(name, MAPS_ELF_SEC) == 0) { 2733 sec = t; 2734 obj->efile.btf_maps_sec_btf_id = i; 2735 break; 2736 } 2737 } 2738 2739 if (!sec) { 2740 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 2741 return -ENOENT; 2742 } 2743 2744 vlen = btf_vlen(sec); 2745 for (i = 0; i < vlen; i++) { 2746 err = bpf_object__init_user_btf_map(obj, sec, i, 2747 obj->efile.btf_maps_shndx, 2748 data, strict, 2749 pin_root_path); 2750 if (err) 2751 return err; 2752 } 2753 2754 return 0; 2755 } 2756 2757 static int bpf_object__init_maps(struct bpf_object *obj, 2758 const struct bpf_object_open_opts *opts) 2759 { 2760 const char *pin_root_path; 2761 bool strict; 2762 int err = 0; 2763 2764 strict = !OPTS_GET(opts, relaxed_maps, false); 2765 pin_root_path = OPTS_GET(opts, pin_root_path, NULL); 2766 2767 err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path); 2768 err = err ?: bpf_object__init_global_data_maps(obj); 2769 err = err ?: bpf_object__init_kconfig_map(obj); 2770 err = err ?: bpf_object_init_struct_ops(obj); 2771 2772 return err; 2773 } 2774 2775 static bool section_have_execinstr(struct bpf_object *obj, int idx) 2776 { 2777 Elf64_Shdr *sh; 2778 2779 sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx)); 2780 if (!sh) 2781 return false; 2782 2783 return sh->sh_flags & SHF_EXECINSTR; 2784 } 2785 2786 static bool btf_needs_sanitization(struct bpf_object *obj) 2787 { 2788 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2789 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2790 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2791 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2792 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); 2793 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); 2794 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); 2795 2796 return !has_func || !has_datasec || !has_func_global || !has_float || 2797 !has_decl_tag || !has_type_tag || !has_enum64; 2798 } 2799 2800 static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) 2801 { 2802 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2803 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2804 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2805 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2806 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); 2807 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); 2808 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); 2809 int enum64_placeholder_id = 0; 2810 struct btf_type *t; 2811 int i, j, vlen; 2812 2813 for (i = 1; i < btf__type_cnt(btf); i++) { 2814 t = (struct btf_type *)btf__type_by_id(btf, i); 2815 2816 if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) { 2817 /* replace VAR/DECL_TAG with INT */ 2818 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 2819 /* 2820 * using size = 1 is the safest choice, 4 will be too 2821 * big and cause kernel BTF validation failure if 2822 * original variable took less than 4 bytes 2823 */ 2824 t->size = 1; 2825 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); 2826 } else if (!has_datasec && btf_is_datasec(t)) { 2827 /* replace DATASEC with STRUCT */ 2828 const struct btf_var_secinfo *v = btf_var_secinfos(t); 2829 struct btf_member *m = btf_members(t); 2830 struct btf_type *vt; 2831 char *name; 2832 2833 name = (char *)btf__name_by_offset(btf, t->name_off); 2834 while (*name) { 2835 if (*name == '.') 2836 *name = '_'; 2837 name++; 2838 } 2839 2840 vlen = btf_vlen(t); 2841 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 2842 for (j = 0; j < vlen; j++, v++, m++) { 2843 /* order of field assignments is important */ 2844 m->offset = v->offset * 8; 2845 m->type = v->type; 2846 /* preserve variable name as member name */ 2847 vt = (void *)btf__type_by_id(btf, v->type); 2848 m->name_off = vt->name_off; 2849 } 2850 } else if (!has_func && btf_is_func_proto(t)) { 2851 /* replace FUNC_PROTO with ENUM */ 2852 vlen = btf_vlen(t); 2853 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 2854 t->size = sizeof(__u32); /* kernel enforced */ 2855 } else if (!has_func && btf_is_func(t)) { 2856 /* replace FUNC with TYPEDEF */ 2857 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 2858 } else if (!has_func_global && btf_is_func(t)) { 2859 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ 2860 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); 2861 } else if (!has_float && btf_is_float(t)) { 2862 /* replace FLOAT with an equally-sized empty STRUCT; 2863 * since C compilers do not accept e.g. "float" as a 2864 * valid struct name, make it anonymous 2865 */ 2866 t->name_off = 0; 2867 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); 2868 } else if (!has_type_tag && btf_is_type_tag(t)) { 2869 /* replace TYPE_TAG with a CONST */ 2870 t->name_off = 0; 2871 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0); 2872 } else if (!has_enum64 && btf_is_enum(t)) { 2873 /* clear the kflag */ 2874 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false); 2875 } else if (!has_enum64 && btf_is_enum64(t)) { 2876 /* replace ENUM64 with a union */ 2877 struct btf_member *m; 2878 2879 if (enum64_placeholder_id == 0) { 2880 enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0); 2881 if (enum64_placeholder_id < 0) 2882 return enum64_placeholder_id; 2883 2884 t = (struct btf_type *)btf__type_by_id(btf, i); 2885 } 2886 2887 m = btf_members(t); 2888 vlen = btf_vlen(t); 2889 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen); 2890 for (j = 0; j < vlen; j++, m++) { 2891 m->type = enum64_placeholder_id; 2892 m->offset = 0; 2893 } 2894 } 2895 } 2896 2897 return 0; 2898 } 2899 2900 static bool libbpf_needs_btf(const struct bpf_object *obj) 2901 { 2902 return obj->efile.btf_maps_shndx >= 0 || 2903 obj->efile.st_ops_shndx >= 0 || 2904 obj->efile.st_ops_link_shndx >= 0 || 2905 obj->nr_extern > 0; 2906 } 2907 2908 static bool kernel_needs_btf(const struct bpf_object *obj) 2909 { 2910 return obj->efile.st_ops_shndx >= 0 || obj->efile.st_ops_link_shndx >= 0; 2911 } 2912 2913 static int bpf_object__init_btf(struct bpf_object *obj, 2914 Elf_Data *btf_data, 2915 Elf_Data *btf_ext_data) 2916 { 2917 int err = -ENOENT; 2918 2919 if (btf_data) { 2920 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 2921 err = libbpf_get_error(obj->btf); 2922 if (err) { 2923 obj->btf = NULL; 2924 pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err); 2925 goto out; 2926 } 2927 /* enforce 8-byte pointers for BPF-targeted BTFs */ 2928 btf__set_pointer_size(obj->btf, 8); 2929 } 2930 if (btf_ext_data) { 2931 struct btf_ext_info *ext_segs[3]; 2932 int seg_num, sec_num; 2933 2934 if (!obj->btf) { 2935 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 2936 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 2937 goto out; 2938 } 2939 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); 2940 err = libbpf_get_error(obj->btf_ext); 2941 if (err) { 2942 pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n", 2943 BTF_EXT_ELF_SEC, err); 2944 obj->btf_ext = NULL; 2945 goto out; 2946 } 2947 2948 /* setup .BTF.ext to ELF section mapping */ 2949 ext_segs[0] = &obj->btf_ext->func_info; 2950 ext_segs[1] = &obj->btf_ext->line_info; 2951 ext_segs[2] = &obj->btf_ext->core_relo_info; 2952 for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) { 2953 struct btf_ext_info *seg = ext_segs[seg_num]; 2954 const struct btf_ext_info_sec *sec; 2955 const char *sec_name; 2956 Elf_Scn *scn; 2957 2958 if (seg->sec_cnt == 0) 2959 continue; 2960 2961 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs)); 2962 if (!seg->sec_idxs) { 2963 err = -ENOMEM; 2964 goto out; 2965 } 2966 2967 sec_num = 0; 2968 for_each_btf_ext_sec(seg, sec) { 2969 /* preventively increment index to avoid doing 2970 * this before every continue below 2971 */ 2972 sec_num++; 2973 2974 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 2975 if (str_is_empty(sec_name)) 2976 continue; 2977 scn = elf_sec_by_name(obj, sec_name); 2978 if (!scn) 2979 continue; 2980 2981 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn); 2982 } 2983 } 2984 } 2985 out: 2986 if (err && libbpf_needs_btf(obj)) { 2987 pr_warn("BTF is required, but is missing or corrupted.\n"); 2988 return err; 2989 } 2990 return 0; 2991 } 2992 2993 static int compare_vsi_off(const void *_a, const void *_b) 2994 { 2995 const struct btf_var_secinfo *a = _a; 2996 const struct btf_var_secinfo *b = _b; 2997 2998 return a->offset - b->offset; 2999 } 3000 3001 static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, 3002 struct btf_type *t) 3003 { 3004 __u32 size = 0, i, vars = btf_vlen(t); 3005 const char *sec_name = btf__name_by_offset(btf, t->name_off); 3006 struct btf_var_secinfo *vsi; 3007 bool fixup_offsets = false; 3008 int err; 3009 3010 if (!sec_name) { 3011 pr_debug("No name found in string section for DATASEC kind.\n"); 3012 return -ENOENT; 3013 } 3014 3015 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and 3016 * variable offsets set at the previous step. Further, not every 3017 * extern BTF VAR has corresponding ELF symbol preserved, so we skip 3018 * all fixups altogether for such sections and go straight to sorting 3019 * VARs within their DATASEC. 3020 */ 3021 if (strcmp(sec_name, KCONFIG_SEC) == 0 || strcmp(sec_name, KSYMS_SEC) == 0) 3022 goto sort_vars; 3023 3024 /* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to 3025 * fix this up. But BPF static linker already fixes this up and fills 3026 * all the sizes and offsets during static linking. So this step has 3027 * to be optional. But the STV_HIDDEN handling is non-optional for any 3028 * non-extern DATASEC, so the variable fixup loop below handles both 3029 * functions at the same time, paying the cost of BTF VAR <-> ELF 3030 * symbol matching just once. 3031 */ 3032 if (t->size == 0) { 3033 err = find_elf_sec_sz(obj, sec_name, &size); 3034 if (err || !size) { 3035 pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n", 3036 sec_name, size, err); 3037 return -ENOENT; 3038 } 3039 3040 t->size = size; 3041 fixup_offsets = true; 3042 } 3043 3044 for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) { 3045 const struct btf_type *t_var; 3046 struct btf_var *var; 3047 const char *var_name; 3048 Elf64_Sym *sym; 3049 3050 t_var = btf__type_by_id(btf, vsi->type); 3051 if (!t_var || !btf_is_var(t_var)) { 3052 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name); 3053 return -EINVAL; 3054 } 3055 3056 var = btf_var(t_var); 3057 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN) 3058 continue; 3059 3060 var_name = btf__name_by_offset(btf, t_var->name_off); 3061 if (!var_name) { 3062 pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n", 3063 sec_name, i); 3064 return -ENOENT; 3065 } 3066 3067 sym = find_elf_var_sym(obj, var_name); 3068 if (IS_ERR(sym)) { 3069 pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n", 3070 sec_name, var_name); 3071 return -ENOENT; 3072 } 3073 3074 if (fixup_offsets) 3075 vsi->offset = sym->st_value; 3076 3077 /* if variable is a global/weak symbol, but has restricted 3078 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR 3079 * as static. This follows similar logic for functions (BPF 3080 * subprogs) and influences libbpf's further decisions about 3081 * whether to make global data BPF array maps as 3082 * BPF_F_MMAPABLE. 3083 */ 3084 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN 3085 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL) 3086 var->linkage = BTF_VAR_STATIC; 3087 } 3088 3089 sort_vars: 3090 qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off); 3091 return 0; 3092 } 3093 3094 static int bpf_object_fixup_btf(struct bpf_object *obj) 3095 { 3096 int i, n, err = 0; 3097 3098 if (!obj->btf) 3099 return 0; 3100 3101 n = btf__type_cnt(obj->btf); 3102 for (i = 1; i < n; i++) { 3103 struct btf_type *t = btf_type_by_id(obj->btf, i); 3104 3105 /* Loader needs to fix up some of the things compiler 3106 * couldn't get its hands on while emitting BTF. This 3107 * is section size and global variable offset. We use 3108 * the info from the ELF itself for this purpose. 3109 */ 3110 if (btf_is_datasec(t)) { 3111 err = btf_fixup_datasec(obj, obj->btf, t); 3112 if (err) 3113 return err; 3114 } 3115 } 3116 3117 return 0; 3118 } 3119 3120 static bool prog_needs_vmlinux_btf(struct bpf_program *prog) 3121 { 3122 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 3123 prog->type == BPF_PROG_TYPE_LSM) 3124 return true; 3125 3126 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs 3127 * also need vmlinux BTF 3128 */ 3129 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) 3130 return true; 3131 3132 return false; 3133 } 3134 3135 static bool map_needs_vmlinux_btf(struct bpf_map *map) 3136 { 3137 return bpf_map__is_struct_ops(map); 3138 } 3139 3140 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) 3141 { 3142 struct bpf_program *prog; 3143 struct bpf_map *map; 3144 int i; 3145 3146 /* CO-RE relocations need kernel BTF, only when btf_custom_path 3147 * is not specified 3148 */ 3149 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path) 3150 return true; 3151 3152 /* Support for typed ksyms needs kernel BTF */ 3153 for (i = 0; i < obj->nr_extern; i++) { 3154 const struct extern_desc *ext; 3155 3156 ext = &obj->externs[i]; 3157 if (ext->type == EXT_KSYM && ext->ksym.type_id) 3158 return true; 3159 } 3160 3161 bpf_object__for_each_program(prog, obj) { 3162 if (!prog->autoload) 3163 continue; 3164 if (prog_needs_vmlinux_btf(prog)) 3165 return true; 3166 } 3167 3168 bpf_object__for_each_map(map, obj) { 3169 if (map_needs_vmlinux_btf(map)) 3170 return true; 3171 } 3172 3173 return false; 3174 } 3175 3176 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) 3177 { 3178 int err; 3179 3180 /* btf_vmlinux could be loaded earlier */ 3181 if (obj->btf_vmlinux || obj->gen_loader) 3182 return 0; 3183 3184 if (!force && !obj_needs_vmlinux_btf(obj)) 3185 return 0; 3186 3187 obj->btf_vmlinux = btf__load_vmlinux_btf(); 3188 err = libbpf_get_error(obj->btf_vmlinux); 3189 if (err) { 3190 pr_warn("Error loading vmlinux BTF: %d\n", err); 3191 obj->btf_vmlinux = NULL; 3192 return err; 3193 } 3194 return 0; 3195 } 3196 3197 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 3198 { 3199 struct btf *kern_btf = obj->btf; 3200 bool btf_mandatory, sanitize; 3201 int i, err = 0; 3202 3203 if (!obj->btf) 3204 return 0; 3205 3206 if (!kernel_supports(obj, FEAT_BTF)) { 3207 if (kernel_needs_btf(obj)) { 3208 err = -EOPNOTSUPP; 3209 goto report; 3210 } 3211 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n"); 3212 return 0; 3213 } 3214 3215 /* Even though some subprogs are global/weak, user might prefer more 3216 * permissive BPF verification process that BPF verifier performs for 3217 * static functions, taking into account more context from the caller 3218 * functions. In such case, they need to mark such subprogs with 3219 * __attribute__((visibility("hidden"))) and libbpf will adjust 3220 * corresponding FUNC BTF type to be marked as static and trigger more 3221 * involved BPF verification process. 3222 */ 3223 for (i = 0; i < obj->nr_programs; i++) { 3224 struct bpf_program *prog = &obj->programs[i]; 3225 struct btf_type *t; 3226 const char *name; 3227 int j, n; 3228 3229 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) 3230 continue; 3231 3232 n = btf__type_cnt(obj->btf); 3233 for (j = 1; j < n; j++) { 3234 t = btf_type_by_id(obj->btf, j); 3235 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) 3236 continue; 3237 3238 name = btf__str_by_offset(obj->btf, t->name_off); 3239 if (strcmp(name, prog->name) != 0) 3240 continue; 3241 3242 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0); 3243 break; 3244 } 3245 } 3246 3247 sanitize = btf_needs_sanitization(obj); 3248 if (sanitize) { 3249 const void *raw_data; 3250 __u32 sz; 3251 3252 /* clone BTF to sanitize a copy and leave the original intact */ 3253 raw_data = btf__raw_data(obj->btf, &sz); 3254 kern_btf = btf__new(raw_data, sz); 3255 err = libbpf_get_error(kern_btf); 3256 if (err) 3257 return err; 3258 3259 /* enforce 8-byte pointers for BPF-targeted BTFs */ 3260 btf__set_pointer_size(obj->btf, 8); 3261 err = bpf_object__sanitize_btf(obj, kern_btf); 3262 if (err) 3263 return err; 3264 } 3265 3266 if (obj->gen_loader) { 3267 __u32 raw_size = 0; 3268 const void *raw_data = btf__raw_data(kern_btf, &raw_size); 3269 3270 if (!raw_data) 3271 return -ENOMEM; 3272 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size); 3273 /* Pretend to have valid FD to pass various fd >= 0 checks. 3274 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 3275 */ 3276 btf__set_fd(kern_btf, 0); 3277 } else { 3278 /* currently BPF_BTF_LOAD only supports log_level 1 */ 3279 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size, 3280 obj->log_level ? 1 : 0, obj->token_fd); 3281 } 3282 if (sanitize) { 3283 if (!err) { 3284 /* move fd to libbpf's BTF */ 3285 btf__set_fd(obj->btf, btf__fd(kern_btf)); 3286 btf__set_fd(kern_btf, -1); 3287 } 3288 btf__free(kern_btf); 3289 } 3290 report: 3291 if (err) { 3292 btf_mandatory = kernel_needs_btf(obj); 3293 pr_warn("Error loading .BTF into kernel: %d. %s\n", err, 3294 btf_mandatory ? "BTF is mandatory, can't proceed." 3295 : "BTF is optional, ignoring."); 3296 if (!btf_mandatory) 3297 err = 0; 3298 } 3299 return err; 3300 } 3301 3302 static const char *elf_sym_str(const struct bpf_object *obj, size_t off) 3303 { 3304 const char *name; 3305 3306 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); 3307 if (!name) { 3308 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 3309 off, obj->path, elf_errmsg(-1)); 3310 return NULL; 3311 } 3312 3313 return name; 3314 } 3315 3316 static const char *elf_sec_str(const struct bpf_object *obj, size_t off) 3317 { 3318 const char *name; 3319 3320 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); 3321 if (!name) { 3322 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 3323 off, obj->path, elf_errmsg(-1)); 3324 return NULL; 3325 } 3326 3327 return name; 3328 } 3329 3330 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx) 3331 { 3332 Elf_Scn *scn; 3333 3334 scn = elf_getscn(obj->efile.elf, idx); 3335 if (!scn) { 3336 pr_warn("elf: failed to get section(%zu) from %s: %s\n", 3337 idx, obj->path, elf_errmsg(-1)); 3338 return NULL; 3339 } 3340 return scn; 3341 } 3342 3343 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name) 3344 { 3345 Elf_Scn *scn = NULL; 3346 Elf *elf = obj->efile.elf; 3347 const char *sec_name; 3348 3349 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3350 sec_name = elf_sec_name(obj, scn); 3351 if (!sec_name) 3352 return NULL; 3353 3354 if (strcmp(sec_name, name) != 0) 3355 continue; 3356 3357 return scn; 3358 } 3359 return NULL; 3360 } 3361 3362 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn) 3363 { 3364 Elf64_Shdr *shdr; 3365 3366 if (!scn) 3367 return NULL; 3368 3369 shdr = elf64_getshdr(scn); 3370 if (!shdr) { 3371 pr_warn("elf: failed to get section(%zu) header from %s: %s\n", 3372 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 3373 return NULL; 3374 } 3375 3376 return shdr; 3377 } 3378 3379 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn) 3380 { 3381 const char *name; 3382 Elf64_Shdr *sh; 3383 3384 if (!scn) 3385 return NULL; 3386 3387 sh = elf_sec_hdr(obj, scn); 3388 if (!sh) 3389 return NULL; 3390 3391 name = elf_sec_str(obj, sh->sh_name); 3392 if (!name) { 3393 pr_warn("elf: failed to get section(%zu) name from %s: %s\n", 3394 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 3395 return NULL; 3396 } 3397 3398 return name; 3399 } 3400 3401 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn) 3402 { 3403 Elf_Data *data; 3404 3405 if (!scn) 3406 return NULL; 3407 3408 data = elf_getdata(scn, 0); 3409 if (!data) { 3410 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n", 3411 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>", 3412 obj->path, elf_errmsg(-1)); 3413 return NULL; 3414 } 3415 3416 return data; 3417 } 3418 3419 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx) 3420 { 3421 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym)) 3422 return NULL; 3423 3424 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx; 3425 } 3426 3427 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx) 3428 { 3429 if (idx >= data->d_size / sizeof(Elf64_Rel)) 3430 return NULL; 3431 3432 return (Elf64_Rel *)data->d_buf + idx; 3433 } 3434 3435 static bool is_sec_name_dwarf(const char *name) 3436 { 3437 /* approximation, but the actual list is too long */ 3438 return str_has_pfx(name, ".debug_"); 3439 } 3440 3441 static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name) 3442 { 3443 /* no special handling of .strtab */ 3444 if (hdr->sh_type == SHT_STRTAB) 3445 return true; 3446 3447 /* ignore .llvm_addrsig section as well */ 3448 if (hdr->sh_type == SHT_LLVM_ADDRSIG) 3449 return true; 3450 3451 /* no subprograms will lead to an empty .text section, ignore it */ 3452 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 && 3453 strcmp(name, ".text") == 0) 3454 return true; 3455 3456 /* DWARF sections */ 3457 if (is_sec_name_dwarf(name)) 3458 return true; 3459 3460 if (str_has_pfx(name, ".rel")) { 3461 name += sizeof(".rel") - 1; 3462 /* DWARF section relocations */ 3463 if (is_sec_name_dwarf(name)) 3464 return true; 3465 3466 /* .BTF and .BTF.ext don't need relocations */ 3467 if (strcmp(name, BTF_ELF_SEC) == 0 || 3468 strcmp(name, BTF_EXT_ELF_SEC) == 0) 3469 return true; 3470 } 3471 3472 return false; 3473 } 3474 3475 static int cmp_progs(const void *_a, const void *_b) 3476 { 3477 const struct bpf_program *a = _a; 3478 const struct bpf_program *b = _b; 3479 3480 if (a->sec_idx != b->sec_idx) 3481 return a->sec_idx < b->sec_idx ? -1 : 1; 3482 3483 /* sec_insn_off can't be the same within the section */ 3484 return a->sec_insn_off < b->sec_insn_off ? -1 : 1; 3485 } 3486 3487 static int bpf_object__elf_collect(struct bpf_object *obj) 3488 { 3489 struct elf_sec_desc *sec_desc; 3490 Elf *elf = obj->efile.elf; 3491 Elf_Data *btf_ext_data = NULL; 3492 Elf_Data *btf_data = NULL; 3493 int idx = 0, err = 0; 3494 const char *name; 3495 Elf_Data *data; 3496 Elf_Scn *scn; 3497 Elf64_Shdr *sh; 3498 3499 /* ELF section indices are 0-based, but sec #0 is special "invalid" 3500 * section. Since section count retrieved by elf_getshdrnum() does 3501 * include sec #0, it is already the necessary size of an array to keep 3502 * all the sections. 3503 */ 3504 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) { 3505 pr_warn("elf: failed to get the number of sections for %s: %s\n", 3506 obj->path, elf_errmsg(-1)); 3507 return -LIBBPF_ERRNO__FORMAT; 3508 } 3509 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs)); 3510 if (!obj->efile.secs) 3511 return -ENOMEM; 3512 3513 /* a bunch of ELF parsing functionality depends on processing symbols, 3514 * so do the first pass and find the symbol table 3515 */ 3516 scn = NULL; 3517 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3518 sh = elf_sec_hdr(obj, scn); 3519 if (!sh) 3520 return -LIBBPF_ERRNO__FORMAT; 3521 3522 if (sh->sh_type == SHT_SYMTAB) { 3523 if (obj->efile.symbols) { 3524 pr_warn("elf: multiple symbol tables in %s\n", obj->path); 3525 return -LIBBPF_ERRNO__FORMAT; 3526 } 3527 3528 data = elf_sec_data(obj, scn); 3529 if (!data) 3530 return -LIBBPF_ERRNO__FORMAT; 3531 3532 idx = elf_ndxscn(scn); 3533 3534 obj->efile.symbols = data; 3535 obj->efile.symbols_shndx = idx; 3536 obj->efile.strtabidx = sh->sh_link; 3537 } 3538 } 3539 3540 if (!obj->efile.symbols) { 3541 pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n", 3542 obj->path); 3543 return -ENOENT; 3544 } 3545 3546 scn = NULL; 3547 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3548 idx = elf_ndxscn(scn); 3549 sec_desc = &obj->efile.secs[idx]; 3550 3551 sh = elf_sec_hdr(obj, scn); 3552 if (!sh) 3553 return -LIBBPF_ERRNO__FORMAT; 3554 3555 name = elf_sec_str(obj, sh->sh_name); 3556 if (!name) 3557 return -LIBBPF_ERRNO__FORMAT; 3558 3559 if (ignore_elf_section(sh, name)) 3560 continue; 3561 3562 data = elf_sec_data(obj, scn); 3563 if (!data) 3564 return -LIBBPF_ERRNO__FORMAT; 3565 3566 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 3567 idx, name, (unsigned long)data->d_size, 3568 (int)sh->sh_link, (unsigned long)sh->sh_flags, 3569 (int)sh->sh_type); 3570 3571 if (strcmp(name, "license") == 0) { 3572 err = bpf_object__init_license(obj, data->d_buf, data->d_size); 3573 if (err) 3574 return err; 3575 } else if (strcmp(name, "version") == 0) { 3576 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); 3577 if (err) 3578 return err; 3579 } else if (strcmp(name, "maps") == 0) { 3580 pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n"); 3581 return -ENOTSUP; 3582 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 3583 obj->efile.btf_maps_shndx = idx; 3584 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 3585 if (sh->sh_type != SHT_PROGBITS) 3586 return -LIBBPF_ERRNO__FORMAT; 3587 btf_data = data; 3588 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 3589 if (sh->sh_type != SHT_PROGBITS) 3590 return -LIBBPF_ERRNO__FORMAT; 3591 btf_ext_data = data; 3592 } else if (sh->sh_type == SHT_SYMTAB) { 3593 /* already processed during the first pass above */ 3594 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) { 3595 if (sh->sh_flags & SHF_EXECINSTR) { 3596 if (strcmp(name, ".text") == 0) 3597 obj->efile.text_shndx = idx; 3598 err = bpf_object__add_programs(obj, data, name, idx); 3599 if (err) 3600 return err; 3601 } else if (strcmp(name, DATA_SEC) == 0 || 3602 str_has_pfx(name, DATA_SEC ".")) { 3603 sec_desc->sec_type = SEC_DATA; 3604 sec_desc->shdr = sh; 3605 sec_desc->data = data; 3606 } else if (strcmp(name, RODATA_SEC) == 0 || 3607 str_has_pfx(name, RODATA_SEC ".")) { 3608 sec_desc->sec_type = SEC_RODATA; 3609 sec_desc->shdr = sh; 3610 sec_desc->data = data; 3611 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) { 3612 obj->efile.st_ops_data = data; 3613 obj->efile.st_ops_shndx = idx; 3614 } else if (strcmp(name, STRUCT_OPS_LINK_SEC) == 0) { 3615 obj->efile.st_ops_link_data = data; 3616 obj->efile.st_ops_link_shndx = idx; 3617 } else { 3618 pr_info("elf: skipping unrecognized data section(%d) %s\n", 3619 idx, name); 3620 } 3621 } else if (sh->sh_type == SHT_REL) { 3622 int targ_sec_idx = sh->sh_info; /* points to other section */ 3623 3624 if (sh->sh_entsize != sizeof(Elf64_Rel) || 3625 targ_sec_idx >= obj->efile.sec_cnt) 3626 return -LIBBPF_ERRNO__FORMAT; 3627 3628 /* Only do relo for section with exec instructions */ 3629 if (!section_have_execinstr(obj, targ_sec_idx) && 3630 strcmp(name, ".rel" STRUCT_OPS_SEC) && 3631 strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) && 3632 strcmp(name, ".rel" MAPS_ELF_SEC)) { 3633 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", 3634 idx, name, targ_sec_idx, 3635 elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>"); 3636 continue; 3637 } 3638 3639 sec_desc->sec_type = SEC_RELO; 3640 sec_desc->shdr = sh; 3641 sec_desc->data = data; 3642 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 || 3643 str_has_pfx(name, BSS_SEC "."))) { 3644 sec_desc->sec_type = SEC_BSS; 3645 sec_desc->shdr = sh; 3646 sec_desc->data = data; 3647 } else { 3648 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name, 3649 (size_t)sh->sh_size); 3650 } 3651 } 3652 3653 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { 3654 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); 3655 return -LIBBPF_ERRNO__FORMAT; 3656 } 3657 3658 /* sort BPF programs by section name and in-section instruction offset 3659 * for faster search 3660 */ 3661 if (obj->nr_programs) 3662 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); 3663 3664 return bpf_object__init_btf(obj, btf_data, btf_ext_data); 3665 } 3666 3667 static bool sym_is_extern(const Elf64_Sym *sym) 3668 { 3669 int bind = ELF64_ST_BIND(sym->st_info); 3670 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ 3671 return sym->st_shndx == SHN_UNDEF && 3672 (bind == STB_GLOBAL || bind == STB_WEAK) && 3673 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE; 3674 } 3675 3676 static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx) 3677 { 3678 int bind = ELF64_ST_BIND(sym->st_info); 3679 int type = ELF64_ST_TYPE(sym->st_info); 3680 3681 /* in .text section */ 3682 if (sym->st_shndx != text_shndx) 3683 return false; 3684 3685 /* local function */ 3686 if (bind == STB_LOCAL && type == STT_SECTION) 3687 return true; 3688 3689 /* global function */ 3690 return bind == STB_GLOBAL && type == STT_FUNC; 3691 } 3692 3693 static int find_extern_btf_id(const struct btf *btf, const char *ext_name) 3694 { 3695 const struct btf_type *t; 3696 const char *tname; 3697 int i, n; 3698 3699 if (!btf) 3700 return -ESRCH; 3701 3702 n = btf__type_cnt(btf); 3703 for (i = 1; i < n; i++) { 3704 t = btf__type_by_id(btf, i); 3705 3706 if (!btf_is_var(t) && !btf_is_func(t)) 3707 continue; 3708 3709 tname = btf__name_by_offset(btf, t->name_off); 3710 if (strcmp(tname, ext_name)) 3711 continue; 3712 3713 if (btf_is_var(t) && 3714 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) 3715 return -EINVAL; 3716 3717 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN) 3718 return -EINVAL; 3719 3720 return i; 3721 } 3722 3723 return -ENOENT; 3724 } 3725 3726 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) { 3727 const struct btf_var_secinfo *vs; 3728 const struct btf_type *t; 3729 int i, j, n; 3730 3731 if (!btf) 3732 return -ESRCH; 3733 3734 n = btf__type_cnt(btf); 3735 for (i = 1; i < n; i++) { 3736 t = btf__type_by_id(btf, i); 3737 3738 if (!btf_is_datasec(t)) 3739 continue; 3740 3741 vs = btf_var_secinfos(t); 3742 for (j = 0; j < btf_vlen(t); j++, vs++) { 3743 if (vs->type == ext_btf_id) 3744 return i; 3745 } 3746 } 3747 3748 return -ENOENT; 3749 } 3750 3751 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, 3752 bool *is_signed) 3753 { 3754 const struct btf_type *t; 3755 const char *name; 3756 3757 t = skip_mods_and_typedefs(btf, id, NULL); 3758 name = btf__name_by_offset(btf, t->name_off); 3759 3760 if (is_signed) 3761 *is_signed = false; 3762 switch (btf_kind(t)) { 3763 case BTF_KIND_INT: { 3764 int enc = btf_int_encoding(t); 3765 3766 if (enc & BTF_INT_BOOL) 3767 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; 3768 if (is_signed) 3769 *is_signed = enc & BTF_INT_SIGNED; 3770 if (t->size == 1) 3771 return KCFG_CHAR; 3772 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) 3773 return KCFG_UNKNOWN; 3774 return KCFG_INT; 3775 } 3776 case BTF_KIND_ENUM: 3777 if (t->size != 4) 3778 return KCFG_UNKNOWN; 3779 if (strcmp(name, "libbpf_tristate")) 3780 return KCFG_UNKNOWN; 3781 return KCFG_TRISTATE; 3782 case BTF_KIND_ENUM64: 3783 if (strcmp(name, "libbpf_tristate")) 3784 return KCFG_UNKNOWN; 3785 return KCFG_TRISTATE; 3786 case BTF_KIND_ARRAY: 3787 if (btf_array(t)->nelems == 0) 3788 return KCFG_UNKNOWN; 3789 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) 3790 return KCFG_UNKNOWN; 3791 return KCFG_CHAR_ARR; 3792 default: 3793 return KCFG_UNKNOWN; 3794 } 3795 } 3796 3797 static int cmp_externs(const void *_a, const void *_b) 3798 { 3799 const struct extern_desc *a = _a; 3800 const struct extern_desc *b = _b; 3801 3802 if (a->type != b->type) 3803 return a->type < b->type ? -1 : 1; 3804 3805 if (a->type == EXT_KCFG) { 3806 /* descending order by alignment requirements */ 3807 if (a->kcfg.align != b->kcfg.align) 3808 return a->kcfg.align > b->kcfg.align ? -1 : 1; 3809 /* ascending order by size, within same alignment class */ 3810 if (a->kcfg.sz != b->kcfg.sz) 3811 return a->kcfg.sz < b->kcfg.sz ? -1 : 1; 3812 } 3813 3814 /* resolve ties by name */ 3815 return strcmp(a->name, b->name); 3816 } 3817 3818 static int find_int_btf_id(const struct btf *btf) 3819 { 3820 const struct btf_type *t; 3821 int i, n; 3822 3823 n = btf__type_cnt(btf); 3824 for (i = 1; i < n; i++) { 3825 t = btf__type_by_id(btf, i); 3826 3827 if (btf_is_int(t) && btf_int_bits(t) == 32) 3828 return i; 3829 } 3830 3831 return 0; 3832 } 3833 3834 static int add_dummy_ksym_var(struct btf *btf) 3835 { 3836 int i, int_btf_id, sec_btf_id, dummy_var_btf_id; 3837 const struct btf_var_secinfo *vs; 3838 const struct btf_type *sec; 3839 3840 if (!btf) 3841 return 0; 3842 3843 sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC, 3844 BTF_KIND_DATASEC); 3845 if (sec_btf_id < 0) 3846 return 0; 3847 3848 sec = btf__type_by_id(btf, sec_btf_id); 3849 vs = btf_var_secinfos(sec); 3850 for (i = 0; i < btf_vlen(sec); i++, vs++) { 3851 const struct btf_type *vt; 3852 3853 vt = btf__type_by_id(btf, vs->type); 3854 if (btf_is_func(vt)) 3855 break; 3856 } 3857 3858 /* No func in ksyms sec. No need to add dummy var. */ 3859 if (i == btf_vlen(sec)) 3860 return 0; 3861 3862 int_btf_id = find_int_btf_id(btf); 3863 dummy_var_btf_id = btf__add_var(btf, 3864 "dummy_ksym", 3865 BTF_VAR_GLOBAL_ALLOCATED, 3866 int_btf_id); 3867 if (dummy_var_btf_id < 0) 3868 pr_warn("cannot create a dummy_ksym var\n"); 3869 3870 return dummy_var_btf_id; 3871 } 3872 3873 static int bpf_object__collect_externs(struct bpf_object *obj) 3874 { 3875 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL; 3876 const struct btf_type *t; 3877 struct extern_desc *ext; 3878 int i, n, off, dummy_var_btf_id; 3879 const char *ext_name, *sec_name; 3880 size_t ext_essent_len; 3881 Elf_Scn *scn; 3882 Elf64_Shdr *sh; 3883 3884 if (!obj->efile.symbols) 3885 return 0; 3886 3887 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); 3888 sh = elf_sec_hdr(obj, scn); 3889 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym)) 3890 return -LIBBPF_ERRNO__FORMAT; 3891 3892 dummy_var_btf_id = add_dummy_ksym_var(obj->btf); 3893 if (dummy_var_btf_id < 0) 3894 return dummy_var_btf_id; 3895 3896 n = sh->sh_size / sh->sh_entsize; 3897 pr_debug("looking for externs among %d symbols...\n", n); 3898 3899 for (i = 0; i < n; i++) { 3900 Elf64_Sym *sym = elf_sym_by_idx(obj, i); 3901 3902 if (!sym) 3903 return -LIBBPF_ERRNO__FORMAT; 3904 if (!sym_is_extern(sym)) 3905 continue; 3906 ext_name = elf_sym_str(obj, sym->st_name); 3907 if (!ext_name || !ext_name[0]) 3908 continue; 3909 3910 ext = obj->externs; 3911 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); 3912 if (!ext) 3913 return -ENOMEM; 3914 obj->externs = ext; 3915 ext = &ext[obj->nr_extern]; 3916 memset(ext, 0, sizeof(*ext)); 3917 obj->nr_extern++; 3918 3919 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); 3920 if (ext->btf_id <= 0) { 3921 pr_warn("failed to find BTF for extern '%s': %d\n", 3922 ext_name, ext->btf_id); 3923 return ext->btf_id; 3924 } 3925 t = btf__type_by_id(obj->btf, ext->btf_id); 3926 ext->name = btf__name_by_offset(obj->btf, t->name_off); 3927 ext->sym_idx = i; 3928 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK; 3929 3930 ext_essent_len = bpf_core_essential_name_len(ext->name); 3931 ext->essent_name = NULL; 3932 if (ext_essent_len != strlen(ext->name)) { 3933 ext->essent_name = strndup(ext->name, ext_essent_len); 3934 if (!ext->essent_name) 3935 return -ENOMEM; 3936 } 3937 3938 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); 3939 if (ext->sec_btf_id <= 0) { 3940 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n", 3941 ext_name, ext->btf_id, ext->sec_btf_id); 3942 return ext->sec_btf_id; 3943 } 3944 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); 3945 sec_name = btf__name_by_offset(obj->btf, sec->name_off); 3946 3947 if (strcmp(sec_name, KCONFIG_SEC) == 0) { 3948 if (btf_is_func(t)) { 3949 pr_warn("extern function %s is unsupported under %s section\n", 3950 ext->name, KCONFIG_SEC); 3951 return -ENOTSUP; 3952 } 3953 kcfg_sec = sec; 3954 ext->type = EXT_KCFG; 3955 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); 3956 if (ext->kcfg.sz <= 0) { 3957 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n", 3958 ext_name, ext->kcfg.sz); 3959 return ext->kcfg.sz; 3960 } 3961 ext->kcfg.align = btf__align_of(obj->btf, t->type); 3962 if (ext->kcfg.align <= 0) { 3963 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n", 3964 ext_name, ext->kcfg.align); 3965 return -EINVAL; 3966 } 3967 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, 3968 &ext->kcfg.is_signed); 3969 if (ext->kcfg.type == KCFG_UNKNOWN) { 3970 pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name); 3971 return -ENOTSUP; 3972 } 3973 } else if (strcmp(sec_name, KSYMS_SEC) == 0) { 3974 ksym_sec = sec; 3975 ext->type = EXT_KSYM; 3976 skip_mods_and_typedefs(obj->btf, t->type, 3977 &ext->ksym.type_id); 3978 } else { 3979 pr_warn("unrecognized extern section '%s'\n", sec_name); 3980 return -ENOTSUP; 3981 } 3982 } 3983 pr_debug("collected %d externs total\n", obj->nr_extern); 3984 3985 if (!obj->nr_extern) 3986 return 0; 3987 3988 /* sort externs by type, for kcfg ones also by (align, size, name) */ 3989 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); 3990 3991 /* for .ksyms section, we need to turn all externs into allocated 3992 * variables in BTF to pass kernel verification; we do this by 3993 * pretending that each extern is a 8-byte variable 3994 */ 3995 if (ksym_sec) { 3996 /* find existing 4-byte integer type in BTF to use for fake 3997 * extern variables in DATASEC 3998 */ 3999 int int_btf_id = find_int_btf_id(obj->btf); 4000 /* For extern function, a dummy_var added earlier 4001 * will be used to replace the vs->type and 4002 * its name string will be used to refill 4003 * the missing param's name. 4004 */ 4005 const struct btf_type *dummy_var; 4006 4007 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); 4008 for (i = 0; i < obj->nr_extern; i++) { 4009 ext = &obj->externs[i]; 4010 if (ext->type != EXT_KSYM) 4011 continue; 4012 pr_debug("extern (ksym) #%d: symbol %d, name %s\n", 4013 i, ext->sym_idx, ext->name); 4014 } 4015 4016 sec = ksym_sec; 4017 n = btf_vlen(sec); 4018 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) { 4019 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 4020 struct btf_type *vt; 4021 4022 vt = (void *)btf__type_by_id(obj->btf, vs->type); 4023 ext_name = btf__name_by_offset(obj->btf, vt->name_off); 4024 ext = find_extern_by_name(obj, ext_name); 4025 if (!ext) { 4026 pr_warn("failed to find extern definition for BTF %s '%s'\n", 4027 btf_kind_str(vt), ext_name); 4028 return -ESRCH; 4029 } 4030 if (btf_is_func(vt)) { 4031 const struct btf_type *func_proto; 4032 struct btf_param *param; 4033 int j; 4034 4035 func_proto = btf__type_by_id(obj->btf, 4036 vt->type); 4037 param = btf_params(func_proto); 4038 /* Reuse the dummy_var string if the 4039 * func proto does not have param name. 4040 */ 4041 for (j = 0; j < btf_vlen(func_proto); j++) 4042 if (param[j].type && !param[j].name_off) 4043 param[j].name_off = 4044 dummy_var->name_off; 4045 vs->type = dummy_var_btf_id; 4046 vt->info &= ~0xffff; 4047 vt->info |= BTF_FUNC_GLOBAL; 4048 } else { 4049 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 4050 vt->type = int_btf_id; 4051 } 4052 vs->offset = off; 4053 vs->size = sizeof(int); 4054 } 4055 sec->size = off; 4056 } 4057 4058 if (kcfg_sec) { 4059 sec = kcfg_sec; 4060 /* for kcfg externs calculate their offsets within a .kconfig map */ 4061 off = 0; 4062 for (i = 0; i < obj->nr_extern; i++) { 4063 ext = &obj->externs[i]; 4064 if (ext->type != EXT_KCFG) 4065 continue; 4066 4067 ext->kcfg.data_off = roundup(off, ext->kcfg.align); 4068 off = ext->kcfg.data_off + ext->kcfg.sz; 4069 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", 4070 i, ext->sym_idx, ext->kcfg.data_off, ext->name); 4071 } 4072 sec->size = off; 4073 n = btf_vlen(sec); 4074 for (i = 0; i < n; i++) { 4075 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 4076 4077 t = btf__type_by_id(obj->btf, vs->type); 4078 ext_name = btf__name_by_offset(obj->btf, t->name_off); 4079 ext = find_extern_by_name(obj, ext_name); 4080 if (!ext) { 4081 pr_warn("failed to find extern definition for BTF var '%s'\n", 4082 ext_name); 4083 return -ESRCH; 4084 } 4085 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 4086 vs->offset = ext->kcfg.data_off; 4087 } 4088 } 4089 return 0; 4090 } 4091 4092 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog) 4093 { 4094 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1; 4095 } 4096 4097 struct bpf_program * 4098 bpf_object__find_program_by_name(const struct bpf_object *obj, 4099 const char *name) 4100 { 4101 struct bpf_program *prog; 4102 4103 bpf_object__for_each_program(prog, obj) { 4104 if (prog_is_subprog(obj, prog)) 4105 continue; 4106 if (!strcmp(prog->name, name)) 4107 return prog; 4108 } 4109 return errno = ENOENT, NULL; 4110 } 4111 4112 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 4113 int shndx) 4114 { 4115 switch (obj->efile.secs[shndx].sec_type) { 4116 case SEC_BSS: 4117 case SEC_DATA: 4118 case SEC_RODATA: 4119 return true; 4120 default: 4121 return false; 4122 } 4123 } 4124 4125 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 4126 int shndx) 4127 { 4128 return shndx == obj->efile.btf_maps_shndx; 4129 } 4130 4131 static enum libbpf_map_type 4132 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 4133 { 4134 if (shndx == obj->efile.symbols_shndx) 4135 return LIBBPF_MAP_KCONFIG; 4136 4137 switch (obj->efile.secs[shndx].sec_type) { 4138 case SEC_BSS: 4139 return LIBBPF_MAP_BSS; 4140 case SEC_DATA: 4141 return LIBBPF_MAP_DATA; 4142 case SEC_RODATA: 4143 return LIBBPF_MAP_RODATA; 4144 default: 4145 return LIBBPF_MAP_UNSPEC; 4146 } 4147 } 4148 4149 static int bpf_program__record_reloc(struct bpf_program *prog, 4150 struct reloc_desc *reloc_desc, 4151 __u32 insn_idx, const char *sym_name, 4152 const Elf64_Sym *sym, const Elf64_Rel *rel) 4153 { 4154 struct bpf_insn *insn = &prog->insns[insn_idx]; 4155 size_t map_idx, nr_maps = prog->obj->nr_maps; 4156 struct bpf_object *obj = prog->obj; 4157 __u32 shdr_idx = sym->st_shndx; 4158 enum libbpf_map_type type; 4159 const char *sym_sec_name; 4160 struct bpf_map *map; 4161 4162 if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { 4163 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", 4164 prog->name, sym_name, insn_idx, insn->code); 4165 return -LIBBPF_ERRNO__RELOC; 4166 } 4167 4168 if (sym_is_extern(sym)) { 4169 int sym_idx = ELF64_R_SYM(rel->r_info); 4170 int i, n = obj->nr_extern; 4171 struct extern_desc *ext; 4172 4173 for (i = 0; i < n; i++) { 4174 ext = &obj->externs[i]; 4175 if (ext->sym_idx == sym_idx) 4176 break; 4177 } 4178 if (i >= n) { 4179 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n", 4180 prog->name, sym_name, sym_idx); 4181 return -LIBBPF_ERRNO__RELOC; 4182 } 4183 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n", 4184 prog->name, i, ext->name, ext->sym_idx, insn_idx); 4185 if (insn->code == (BPF_JMP | BPF_CALL)) 4186 reloc_desc->type = RELO_EXTERN_CALL; 4187 else 4188 reloc_desc->type = RELO_EXTERN_LD64; 4189 reloc_desc->insn_idx = insn_idx; 4190 reloc_desc->ext_idx = i; 4191 return 0; 4192 } 4193 4194 /* sub-program call relocation */ 4195 if (is_call_insn(insn)) { 4196 if (insn->src_reg != BPF_PSEUDO_CALL) { 4197 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); 4198 return -LIBBPF_ERRNO__RELOC; 4199 } 4200 /* text_shndx can be 0, if no default "main" program exists */ 4201 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { 4202 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 4203 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n", 4204 prog->name, sym_name, sym_sec_name); 4205 return -LIBBPF_ERRNO__RELOC; 4206 } 4207 if (sym->st_value % BPF_INSN_SZ) { 4208 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n", 4209 prog->name, sym_name, (size_t)sym->st_value); 4210 return -LIBBPF_ERRNO__RELOC; 4211 } 4212 reloc_desc->type = RELO_CALL; 4213 reloc_desc->insn_idx = insn_idx; 4214 reloc_desc->sym_off = sym->st_value; 4215 return 0; 4216 } 4217 4218 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { 4219 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n", 4220 prog->name, sym_name, shdr_idx); 4221 return -LIBBPF_ERRNO__RELOC; 4222 } 4223 4224 /* loading subprog addresses */ 4225 if (sym_is_subprog(sym, obj->efile.text_shndx)) { 4226 /* global_func: sym->st_value = offset in the section, insn->imm = 0. 4227 * local_func: sym->st_value = 0, insn->imm = offset in the section. 4228 */ 4229 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { 4230 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n", 4231 prog->name, sym_name, (size_t)sym->st_value, insn->imm); 4232 return -LIBBPF_ERRNO__RELOC; 4233 } 4234 4235 reloc_desc->type = RELO_SUBPROG_ADDR; 4236 reloc_desc->insn_idx = insn_idx; 4237 reloc_desc->sym_off = sym->st_value; 4238 return 0; 4239 } 4240 4241 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 4242 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 4243 4244 /* generic map reference relocation */ 4245 if (type == LIBBPF_MAP_UNSPEC) { 4246 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { 4247 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n", 4248 prog->name, sym_name, sym_sec_name); 4249 return -LIBBPF_ERRNO__RELOC; 4250 } 4251 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 4252 map = &obj->maps[map_idx]; 4253 if (map->libbpf_type != type || 4254 map->sec_idx != sym->st_shndx || 4255 map->sec_offset != sym->st_value) 4256 continue; 4257 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", 4258 prog->name, map_idx, map->name, map->sec_idx, 4259 map->sec_offset, insn_idx); 4260 break; 4261 } 4262 if (map_idx >= nr_maps) { 4263 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n", 4264 prog->name, sym_sec_name, (size_t)sym->st_value); 4265 return -LIBBPF_ERRNO__RELOC; 4266 } 4267 reloc_desc->type = RELO_LD64; 4268 reloc_desc->insn_idx = insn_idx; 4269 reloc_desc->map_idx = map_idx; 4270 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ 4271 return 0; 4272 } 4273 4274 /* global data map relocation */ 4275 if (!bpf_object__shndx_is_data(obj, shdr_idx)) { 4276 pr_warn("prog '%s': bad data relo against section '%s'\n", 4277 prog->name, sym_sec_name); 4278 return -LIBBPF_ERRNO__RELOC; 4279 } 4280 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 4281 map = &obj->maps[map_idx]; 4282 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx) 4283 continue; 4284 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", 4285 prog->name, map_idx, map->name, map->sec_idx, 4286 map->sec_offset, insn_idx); 4287 break; 4288 } 4289 if (map_idx >= nr_maps) { 4290 pr_warn("prog '%s': data relo failed to find map for section '%s'\n", 4291 prog->name, sym_sec_name); 4292 return -LIBBPF_ERRNO__RELOC; 4293 } 4294 4295 reloc_desc->type = RELO_DATA; 4296 reloc_desc->insn_idx = insn_idx; 4297 reloc_desc->map_idx = map_idx; 4298 reloc_desc->sym_off = sym->st_value; 4299 return 0; 4300 } 4301 4302 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx) 4303 { 4304 return insn_idx >= prog->sec_insn_off && 4305 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; 4306 } 4307 4308 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, 4309 size_t sec_idx, size_t insn_idx) 4310 { 4311 int l = 0, r = obj->nr_programs - 1, m; 4312 struct bpf_program *prog; 4313 4314 if (!obj->nr_programs) 4315 return NULL; 4316 4317 while (l < r) { 4318 m = l + (r - l + 1) / 2; 4319 prog = &obj->programs[m]; 4320 4321 if (prog->sec_idx < sec_idx || 4322 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) 4323 l = m; 4324 else 4325 r = m - 1; 4326 } 4327 /* matching program could be at index l, but it still might be the 4328 * wrong one, so we need to double check conditions for the last time 4329 */ 4330 prog = &obj->programs[l]; 4331 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) 4332 return prog; 4333 return NULL; 4334 } 4335 4336 static int 4337 bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) 4338 { 4339 const char *relo_sec_name, *sec_name; 4340 size_t sec_idx = shdr->sh_info, sym_idx; 4341 struct bpf_program *prog; 4342 struct reloc_desc *relos; 4343 int err, i, nrels; 4344 const char *sym_name; 4345 __u32 insn_idx; 4346 Elf_Scn *scn; 4347 Elf_Data *scn_data; 4348 Elf64_Sym *sym; 4349 Elf64_Rel *rel; 4350 4351 if (sec_idx >= obj->efile.sec_cnt) 4352 return -EINVAL; 4353 4354 scn = elf_sec_by_idx(obj, sec_idx); 4355 scn_data = elf_sec_data(obj, scn); 4356 if (!scn_data) 4357 return -LIBBPF_ERRNO__FORMAT; 4358 4359 relo_sec_name = elf_sec_str(obj, shdr->sh_name); 4360 sec_name = elf_sec_name(obj, scn); 4361 if (!relo_sec_name || !sec_name) 4362 return -EINVAL; 4363 4364 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n", 4365 relo_sec_name, sec_idx, sec_name); 4366 nrels = shdr->sh_size / shdr->sh_entsize; 4367 4368 for (i = 0; i < nrels; i++) { 4369 rel = elf_rel_by_idx(data, i); 4370 if (!rel) { 4371 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); 4372 return -LIBBPF_ERRNO__FORMAT; 4373 } 4374 4375 sym_idx = ELF64_R_SYM(rel->r_info); 4376 sym = elf_sym_by_idx(obj, sym_idx); 4377 if (!sym) { 4378 pr_warn("sec '%s': symbol #%zu not found for relo #%d\n", 4379 relo_sec_name, sym_idx, i); 4380 return -LIBBPF_ERRNO__FORMAT; 4381 } 4382 4383 if (sym->st_shndx >= obj->efile.sec_cnt) { 4384 pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n", 4385 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i); 4386 return -LIBBPF_ERRNO__FORMAT; 4387 } 4388 4389 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) { 4390 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n", 4391 relo_sec_name, (size_t)rel->r_offset, i); 4392 return -LIBBPF_ERRNO__FORMAT; 4393 } 4394 4395 insn_idx = rel->r_offset / BPF_INSN_SZ; 4396 /* relocations against static functions are recorded as 4397 * relocations against the section that contains a function; 4398 * in such case, symbol will be STT_SECTION and sym.st_name 4399 * will point to empty string (0), so fetch section name 4400 * instead 4401 */ 4402 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0) 4403 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)); 4404 else 4405 sym_name = elf_sym_str(obj, sym->st_name); 4406 sym_name = sym_name ?: "<?"; 4407 4408 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n", 4409 relo_sec_name, i, insn_idx, sym_name); 4410 4411 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 4412 if (!prog) { 4413 pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n", 4414 relo_sec_name, i, sec_name, insn_idx); 4415 continue; 4416 } 4417 4418 relos = libbpf_reallocarray(prog->reloc_desc, 4419 prog->nr_reloc + 1, sizeof(*relos)); 4420 if (!relos) 4421 return -ENOMEM; 4422 prog->reloc_desc = relos; 4423 4424 /* adjust insn_idx to local BPF program frame of reference */ 4425 insn_idx -= prog->sec_insn_off; 4426 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], 4427 insn_idx, sym_name, sym, rel); 4428 if (err) 4429 return err; 4430 4431 prog->nr_reloc++; 4432 } 4433 return 0; 4434 } 4435 4436 static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map) 4437 { 4438 int id; 4439 4440 if (!obj->btf) 4441 return -ENOENT; 4442 4443 /* if it's BTF-defined map, we don't need to search for type IDs. 4444 * For struct_ops map, it does not need btf_key_type_id and 4445 * btf_value_type_id. 4446 */ 4447 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map)) 4448 return 0; 4449 4450 /* 4451 * LLVM annotates global data differently in BTF, that is, 4452 * only as '.data', '.bss' or '.rodata'. 4453 */ 4454 if (!bpf_map__is_internal(map)) 4455 return -ENOENT; 4456 4457 id = btf__find_by_name(obj->btf, map->real_name); 4458 if (id < 0) 4459 return id; 4460 4461 map->btf_key_type_id = 0; 4462 map->btf_value_type_id = id; 4463 return 0; 4464 } 4465 4466 static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) 4467 { 4468 char file[PATH_MAX], buff[4096]; 4469 FILE *fp; 4470 __u32 val; 4471 int err; 4472 4473 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd); 4474 memset(info, 0, sizeof(*info)); 4475 4476 fp = fopen(file, "re"); 4477 if (!fp) { 4478 err = -errno; 4479 pr_warn("failed to open %s: %d. No procfs support?\n", file, 4480 err); 4481 return err; 4482 } 4483 4484 while (fgets(buff, sizeof(buff), fp)) { 4485 if (sscanf(buff, "map_type:\t%u", &val) == 1) 4486 info->type = val; 4487 else if (sscanf(buff, "key_size:\t%u", &val) == 1) 4488 info->key_size = val; 4489 else if (sscanf(buff, "value_size:\t%u", &val) == 1) 4490 info->value_size = val; 4491 else if (sscanf(buff, "max_entries:\t%u", &val) == 1) 4492 info->max_entries = val; 4493 else if (sscanf(buff, "map_flags:\t%i", &val) == 1) 4494 info->map_flags = val; 4495 } 4496 4497 fclose(fp); 4498 4499 return 0; 4500 } 4501 4502 bool bpf_map__autocreate(const struct bpf_map *map) 4503 { 4504 return map->autocreate; 4505 } 4506 4507 int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) 4508 { 4509 if (map->obj->loaded) 4510 return libbpf_err(-EBUSY); 4511 4512 map->autocreate = autocreate; 4513 return 0; 4514 } 4515 4516 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 4517 { 4518 struct bpf_map_info info; 4519 __u32 len = sizeof(info), name_len; 4520 int new_fd, err; 4521 char *new_name; 4522 4523 memset(&info, 0, len); 4524 err = bpf_map_get_info_by_fd(fd, &info, &len); 4525 if (err && errno == EINVAL) 4526 err = bpf_get_map_info_from_fdinfo(fd, &info); 4527 if (err) 4528 return libbpf_err(err); 4529 4530 name_len = strlen(info.name); 4531 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0) 4532 new_name = strdup(map->name); 4533 else 4534 new_name = strdup(info.name); 4535 4536 if (!new_name) 4537 return libbpf_err(-errno); 4538 4539 /* 4540 * Like dup(), but make sure new FD is >= 3 and has O_CLOEXEC set. 4541 * This is similar to what we do in ensure_good_fd(), but without 4542 * closing original FD. 4543 */ 4544 new_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); 4545 if (new_fd < 0) { 4546 err = -errno; 4547 goto err_free_new_name; 4548 } 4549 4550 err = reuse_fd(map->fd, new_fd); 4551 if (err) 4552 goto err_free_new_name; 4553 4554 free(map->name); 4555 4556 map->name = new_name; 4557 map->def.type = info.type; 4558 map->def.key_size = info.key_size; 4559 map->def.value_size = info.value_size; 4560 map->def.max_entries = info.max_entries; 4561 map->def.map_flags = info.map_flags; 4562 map->btf_key_type_id = info.btf_key_type_id; 4563 map->btf_value_type_id = info.btf_value_type_id; 4564 map->reused = true; 4565 map->map_extra = info.map_extra; 4566 4567 return 0; 4568 4569 err_free_new_name: 4570 free(new_name); 4571 return libbpf_err(err); 4572 } 4573 4574 __u32 bpf_map__max_entries(const struct bpf_map *map) 4575 { 4576 return map->def.max_entries; 4577 } 4578 4579 struct bpf_map *bpf_map__inner_map(struct bpf_map *map) 4580 { 4581 if (!bpf_map_type__is_map_in_map(map->def.type)) 4582 return errno = EINVAL, NULL; 4583 4584 return map->inner_map; 4585 } 4586 4587 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) 4588 { 4589 if (map->obj->loaded) 4590 return libbpf_err(-EBUSY); 4591 4592 map->def.max_entries = max_entries; 4593 4594 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ 4595 if (map_is_ringbuf(map)) 4596 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); 4597 4598 return 0; 4599 } 4600 4601 static int bpf_object_prepare_token(struct bpf_object *obj) 4602 { 4603 const char *bpffs_path; 4604 int bpffs_fd = -1, token_fd, err; 4605 bool mandatory; 4606 enum libbpf_print_level level; 4607 4608 /* token is explicitly prevented */ 4609 if (obj->token_path && obj->token_path[0] == '\0') { 4610 pr_debug("object '%s': token is prevented, skipping...\n", obj->name); 4611 return 0; 4612 } 4613 4614 mandatory = obj->token_path != NULL; 4615 level = mandatory ? LIBBPF_WARN : LIBBPF_DEBUG; 4616 4617 bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH; 4618 bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR); 4619 if (bpffs_fd < 0) { 4620 err = -errno; 4621 __pr(level, "object '%s': failed (%d) to open BPF FS mount at '%s'%s\n", 4622 obj->name, err, bpffs_path, 4623 mandatory ? "" : ", skipping optional step..."); 4624 return mandatory ? err : 0; 4625 } 4626 4627 token_fd = bpf_token_create(bpffs_fd, 0); 4628 close(bpffs_fd); 4629 if (token_fd < 0) { 4630 if (!mandatory && token_fd == -ENOENT) { 4631 pr_debug("object '%s': BPF FS at '%s' doesn't have BPF token delegation set up, skipping...\n", 4632 obj->name, bpffs_path); 4633 return 0; 4634 } 4635 __pr(level, "object '%s': failed (%d) to create BPF token from '%s'%s\n", 4636 obj->name, token_fd, bpffs_path, 4637 mandatory ? "" : ", skipping optional step..."); 4638 return mandatory ? token_fd : 0; 4639 } 4640 4641 obj->feat_cache = calloc(1, sizeof(*obj->feat_cache)); 4642 if (!obj->feat_cache) { 4643 close(token_fd); 4644 return -ENOMEM; 4645 } 4646 4647 obj->token_fd = token_fd; 4648 obj->feat_cache->token_fd = token_fd; 4649 4650 return 0; 4651 } 4652 4653 static int 4654 bpf_object__probe_loading(struct bpf_object *obj) 4655 { 4656 char *cp, errmsg[STRERR_BUFSIZE]; 4657 struct bpf_insn insns[] = { 4658 BPF_MOV64_IMM(BPF_REG_0, 0), 4659 BPF_EXIT_INSN(), 4660 }; 4661 int ret, insn_cnt = ARRAY_SIZE(insns); 4662 LIBBPF_OPTS(bpf_prog_load_opts, opts, 4663 .token_fd = obj->token_fd, 4664 .prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0, 4665 ); 4666 4667 if (obj->gen_loader) 4668 return 0; 4669 4670 ret = bump_rlimit_memlock(); 4671 if (ret) 4672 pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret); 4673 4674 /* make sure basic loading works */ 4675 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts); 4676 if (ret < 0) 4677 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); 4678 if (ret < 0) { 4679 ret = errno; 4680 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4681 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF " 4682 "program. Make sure your kernel supports BPF " 4683 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is " 4684 "set to big enough value.\n", __func__, cp, ret); 4685 return -ret; 4686 } 4687 close(ret); 4688 4689 return 0; 4690 } 4691 4692 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) 4693 { 4694 if (obj->gen_loader) 4695 /* To generate loader program assume the latest kernel 4696 * to avoid doing extra prog_load, map_create syscalls. 4697 */ 4698 return true; 4699 4700 if (obj->token_fd) 4701 return feat_supported(obj->feat_cache, feat_id); 4702 4703 return feat_supported(NULL, feat_id); 4704 } 4705 4706 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) 4707 { 4708 struct bpf_map_info map_info; 4709 char msg[STRERR_BUFSIZE]; 4710 __u32 map_info_len = sizeof(map_info); 4711 int err; 4712 4713 memset(&map_info, 0, map_info_len); 4714 err = bpf_map_get_info_by_fd(map_fd, &map_info, &map_info_len); 4715 if (err && errno == EINVAL) 4716 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info); 4717 if (err) { 4718 pr_warn("failed to get map info for map FD %d: %s\n", map_fd, 4719 libbpf_strerror_r(errno, msg, sizeof(msg))); 4720 return false; 4721 } 4722 4723 return (map_info.type == map->def.type && 4724 map_info.key_size == map->def.key_size && 4725 map_info.value_size == map->def.value_size && 4726 map_info.max_entries == map->def.max_entries && 4727 map_info.map_flags == map->def.map_flags && 4728 map_info.map_extra == map->map_extra); 4729 } 4730 4731 static int 4732 bpf_object__reuse_map(struct bpf_map *map) 4733 { 4734 char *cp, errmsg[STRERR_BUFSIZE]; 4735 int err, pin_fd; 4736 4737 pin_fd = bpf_obj_get(map->pin_path); 4738 if (pin_fd < 0) { 4739 err = -errno; 4740 if (err == -ENOENT) { 4741 pr_debug("found no pinned map to reuse at '%s'\n", 4742 map->pin_path); 4743 return 0; 4744 } 4745 4746 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 4747 pr_warn("couldn't retrieve pinned map '%s': %s\n", 4748 map->pin_path, cp); 4749 return err; 4750 } 4751 4752 if (!map_is_reuse_compat(map, pin_fd)) { 4753 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", 4754 map->pin_path); 4755 close(pin_fd); 4756 return -EINVAL; 4757 } 4758 4759 err = bpf_map__reuse_fd(map, pin_fd); 4760 close(pin_fd); 4761 if (err) 4762 return err; 4763 4764 map->pinned = true; 4765 pr_debug("reused pinned map at '%s'\n", map->pin_path); 4766 4767 return 0; 4768 } 4769 4770 static int 4771 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 4772 { 4773 enum libbpf_map_type map_type = map->libbpf_type; 4774 char *cp, errmsg[STRERR_BUFSIZE]; 4775 int err, zero = 0; 4776 4777 if (obj->gen_loader) { 4778 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, 4779 map->mmaped, map->def.value_size); 4780 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) 4781 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); 4782 return 0; 4783 } 4784 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); 4785 if (err) { 4786 err = -errno; 4787 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4788 pr_warn("Error setting initial map(%s) contents: %s\n", 4789 map->name, cp); 4790 return err; 4791 } 4792 4793 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ 4794 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { 4795 err = bpf_map_freeze(map->fd); 4796 if (err) { 4797 err = -errno; 4798 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4799 pr_warn("Error freezing map(%s) as read-only: %s\n", 4800 map->name, cp); 4801 return err; 4802 } 4803 } 4804 return 0; 4805 } 4806 4807 static void bpf_map__destroy(struct bpf_map *map); 4808 4809 static bool map_is_created(const struct bpf_map *map) 4810 { 4811 return map->obj->loaded || map->reused; 4812 } 4813 4814 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner) 4815 { 4816 LIBBPF_OPTS(bpf_map_create_opts, create_attr); 4817 struct bpf_map_def *def = &map->def; 4818 const char *map_name = NULL; 4819 int err = 0, map_fd; 4820 4821 if (kernel_supports(obj, FEAT_PROG_NAME)) 4822 map_name = map->name; 4823 create_attr.map_ifindex = map->map_ifindex; 4824 create_attr.map_flags = def->map_flags; 4825 create_attr.numa_node = map->numa_node; 4826 create_attr.map_extra = map->map_extra; 4827 create_attr.token_fd = obj->token_fd; 4828 if (obj->token_fd) 4829 create_attr.map_flags |= BPF_F_TOKEN_FD; 4830 4831 if (bpf_map__is_struct_ops(map)) { 4832 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 4833 if (map->mod_btf_fd >= 0) { 4834 create_attr.value_type_btf_obj_fd = map->mod_btf_fd; 4835 create_attr.map_flags |= BPF_F_VTYPE_BTF_OBJ_FD; 4836 } 4837 } 4838 4839 if (obj->btf && btf__fd(obj->btf) >= 0) { 4840 create_attr.btf_fd = btf__fd(obj->btf); 4841 create_attr.btf_key_type_id = map->btf_key_type_id; 4842 create_attr.btf_value_type_id = map->btf_value_type_id; 4843 } 4844 4845 if (bpf_map_type__is_map_in_map(def->type)) { 4846 if (map->inner_map) { 4847 err = map_set_def_max_entries(map->inner_map); 4848 if (err) 4849 return err; 4850 err = bpf_object__create_map(obj, map->inner_map, true); 4851 if (err) { 4852 pr_warn("map '%s': failed to create inner map: %d\n", 4853 map->name, err); 4854 return err; 4855 } 4856 map->inner_map_fd = map->inner_map->fd; 4857 } 4858 if (map->inner_map_fd >= 0) 4859 create_attr.inner_map_fd = map->inner_map_fd; 4860 } 4861 4862 switch (def->type) { 4863 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 4864 case BPF_MAP_TYPE_CGROUP_ARRAY: 4865 case BPF_MAP_TYPE_STACK_TRACE: 4866 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 4867 case BPF_MAP_TYPE_HASH_OF_MAPS: 4868 case BPF_MAP_TYPE_DEVMAP: 4869 case BPF_MAP_TYPE_DEVMAP_HASH: 4870 case BPF_MAP_TYPE_CPUMAP: 4871 case BPF_MAP_TYPE_XSKMAP: 4872 case BPF_MAP_TYPE_SOCKMAP: 4873 case BPF_MAP_TYPE_SOCKHASH: 4874 case BPF_MAP_TYPE_QUEUE: 4875 case BPF_MAP_TYPE_STACK: 4876 create_attr.btf_fd = 0; 4877 create_attr.btf_key_type_id = 0; 4878 create_attr.btf_value_type_id = 0; 4879 map->btf_key_type_id = 0; 4880 map->btf_value_type_id = 0; 4881 break; 4882 case BPF_MAP_TYPE_STRUCT_OPS: 4883 create_attr.btf_value_type_id = 0; 4884 break; 4885 default: 4886 break; 4887 } 4888 4889 if (obj->gen_loader) { 4890 bpf_gen__map_create(obj->gen_loader, def->type, map_name, 4891 def->key_size, def->value_size, def->max_entries, 4892 &create_attr, is_inner ? -1 : map - obj->maps); 4893 /* We keep pretenting we have valid FD to pass various fd >= 0 4894 * checks by just keeping original placeholder FDs in place. 4895 * See bpf_object__add_map() comment. 4896 * This placeholder fd will not be used with any syscall and 4897 * will be reset to -1 eventually. 4898 */ 4899 map_fd = map->fd; 4900 } else { 4901 map_fd = bpf_map_create(def->type, map_name, 4902 def->key_size, def->value_size, 4903 def->max_entries, &create_attr); 4904 } 4905 if (map_fd < 0 && (create_attr.btf_key_type_id || create_attr.btf_value_type_id)) { 4906 char *cp, errmsg[STRERR_BUFSIZE]; 4907 4908 err = -errno; 4909 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 4910 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 4911 map->name, cp, err); 4912 create_attr.btf_fd = 0; 4913 create_attr.btf_key_type_id = 0; 4914 create_attr.btf_value_type_id = 0; 4915 map->btf_key_type_id = 0; 4916 map->btf_value_type_id = 0; 4917 map_fd = bpf_map_create(def->type, map_name, 4918 def->key_size, def->value_size, 4919 def->max_entries, &create_attr); 4920 } 4921 4922 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { 4923 if (obj->gen_loader) 4924 map->inner_map->fd = -1; 4925 bpf_map__destroy(map->inner_map); 4926 zfree(&map->inner_map); 4927 } 4928 4929 if (map_fd < 0) 4930 return map_fd; 4931 4932 /* obj->gen_loader case, prevent reuse_fd() from closing map_fd */ 4933 if (map->fd == map_fd) 4934 return 0; 4935 4936 /* Keep placeholder FD value but now point it to the BPF map object. 4937 * This way everything that relied on this map's FD (e.g., relocated 4938 * ldimm64 instructions) will stay valid and won't need adjustments. 4939 * map->fd stays valid but now point to what map_fd points to. 4940 */ 4941 return reuse_fd(map->fd, map_fd); 4942 } 4943 4944 static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) 4945 { 4946 const struct bpf_map *targ_map; 4947 unsigned int i; 4948 int fd, err = 0; 4949 4950 for (i = 0; i < map->init_slots_sz; i++) { 4951 if (!map->init_slots[i]) 4952 continue; 4953 4954 targ_map = map->init_slots[i]; 4955 fd = targ_map->fd; 4956 4957 if (obj->gen_loader) { 4958 bpf_gen__populate_outer_map(obj->gen_loader, 4959 map - obj->maps, i, 4960 targ_map - obj->maps); 4961 } else { 4962 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 4963 } 4964 if (err) { 4965 err = -errno; 4966 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n", 4967 map->name, i, targ_map->name, fd, err); 4968 return err; 4969 } 4970 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", 4971 map->name, i, targ_map->name, fd); 4972 } 4973 4974 zfree(&map->init_slots); 4975 map->init_slots_sz = 0; 4976 4977 return 0; 4978 } 4979 4980 static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) 4981 { 4982 const struct bpf_program *targ_prog; 4983 unsigned int i; 4984 int fd, err; 4985 4986 if (obj->gen_loader) 4987 return -ENOTSUP; 4988 4989 for (i = 0; i < map->init_slots_sz; i++) { 4990 if (!map->init_slots[i]) 4991 continue; 4992 4993 targ_prog = map->init_slots[i]; 4994 fd = bpf_program__fd(targ_prog); 4995 4996 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 4997 if (err) { 4998 err = -errno; 4999 pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n", 5000 map->name, i, targ_prog->name, fd, err); 5001 return err; 5002 } 5003 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n", 5004 map->name, i, targ_prog->name, fd); 5005 } 5006 5007 zfree(&map->init_slots); 5008 map->init_slots_sz = 0; 5009 5010 return 0; 5011 } 5012 5013 static int bpf_object_init_prog_arrays(struct bpf_object *obj) 5014 { 5015 struct bpf_map *map; 5016 int i, err; 5017 5018 for (i = 0; i < obj->nr_maps; i++) { 5019 map = &obj->maps[i]; 5020 5021 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY) 5022 continue; 5023 5024 err = init_prog_array_slots(obj, map); 5025 if (err < 0) 5026 return err; 5027 } 5028 return 0; 5029 } 5030 5031 static int map_set_def_max_entries(struct bpf_map *map) 5032 { 5033 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) { 5034 int nr_cpus; 5035 5036 nr_cpus = libbpf_num_possible_cpus(); 5037 if (nr_cpus < 0) { 5038 pr_warn("map '%s': failed to determine number of system CPUs: %d\n", 5039 map->name, nr_cpus); 5040 return nr_cpus; 5041 } 5042 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); 5043 map->def.max_entries = nr_cpus; 5044 } 5045 5046 return 0; 5047 } 5048 5049 static int 5050 bpf_object__create_maps(struct bpf_object *obj) 5051 { 5052 struct bpf_map *map; 5053 char *cp, errmsg[STRERR_BUFSIZE]; 5054 unsigned int i, j; 5055 int err; 5056 bool retried; 5057 5058 for (i = 0; i < obj->nr_maps; i++) { 5059 map = &obj->maps[i]; 5060 5061 /* To support old kernels, we skip creating global data maps 5062 * (.rodata, .data, .kconfig, etc); later on, during program 5063 * loading, if we detect that at least one of the to-be-loaded 5064 * programs is referencing any global data map, we'll error 5065 * out with program name and relocation index logged. 5066 * This approach allows to accommodate Clang emitting 5067 * unnecessary .rodata.str1.1 sections for string literals, 5068 * but also it allows to have CO-RE applications that use 5069 * global variables in some of BPF programs, but not others. 5070 * If those global variable-using programs are not loaded at 5071 * runtime due to bpf_program__set_autoload(prog, false), 5072 * bpf_object loading will succeed just fine even on old 5073 * kernels. 5074 */ 5075 if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA)) 5076 map->autocreate = false; 5077 5078 if (!map->autocreate) { 5079 pr_debug("map '%s': skipped auto-creating...\n", map->name); 5080 continue; 5081 } 5082 5083 err = map_set_def_max_entries(map); 5084 if (err) 5085 goto err_out; 5086 5087 retried = false; 5088 retry: 5089 if (map->pin_path) { 5090 err = bpf_object__reuse_map(map); 5091 if (err) { 5092 pr_warn("map '%s': error reusing pinned map\n", 5093 map->name); 5094 goto err_out; 5095 } 5096 if (retried && map->fd < 0) { 5097 pr_warn("map '%s': cannot find pinned map\n", 5098 map->name); 5099 err = -ENOENT; 5100 goto err_out; 5101 } 5102 } 5103 5104 if (map->reused) { 5105 pr_debug("map '%s': skipping creation (preset fd=%d)\n", 5106 map->name, map->fd); 5107 } else { 5108 err = bpf_object__create_map(obj, map, false); 5109 if (err) 5110 goto err_out; 5111 5112 pr_debug("map '%s': created successfully, fd=%d\n", 5113 map->name, map->fd); 5114 5115 if (bpf_map__is_internal(map)) { 5116 err = bpf_object__populate_internal_map(obj, map); 5117 if (err < 0) 5118 goto err_out; 5119 } 5120 5121 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { 5122 err = init_map_in_map_slots(obj, map); 5123 if (err < 0) 5124 goto err_out; 5125 } 5126 } 5127 5128 if (map->pin_path && !map->pinned) { 5129 err = bpf_map__pin(map, NULL); 5130 if (err) { 5131 if (!retried && err == -EEXIST) { 5132 retried = true; 5133 goto retry; 5134 } 5135 pr_warn("map '%s': failed to auto-pin at '%s': %d\n", 5136 map->name, map->pin_path, err); 5137 goto err_out; 5138 } 5139 } 5140 } 5141 5142 return 0; 5143 5144 err_out: 5145 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 5146 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err); 5147 pr_perm_msg(err); 5148 for (j = 0; j < i; j++) 5149 zclose(obj->maps[j].fd); 5150 return err; 5151 } 5152 5153 static bool bpf_core_is_flavor_sep(const char *s) 5154 { 5155 /* check X___Y name pattern, where X and Y are not underscores */ 5156 return s[0] != '_' && /* X */ 5157 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 5158 s[4] != '_'; /* Y */ 5159 } 5160 5161 /* Given 'some_struct_name___with_flavor' return the length of a name prefix 5162 * before last triple underscore. Struct name part after last triple 5163 * underscore is ignored by BPF CO-RE relocation during relocation matching. 5164 */ 5165 size_t bpf_core_essential_name_len(const char *name) 5166 { 5167 size_t n = strlen(name); 5168 int i; 5169 5170 for (i = n - 5; i >= 0; i--) { 5171 if (bpf_core_is_flavor_sep(name + i)) 5172 return i + 1; 5173 } 5174 return n; 5175 } 5176 5177 void bpf_core_free_cands(struct bpf_core_cand_list *cands) 5178 { 5179 if (!cands) 5180 return; 5181 5182 free(cands->cands); 5183 free(cands); 5184 } 5185 5186 int bpf_core_add_cands(struct bpf_core_cand *local_cand, 5187 size_t local_essent_len, 5188 const struct btf *targ_btf, 5189 const char *targ_btf_name, 5190 int targ_start_id, 5191 struct bpf_core_cand_list *cands) 5192 { 5193 struct bpf_core_cand *new_cands, *cand; 5194 const struct btf_type *t, *local_t; 5195 const char *targ_name, *local_name; 5196 size_t targ_essent_len; 5197 int n, i; 5198 5199 local_t = btf__type_by_id(local_cand->btf, local_cand->id); 5200 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off); 5201 5202 n = btf__type_cnt(targ_btf); 5203 for (i = targ_start_id; i < n; i++) { 5204 t = btf__type_by_id(targ_btf, i); 5205 if (!btf_kind_core_compat(t, local_t)) 5206 continue; 5207 5208 targ_name = btf__name_by_offset(targ_btf, t->name_off); 5209 if (str_is_empty(targ_name)) 5210 continue; 5211 5212 targ_essent_len = bpf_core_essential_name_len(targ_name); 5213 if (targ_essent_len != local_essent_len) 5214 continue; 5215 5216 if (strncmp(local_name, targ_name, local_essent_len) != 0) 5217 continue; 5218 5219 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", 5220 local_cand->id, btf_kind_str(local_t), 5221 local_name, i, btf_kind_str(t), targ_name, 5222 targ_btf_name); 5223 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, 5224 sizeof(*cands->cands)); 5225 if (!new_cands) 5226 return -ENOMEM; 5227 5228 cand = &new_cands[cands->len]; 5229 cand->btf = targ_btf; 5230 cand->id = i; 5231 5232 cands->cands = new_cands; 5233 cands->len++; 5234 } 5235 return 0; 5236 } 5237 5238 static int load_module_btfs(struct bpf_object *obj) 5239 { 5240 struct bpf_btf_info info; 5241 struct module_btf *mod_btf; 5242 struct btf *btf; 5243 char name[64]; 5244 __u32 id = 0, len; 5245 int err, fd; 5246 5247 if (obj->btf_modules_loaded) 5248 return 0; 5249 5250 if (obj->gen_loader) 5251 return 0; 5252 5253 /* don't do this again, even if we find no module BTFs */ 5254 obj->btf_modules_loaded = true; 5255 5256 /* kernel too old to support module BTFs */ 5257 if (!kernel_supports(obj, FEAT_MODULE_BTF)) 5258 return 0; 5259 5260 while (true) { 5261 err = bpf_btf_get_next_id(id, &id); 5262 if (err && errno == ENOENT) 5263 return 0; 5264 if (err && errno == EPERM) { 5265 pr_debug("skipping module BTFs loading, missing privileges\n"); 5266 return 0; 5267 } 5268 if (err) { 5269 err = -errno; 5270 pr_warn("failed to iterate BTF objects: %d\n", err); 5271 return err; 5272 } 5273 5274 fd = bpf_btf_get_fd_by_id(id); 5275 if (fd < 0) { 5276 if (errno == ENOENT) 5277 continue; /* expected race: BTF was unloaded */ 5278 err = -errno; 5279 pr_warn("failed to get BTF object #%d FD: %d\n", id, err); 5280 return err; 5281 } 5282 5283 len = sizeof(info); 5284 memset(&info, 0, sizeof(info)); 5285 info.name = ptr_to_u64(name); 5286 info.name_len = sizeof(name); 5287 5288 err = bpf_btf_get_info_by_fd(fd, &info, &len); 5289 if (err) { 5290 err = -errno; 5291 pr_warn("failed to get BTF object #%d info: %d\n", id, err); 5292 goto err_out; 5293 } 5294 5295 /* ignore non-module BTFs */ 5296 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) { 5297 close(fd); 5298 continue; 5299 } 5300 5301 btf = btf_get_from_fd(fd, obj->btf_vmlinux); 5302 err = libbpf_get_error(btf); 5303 if (err) { 5304 pr_warn("failed to load module [%s]'s BTF object #%d: %d\n", 5305 name, id, err); 5306 goto err_out; 5307 } 5308 5309 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, 5310 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); 5311 if (err) 5312 goto err_out; 5313 5314 mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; 5315 5316 mod_btf->btf = btf; 5317 mod_btf->id = id; 5318 mod_btf->fd = fd; 5319 mod_btf->name = strdup(name); 5320 if (!mod_btf->name) { 5321 err = -ENOMEM; 5322 goto err_out; 5323 } 5324 continue; 5325 5326 err_out: 5327 close(fd); 5328 return err; 5329 } 5330 5331 return 0; 5332 } 5333 5334 static struct bpf_core_cand_list * 5335 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id) 5336 { 5337 struct bpf_core_cand local_cand = {}; 5338 struct bpf_core_cand_list *cands; 5339 const struct btf *main_btf; 5340 const struct btf_type *local_t; 5341 const char *local_name; 5342 size_t local_essent_len; 5343 int err, i; 5344 5345 local_cand.btf = local_btf; 5346 local_cand.id = local_type_id; 5347 local_t = btf__type_by_id(local_btf, local_type_id); 5348 if (!local_t) 5349 return ERR_PTR(-EINVAL); 5350 5351 local_name = btf__name_by_offset(local_btf, local_t->name_off); 5352 if (str_is_empty(local_name)) 5353 return ERR_PTR(-EINVAL); 5354 local_essent_len = bpf_core_essential_name_len(local_name); 5355 5356 cands = calloc(1, sizeof(*cands)); 5357 if (!cands) 5358 return ERR_PTR(-ENOMEM); 5359 5360 /* Attempt to find target candidates in vmlinux BTF first */ 5361 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; 5362 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands); 5363 if (err) 5364 goto err_out; 5365 5366 /* if vmlinux BTF has any candidate, don't got for module BTFs */ 5367 if (cands->len) 5368 return cands; 5369 5370 /* if vmlinux BTF was overridden, don't attempt to load module BTFs */ 5371 if (obj->btf_vmlinux_override) 5372 return cands; 5373 5374 /* now look through module BTFs, trying to still find candidates */ 5375 err = load_module_btfs(obj); 5376 if (err) 5377 goto err_out; 5378 5379 for (i = 0; i < obj->btf_module_cnt; i++) { 5380 err = bpf_core_add_cands(&local_cand, local_essent_len, 5381 obj->btf_modules[i].btf, 5382 obj->btf_modules[i].name, 5383 btf__type_cnt(obj->btf_vmlinux), 5384 cands); 5385 if (err) 5386 goto err_out; 5387 } 5388 5389 return cands; 5390 err_out: 5391 bpf_core_free_cands(cands); 5392 return ERR_PTR(err); 5393 } 5394 5395 /* Check local and target types for compatibility. This check is used for 5396 * type-based CO-RE relocations and follow slightly different rules than 5397 * field-based relocations. This function assumes that root types were already 5398 * checked for name match. Beyond that initial root-level name check, names 5399 * are completely ignored. Compatibility rules are as follows: 5400 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but 5401 * kind should match for local and target types (i.e., STRUCT is not 5402 * compatible with UNION); 5403 * - for ENUMs, the size is ignored; 5404 * - for INT, size and signedness are ignored; 5405 * - for ARRAY, dimensionality is ignored, element types are checked for 5406 * compatibility recursively; 5407 * - CONST/VOLATILE/RESTRICT modifiers are ignored; 5408 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; 5409 * - FUNC_PROTOs are compatible if they have compatible signature: same 5410 * number of input args and compatible return and argument types. 5411 * These rules are not set in stone and probably will be adjusted as we get 5412 * more experience with using BPF CO-RE relocations. 5413 */ 5414 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, 5415 const struct btf *targ_btf, __u32 targ_id) 5416 { 5417 return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32); 5418 } 5419 5420 int bpf_core_types_match(const struct btf *local_btf, __u32 local_id, 5421 const struct btf *targ_btf, __u32 targ_id) 5422 { 5423 return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32); 5424 } 5425 5426 static size_t bpf_core_hash_fn(const long key, void *ctx) 5427 { 5428 return key; 5429 } 5430 5431 static bool bpf_core_equal_fn(const long k1, const long k2, void *ctx) 5432 { 5433 return k1 == k2; 5434 } 5435 5436 static int record_relo_core(struct bpf_program *prog, 5437 const struct bpf_core_relo *core_relo, int insn_idx) 5438 { 5439 struct reloc_desc *relos, *relo; 5440 5441 relos = libbpf_reallocarray(prog->reloc_desc, 5442 prog->nr_reloc + 1, sizeof(*relos)); 5443 if (!relos) 5444 return -ENOMEM; 5445 relo = &relos[prog->nr_reloc]; 5446 relo->type = RELO_CORE; 5447 relo->insn_idx = insn_idx; 5448 relo->core_relo = core_relo; 5449 prog->reloc_desc = relos; 5450 prog->nr_reloc++; 5451 return 0; 5452 } 5453 5454 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx) 5455 { 5456 struct reloc_desc *relo; 5457 int i; 5458 5459 for (i = 0; i < prog->nr_reloc; i++) { 5460 relo = &prog->reloc_desc[i]; 5461 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx) 5462 continue; 5463 5464 return relo->core_relo; 5465 } 5466 5467 return NULL; 5468 } 5469 5470 static int bpf_core_resolve_relo(struct bpf_program *prog, 5471 const struct bpf_core_relo *relo, 5472 int relo_idx, 5473 const struct btf *local_btf, 5474 struct hashmap *cand_cache, 5475 struct bpf_core_relo_res *targ_res) 5476 { 5477 struct bpf_core_spec specs_scratch[3] = {}; 5478 struct bpf_core_cand_list *cands = NULL; 5479 const char *prog_name = prog->name; 5480 const struct btf_type *local_type; 5481 const char *local_name; 5482 __u32 local_id = relo->type_id; 5483 int err; 5484 5485 local_type = btf__type_by_id(local_btf, local_id); 5486 if (!local_type) 5487 return -EINVAL; 5488 5489 local_name = btf__name_by_offset(local_btf, local_type->name_off); 5490 if (!local_name) 5491 return -EINVAL; 5492 5493 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL && 5494 !hashmap__find(cand_cache, local_id, &cands)) { 5495 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); 5496 if (IS_ERR(cands)) { 5497 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", 5498 prog_name, relo_idx, local_id, btf_kind_str(local_type), 5499 local_name, PTR_ERR(cands)); 5500 return PTR_ERR(cands); 5501 } 5502 err = hashmap__set(cand_cache, local_id, cands, NULL, NULL); 5503 if (err) { 5504 bpf_core_free_cands(cands); 5505 return err; 5506 } 5507 } 5508 5509 return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch, 5510 targ_res); 5511 } 5512 5513 static int 5514 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) 5515 { 5516 const struct btf_ext_info_sec *sec; 5517 struct bpf_core_relo_res targ_res; 5518 const struct bpf_core_relo *rec; 5519 const struct btf_ext_info *seg; 5520 struct hashmap_entry *entry; 5521 struct hashmap *cand_cache = NULL; 5522 struct bpf_program *prog; 5523 struct bpf_insn *insn; 5524 const char *sec_name; 5525 int i, err = 0, insn_idx, sec_idx, sec_num; 5526 5527 if (obj->btf_ext->core_relo_info.len == 0) 5528 return 0; 5529 5530 if (targ_btf_path) { 5531 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); 5532 err = libbpf_get_error(obj->btf_vmlinux_override); 5533 if (err) { 5534 pr_warn("failed to parse target BTF: %d\n", err); 5535 return err; 5536 } 5537 } 5538 5539 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); 5540 if (IS_ERR(cand_cache)) { 5541 err = PTR_ERR(cand_cache); 5542 goto out; 5543 } 5544 5545 seg = &obj->btf_ext->core_relo_info; 5546 sec_num = 0; 5547 for_each_btf_ext_sec(seg, sec) { 5548 sec_idx = seg->sec_idxs[sec_num]; 5549 sec_num++; 5550 5551 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 5552 if (str_is_empty(sec_name)) { 5553 err = -EINVAL; 5554 goto out; 5555 } 5556 5557 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info); 5558 5559 for_each_btf_ext_rec(seg, sec, i, rec) { 5560 if (rec->insn_off % BPF_INSN_SZ) 5561 return -EINVAL; 5562 insn_idx = rec->insn_off / BPF_INSN_SZ; 5563 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 5564 if (!prog) { 5565 /* When __weak subprog is "overridden" by another instance 5566 * of the subprog from a different object file, linker still 5567 * appends all the .BTF.ext info that used to belong to that 5568 * eliminated subprogram. 5569 * This is similar to what x86-64 linker does for relocations. 5570 * So just ignore such relocations just like we ignore 5571 * subprog instructions when discovering subprograms. 5572 */ 5573 pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n", 5574 sec_name, i, insn_idx); 5575 continue; 5576 } 5577 /* no need to apply CO-RE relocation if the program is 5578 * not going to be loaded 5579 */ 5580 if (!prog->autoload) 5581 continue; 5582 5583 /* adjust insn_idx from section frame of reference to the local 5584 * program's frame of reference; (sub-)program code is not yet 5585 * relocated, so it's enough to just subtract in-section offset 5586 */ 5587 insn_idx = insn_idx - prog->sec_insn_off; 5588 if (insn_idx >= prog->insns_cnt) 5589 return -EINVAL; 5590 insn = &prog->insns[insn_idx]; 5591 5592 err = record_relo_core(prog, rec, insn_idx); 5593 if (err) { 5594 pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n", 5595 prog->name, i, err); 5596 goto out; 5597 } 5598 5599 if (prog->obj->gen_loader) 5600 continue; 5601 5602 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res); 5603 if (err) { 5604 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", 5605 prog->name, i, err); 5606 goto out; 5607 } 5608 5609 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); 5610 if (err) { 5611 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n", 5612 prog->name, i, insn_idx, err); 5613 goto out; 5614 } 5615 } 5616 } 5617 5618 out: 5619 /* obj->btf_vmlinux and module BTFs are freed after object load */ 5620 btf__free(obj->btf_vmlinux_override); 5621 obj->btf_vmlinux_override = NULL; 5622 5623 if (!IS_ERR_OR_NULL(cand_cache)) { 5624 hashmap__for_each_entry(cand_cache, entry, i) { 5625 bpf_core_free_cands(entry->pvalue); 5626 } 5627 hashmap__free(cand_cache); 5628 } 5629 return err; 5630 } 5631 5632 /* base map load ldimm64 special constant, used also for log fixup logic */ 5633 #define POISON_LDIMM64_MAP_BASE 2001000000 5634 #define POISON_LDIMM64_MAP_PFX "200100" 5635 5636 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx, 5637 int insn_idx, struct bpf_insn *insn, 5638 int map_idx, const struct bpf_map *map) 5639 { 5640 int i; 5641 5642 pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n", 5643 prog->name, relo_idx, insn_idx, map_idx, map->name); 5644 5645 /* we turn single ldimm64 into two identical invalid calls */ 5646 for (i = 0; i < 2; i++) { 5647 insn->code = BPF_JMP | BPF_CALL; 5648 insn->dst_reg = 0; 5649 insn->src_reg = 0; 5650 insn->off = 0; 5651 /* if this instruction is reachable (not a dead code), 5652 * verifier will complain with something like: 5653 * invalid func unknown#2001000123 5654 * where lower 123 is map index into obj->maps[] array 5655 */ 5656 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx; 5657 5658 insn++; 5659 } 5660 } 5661 5662 /* unresolved kfunc call special constant, used also for log fixup logic */ 5663 #define POISON_CALL_KFUNC_BASE 2002000000 5664 #define POISON_CALL_KFUNC_PFX "2002" 5665 5666 static void poison_kfunc_call(struct bpf_program *prog, int relo_idx, 5667 int insn_idx, struct bpf_insn *insn, 5668 int ext_idx, const struct extern_desc *ext) 5669 { 5670 pr_debug("prog '%s': relo #%d: poisoning insn #%d that calls kfunc '%s'\n", 5671 prog->name, relo_idx, insn_idx, ext->name); 5672 5673 /* we turn kfunc call into invalid helper call with identifiable constant */ 5674 insn->code = BPF_JMP | BPF_CALL; 5675 insn->dst_reg = 0; 5676 insn->src_reg = 0; 5677 insn->off = 0; 5678 /* if this instruction is reachable (not a dead code), 5679 * verifier will complain with something like: 5680 * invalid func unknown#2001000123 5681 * where lower 123 is extern index into obj->externs[] array 5682 */ 5683 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx; 5684 } 5685 5686 /* Relocate data references within program code: 5687 * - map references; 5688 * - global variable references; 5689 * - extern references. 5690 */ 5691 static int 5692 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) 5693 { 5694 int i; 5695 5696 for (i = 0; i < prog->nr_reloc; i++) { 5697 struct reloc_desc *relo = &prog->reloc_desc[i]; 5698 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 5699 const struct bpf_map *map; 5700 struct extern_desc *ext; 5701 5702 switch (relo->type) { 5703 case RELO_LD64: 5704 map = &obj->maps[relo->map_idx]; 5705 if (obj->gen_loader) { 5706 insn[0].src_reg = BPF_PSEUDO_MAP_IDX; 5707 insn[0].imm = relo->map_idx; 5708 } else if (map->autocreate) { 5709 insn[0].src_reg = BPF_PSEUDO_MAP_FD; 5710 insn[0].imm = map->fd; 5711 } else { 5712 poison_map_ldimm64(prog, i, relo->insn_idx, insn, 5713 relo->map_idx, map); 5714 } 5715 break; 5716 case RELO_DATA: 5717 map = &obj->maps[relo->map_idx]; 5718 insn[1].imm = insn[0].imm + relo->sym_off; 5719 if (obj->gen_loader) { 5720 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 5721 insn[0].imm = relo->map_idx; 5722 } else if (map->autocreate) { 5723 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 5724 insn[0].imm = map->fd; 5725 } else { 5726 poison_map_ldimm64(prog, i, relo->insn_idx, insn, 5727 relo->map_idx, map); 5728 } 5729 break; 5730 case RELO_EXTERN_LD64: 5731 ext = &obj->externs[relo->ext_idx]; 5732 if (ext->type == EXT_KCFG) { 5733 if (obj->gen_loader) { 5734 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 5735 insn[0].imm = obj->kconfig_map_idx; 5736 } else { 5737 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 5738 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; 5739 } 5740 insn[1].imm = ext->kcfg.data_off; 5741 } else /* EXT_KSYM */ { 5742 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */ 5743 insn[0].src_reg = BPF_PSEUDO_BTF_ID; 5744 insn[0].imm = ext->ksym.kernel_btf_id; 5745 insn[1].imm = ext->ksym.kernel_btf_obj_fd; 5746 } else { /* typeless ksyms or unresolved typed ksyms */ 5747 insn[0].imm = (__u32)ext->ksym.addr; 5748 insn[1].imm = ext->ksym.addr >> 32; 5749 } 5750 } 5751 break; 5752 case RELO_EXTERN_CALL: 5753 ext = &obj->externs[relo->ext_idx]; 5754 insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL; 5755 if (ext->is_set) { 5756 insn[0].imm = ext->ksym.kernel_btf_id; 5757 insn[0].off = ext->ksym.btf_fd_idx; 5758 } else { /* unresolved weak kfunc call */ 5759 poison_kfunc_call(prog, i, relo->insn_idx, insn, 5760 relo->ext_idx, ext); 5761 } 5762 break; 5763 case RELO_SUBPROG_ADDR: 5764 if (insn[0].src_reg != BPF_PSEUDO_FUNC) { 5765 pr_warn("prog '%s': relo #%d: bad insn\n", 5766 prog->name, i); 5767 return -EINVAL; 5768 } 5769 /* handled already */ 5770 break; 5771 case RELO_CALL: 5772 /* handled already */ 5773 break; 5774 case RELO_CORE: 5775 /* will be handled by bpf_program_record_relos() */ 5776 break; 5777 default: 5778 pr_warn("prog '%s': relo #%d: bad relo type %d\n", 5779 prog->name, i, relo->type); 5780 return -EINVAL; 5781 } 5782 } 5783 5784 return 0; 5785 } 5786 5787 static int adjust_prog_btf_ext_info(const struct bpf_object *obj, 5788 const struct bpf_program *prog, 5789 const struct btf_ext_info *ext_info, 5790 void **prog_info, __u32 *prog_rec_cnt, 5791 __u32 *prog_rec_sz) 5792 { 5793 void *copy_start = NULL, *copy_end = NULL; 5794 void *rec, *rec_end, *new_prog_info; 5795 const struct btf_ext_info_sec *sec; 5796 size_t old_sz, new_sz; 5797 int i, sec_num, sec_idx, off_adj; 5798 5799 sec_num = 0; 5800 for_each_btf_ext_sec(ext_info, sec) { 5801 sec_idx = ext_info->sec_idxs[sec_num]; 5802 sec_num++; 5803 if (prog->sec_idx != sec_idx) 5804 continue; 5805 5806 for_each_btf_ext_rec(ext_info, sec, i, rec) { 5807 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ; 5808 5809 if (insn_off < prog->sec_insn_off) 5810 continue; 5811 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) 5812 break; 5813 5814 if (!copy_start) 5815 copy_start = rec; 5816 copy_end = rec + ext_info->rec_size; 5817 } 5818 5819 if (!copy_start) 5820 return -ENOENT; 5821 5822 /* append func/line info of a given (sub-)program to the main 5823 * program func/line info 5824 */ 5825 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size; 5826 new_sz = old_sz + (copy_end - copy_start); 5827 new_prog_info = realloc(*prog_info, new_sz); 5828 if (!new_prog_info) 5829 return -ENOMEM; 5830 *prog_info = new_prog_info; 5831 *prog_rec_cnt = new_sz / ext_info->rec_size; 5832 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start); 5833 5834 /* Kernel instruction offsets are in units of 8-byte 5835 * instructions, while .BTF.ext instruction offsets generated 5836 * by Clang are in units of bytes. So convert Clang offsets 5837 * into kernel offsets and adjust offset according to program 5838 * relocated position. 5839 */ 5840 off_adj = prog->sub_insn_off - prog->sec_insn_off; 5841 rec = new_prog_info + old_sz; 5842 rec_end = new_prog_info + new_sz; 5843 for (; rec < rec_end; rec += ext_info->rec_size) { 5844 __u32 *insn_off = rec; 5845 5846 *insn_off = *insn_off / BPF_INSN_SZ + off_adj; 5847 } 5848 *prog_rec_sz = ext_info->rec_size; 5849 return 0; 5850 } 5851 5852 return -ENOENT; 5853 } 5854 5855 static int 5856 reloc_prog_func_and_line_info(const struct bpf_object *obj, 5857 struct bpf_program *main_prog, 5858 const struct bpf_program *prog) 5859 { 5860 int err; 5861 5862 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't 5863 * support func/line info 5864 */ 5865 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC)) 5866 return 0; 5867 5868 /* only attempt func info relocation if main program's func_info 5869 * relocation was successful 5870 */ 5871 if (main_prog != prog && !main_prog->func_info) 5872 goto line_info; 5873 5874 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, 5875 &main_prog->func_info, 5876 &main_prog->func_info_cnt, 5877 &main_prog->func_info_rec_size); 5878 if (err) { 5879 if (err != -ENOENT) { 5880 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n", 5881 prog->name, err); 5882 return err; 5883 } 5884 if (main_prog->func_info) { 5885 /* 5886 * Some info has already been found but has problem 5887 * in the last btf_ext reloc. Must have to error out. 5888 */ 5889 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); 5890 return err; 5891 } 5892 /* Have problem loading the very first info. Ignore the rest. */ 5893 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n", 5894 prog->name); 5895 } 5896 5897 line_info: 5898 /* don't relocate line info if main program's relocation failed */ 5899 if (main_prog != prog && !main_prog->line_info) 5900 return 0; 5901 5902 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, 5903 &main_prog->line_info, 5904 &main_prog->line_info_cnt, 5905 &main_prog->line_info_rec_size); 5906 if (err) { 5907 if (err != -ENOENT) { 5908 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n", 5909 prog->name, err); 5910 return err; 5911 } 5912 if (main_prog->line_info) { 5913 /* 5914 * Some info has already been found but has problem 5915 * in the last btf_ext reloc. Must have to error out. 5916 */ 5917 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); 5918 return err; 5919 } 5920 /* Have problem loading the very first info. Ignore the rest. */ 5921 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n", 5922 prog->name); 5923 } 5924 return 0; 5925 } 5926 5927 static int cmp_relo_by_insn_idx(const void *key, const void *elem) 5928 { 5929 size_t insn_idx = *(const size_t *)key; 5930 const struct reloc_desc *relo = elem; 5931 5932 if (insn_idx == relo->insn_idx) 5933 return 0; 5934 return insn_idx < relo->insn_idx ? -1 : 1; 5935 } 5936 5937 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx) 5938 { 5939 if (!prog->nr_reloc) 5940 return NULL; 5941 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, 5942 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); 5943 } 5944 5945 static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog) 5946 { 5947 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc; 5948 struct reloc_desc *relos; 5949 int i; 5950 5951 if (main_prog == subprog) 5952 return 0; 5953 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); 5954 /* if new count is zero, reallocarray can return a valid NULL result; 5955 * in this case the previous pointer will be freed, so we *have to* 5956 * reassign old pointer to the new value (even if it's NULL) 5957 */ 5958 if (!relos && new_cnt) 5959 return -ENOMEM; 5960 if (subprog->nr_reloc) 5961 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, 5962 sizeof(*relos) * subprog->nr_reloc); 5963 5964 for (i = main_prog->nr_reloc; i < new_cnt; i++) 5965 relos[i].insn_idx += subprog->sub_insn_off; 5966 /* After insn_idx adjustment the 'relos' array is still sorted 5967 * by insn_idx and doesn't break bsearch. 5968 */ 5969 main_prog->reloc_desc = relos; 5970 main_prog->nr_reloc = new_cnt; 5971 return 0; 5972 } 5973 5974 static int 5975 bpf_object__append_subprog_code(struct bpf_object *obj, struct bpf_program *main_prog, 5976 struct bpf_program *subprog) 5977 { 5978 struct bpf_insn *insns; 5979 size_t new_cnt; 5980 int err; 5981 5982 subprog->sub_insn_off = main_prog->insns_cnt; 5983 5984 new_cnt = main_prog->insns_cnt + subprog->insns_cnt; 5985 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); 5986 if (!insns) { 5987 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); 5988 return -ENOMEM; 5989 } 5990 main_prog->insns = insns; 5991 main_prog->insns_cnt = new_cnt; 5992 5993 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, 5994 subprog->insns_cnt * sizeof(*insns)); 5995 5996 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", 5997 main_prog->name, subprog->insns_cnt, subprog->name); 5998 5999 /* The subprog insns are now appended. Append its relos too. */ 6000 err = append_subprog_relos(main_prog, subprog); 6001 if (err) 6002 return err; 6003 return 0; 6004 } 6005 6006 static int 6007 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, 6008 struct bpf_program *prog) 6009 { 6010 size_t sub_insn_idx, insn_idx; 6011 struct bpf_program *subprog; 6012 struct reloc_desc *relo; 6013 struct bpf_insn *insn; 6014 int err; 6015 6016 err = reloc_prog_func_and_line_info(obj, main_prog, prog); 6017 if (err) 6018 return err; 6019 6020 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { 6021 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 6022 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn)) 6023 continue; 6024 6025 relo = find_prog_insn_relo(prog, insn_idx); 6026 if (relo && relo->type == RELO_EXTERN_CALL) 6027 /* kfunc relocations will be handled later 6028 * in bpf_object__relocate_data() 6029 */ 6030 continue; 6031 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { 6032 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", 6033 prog->name, insn_idx, relo->type); 6034 return -LIBBPF_ERRNO__RELOC; 6035 } 6036 if (relo) { 6037 /* sub-program instruction index is a combination of 6038 * an offset of a symbol pointed to by relocation and 6039 * call instruction's imm field; for global functions, 6040 * call always has imm = -1, but for static functions 6041 * relocation is against STT_SECTION and insn->imm 6042 * points to a start of a static function 6043 * 6044 * for subprog addr relocation, the relo->sym_off + insn->imm is 6045 * the byte offset in the corresponding section. 6046 */ 6047 if (relo->type == RELO_CALL) 6048 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; 6049 else 6050 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; 6051 } else if (insn_is_pseudo_func(insn)) { 6052 /* 6053 * RELO_SUBPROG_ADDR relo is always emitted even if both 6054 * functions are in the same section, so it shouldn't reach here. 6055 */ 6056 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n", 6057 prog->name, insn_idx); 6058 return -LIBBPF_ERRNO__RELOC; 6059 } else { 6060 /* if subprogram call is to a static function within 6061 * the same ELF section, there won't be any relocation 6062 * emitted, but it also means there is no additional 6063 * offset necessary, insns->imm is relative to 6064 * instruction's original position within the section 6065 */ 6066 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; 6067 } 6068 6069 /* we enforce that sub-programs should be in .text section */ 6070 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); 6071 if (!subprog) { 6072 pr_warn("prog '%s': no .text section found yet sub-program call exists\n", 6073 prog->name); 6074 return -LIBBPF_ERRNO__RELOC; 6075 } 6076 6077 /* if it's the first call instruction calling into this 6078 * subprogram (meaning this subprog hasn't been processed 6079 * yet) within the context of current main program: 6080 * - append it at the end of main program's instructions blog; 6081 * - process is recursively, while current program is put on hold; 6082 * - if that subprogram calls some other not yet processes 6083 * subprogram, same thing will happen recursively until 6084 * there are no more unprocesses subprograms left to append 6085 * and relocate. 6086 */ 6087 if (subprog->sub_insn_off == 0) { 6088 err = bpf_object__append_subprog_code(obj, main_prog, subprog); 6089 if (err) 6090 return err; 6091 err = bpf_object__reloc_code(obj, main_prog, subprog); 6092 if (err) 6093 return err; 6094 } 6095 6096 /* main_prog->insns memory could have been re-allocated, so 6097 * calculate pointer again 6098 */ 6099 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 6100 /* calculate correct instruction position within current main 6101 * prog; each main prog can have a different set of 6102 * subprograms appended (potentially in different order as 6103 * well), so position of any subprog can be different for 6104 * different main programs 6105 */ 6106 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; 6107 6108 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n", 6109 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); 6110 } 6111 6112 return 0; 6113 } 6114 6115 /* 6116 * Relocate sub-program calls. 6117 * 6118 * Algorithm operates as follows. Each entry-point BPF program (referred to as 6119 * main prog) is processed separately. For each subprog (non-entry functions, 6120 * that can be called from either entry progs or other subprogs) gets their 6121 * sub_insn_off reset to zero. This serves as indicator that this subprogram 6122 * hasn't been yet appended and relocated within current main prog. Once its 6123 * relocated, sub_insn_off will point at the position within current main prog 6124 * where given subprog was appended. This will further be used to relocate all 6125 * the call instructions jumping into this subprog. 6126 * 6127 * We start with main program and process all call instructions. If the call 6128 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off 6129 * is zero), subprog instructions are appended at the end of main program's 6130 * instruction array. Then main program is "put on hold" while we recursively 6131 * process newly appended subprogram. If that subprogram calls into another 6132 * subprogram that hasn't been appended, new subprogram is appended again to 6133 * the *main* prog's instructions (subprog's instructions are always left 6134 * untouched, as they need to be in unmodified state for subsequent main progs 6135 * and subprog instructions are always sent only as part of a main prog) and 6136 * the process continues recursively. Once all the subprogs called from a main 6137 * prog or any of its subprogs are appended (and relocated), all their 6138 * positions within finalized instructions array are known, so it's easy to 6139 * rewrite call instructions with correct relative offsets, corresponding to 6140 * desired target subprog. 6141 * 6142 * Its important to realize that some subprogs might not be called from some 6143 * main prog and any of its called/used subprogs. Those will keep their 6144 * subprog->sub_insn_off as zero at all times and won't be appended to current 6145 * main prog and won't be relocated within the context of current main prog. 6146 * They might still be used from other main progs later. 6147 * 6148 * Visually this process can be shown as below. Suppose we have two main 6149 * programs mainA and mainB and BPF object contains three subprogs: subA, 6150 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and 6151 * subC both call subB: 6152 * 6153 * +--------+ +-------+ 6154 * | v v | 6155 * +--+---+ +--+-+-+ +---+--+ 6156 * | subA | | subB | | subC | 6157 * +--+---+ +------+ +---+--+ 6158 * ^ ^ 6159 * | | 6160 * +---+-------+ +------+----+ 6161 * | mainA | | mainB | 6162 * +-----------+ +-----------+ 6163 * 6164 * We'll start relocating mainA, will find subA, append it and start 6165 * processing sub A recursively: 6166 * 6167 * +-----------+------+ 6168 * | mainA | subA | 6169 * +-----------+------+ 6170 * 6171 * At this point we notice that subB is used from subA, so we append it and 6172 * relocate (there are no further subcalls from subB): 6173 * 6174 * +-----------+------+------+ 6175 * | mainA | subA | subB | 6176 * +-----------+------+------+ 6177 * 6178 * At this point, we relocate subA calls, then go one level up and finish with 6179 * relocatin mainA calls. mainA is done. 6180 * 6181 * For mainB process is similar but results in different order. We start with 6182 * mainB and skip subA and subB, as mainB never calls them (at least 6183 * directly), but we see subC is needed, so we append and start processing it: 6184 * 6185 * +-----------+------+ 6186 * | mainB | subC | 6187 * +-----------+------+ 6188 * Now we see subC needs subB, so we go back to it, append and relocate it: 6189 * 6190 * +-----------+------+------+ 6191 * | mainB | subC | subB | 6192 * +-----------+------+------+ 6193 * 6194 * At this point we unwind recursion, relocate calls in subC, then in mainB. 6195 */ 6196 static int 6197 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) 6198 { 6199 struct bpf_program *subprog; 6200 int i, err; 6201 6202 /* mark all subprogs as not relocated (yet) within the context of 6203 * current main program 6204 */ 6205 for (i = 0; i < obj->nr_programs; i++) { 6206 subprog = &obj->programs[i]; 6207 if (!prog_is_subprog(obj, subprog)) 6208 continue; 6209 6210 subprog->sub_insn_off = 0; 6211 } 6212 6213 err = bpf_object__reloc_code(obj, prog, prog); 6214 if (err) 6215 return err; 6216 6217 return 0; 6218 } 6219 6220 static void 6221 bpf_object__free_relocs(struct bpf_object *obj) 6222 { 6223 struct bpf_program *prog; 6224 int i; 6225 6226 /* free up relocation descriptors */ 6227 for (i = 0; i < obj->nr_programs; i++) { 6228 prog = &obj->programs[i]; 6229 zfree(&prog->reloc_desc); 6230 prog->nr_reloc = 0; 6231 } 6232 } 6233 6234 static int cmp_relocs(const void *_a, const void *_b) 6235 { 6236 const struct reloc_desc *a = _a; 6237 const struct reloc_desc *b = _b; 6238 6239 if (a->insn_idx != b->insn_idx) 6240 return a->insn_idx < b->insn_idx ? -1 : 1; 6241 6242 /* no two relocations should have the same insn_idx, but ... */ 6243 if (a->type != b->type) 6244 return a->type < b->type ? -1 : 1; 6245 6246 return 0; 6247 } 6248 6249 static void bpf_object__sort_relos(struct bpf_object *obj) 6250 { 6251 int i; 6252 6253 for (i = 0; i < obj->nr_programs; i++) { 6254 struct bpf_program *p = &obj->programs[i]; 6255 6256 if (!p->nr_reloc) 6257 continue; 6258 6259 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); 6260 } 6261 } 6262 6263 static int bpf_prog_assign_exc_cb(struct bpf_object *obj, struct bpf_program *prog) 6264 { 6265 const char *str = "exception_callback:"; 6266 size_t pfx_len = strlen(str); 6267 int i, j, n; 6268 6269 if (!obj->btf || !kernel_supports(obj, FEAT_BTF_DECL_TAG)) 6270 return 0; 6271 6272 n = btf__type_cnt(obj->btf); 6273 for (i = 1; i < n; i++) { 6274 const char *name; 6275 struct btf_type *t; 6276 6277 t = btf_type_by_id(obj->btf, i); 6278 if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1) 6279 continue; 6280 6281 name = btf__str_by_offset(obj->btf, t->name_off); 6282 if (strncmp(name, str, pfx_len) != 0) 6283 continue; 6284 6285 t = btf_type_by_id(obj->btf, t->type); 6286 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) { 6287 pr_warn("prog '%s': exception_callback:<value> decl tag not applied to the main program\n", 6288 prog->name); 6289 return -EINVAL; 6290 } 6291 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0) 6292 continue; 6293 /* Multiple callbacks are specified for the same prog, 6294 * the verifier will eventually return an error for this 6295 * case, hence simply skip appending a subprog. 6296 */ 6297 if (prog->exception_cb_idx >= 0) { 6298 prog->exception_cb_idx = -1; 6299 break; 6300 } 6301 6302 name += pfx_len; 6303 if (str_is_empty(name)) { 6304 pr_warn("prog '%s': exception_callback:<value> decl tag contains empty value\n", 6305 prog->name); 6306 return -EINVAL; 6307 } 6308 6309 for (j = 0; j < obj->nr_programs; j++) { 6310 struct bpf_program *subprog = &obj->programs[j]; 6311 6312 if (!prog_is_subprog(obj, subprog)) 6313 continue; 6314 if (strcmp(name, subprog->name) != 0) 6315 continue; 6316 /* Enforce non-hidden, as from verifier point of 6317 * view it expects global functions, whereas the 6318 * mark_btf_static fixes up linkage as static. 6319 */ 6320 if (!subprog->sym_global || subprog->mark_btf_static) { 6321 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n", 6322 prog->name, subprog->name); 6323 return -EINVAL; 6324 } 6325 /* Let's see if we already saw a static exception callback with the same name */ 6326 if (prog->exception_cb_idx >= 0) { 6327 pr_warn("prog '%s': multiple subprogs with same name as exception callback '%s'\n", 6328 prog->name, subprog->name); 6329 return -EINVAL; 6330 } 6331 prog->exception_cb_idx = j; 6332 break; 6333 } 6334 6335 if (prog->exception_cb_idx >= 0) 6336 continue; 6337 6338 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name); 6339 return -ENOENT; 6340 } 6341 6342 return 0; 6343 } 6344 6345 static struct { 6346 enum bpf_prog_type prog_type; 6347 const char *ctx_name; 6348 } global_ctx_map[] = { 6349 { BPF_PROG_TYPE_CGROUP_DEVICE, "bpf_cgroup_dev_ctx" }, 6350 { BPF_PROG_TYPE_CGROUP_SKB, "__sk_buff" }, 6351 { BPF_PROG_TYPE_CGROUP_SOCK, "bpf_sock" }, 6352 { BPF_PROG_TYPE_CGROUP_SOCK_ADDR, "bpf_sock_addr" }, 6353 { BPF_PROG_TYPE_CGROUP_SOCKOPT, "bpf_sockopt" }, 6354 { BPF_PROG_TYPE_CGROUP_SYSCTL, "bpf_sysctl" }, 6355 { BPF_PROG_TYPE_FLOW_DISSECTOR, "__sk_buff" }, 6356 { BPF_PROG_TYPE_KPROBE, "bpf_user_pt_regs_t" }, 6357 { BPF_PROG_TYPE_LWT_IN, "__sk_buff" }, 6358 { BPF_PROG_TYPE_LWT_OUT, "__sk_buff" }, 6359 { BPF_PROG_TYPE_LWT_SEG6LOCAL, "__sk_buff" }, 6360 { BPF_PROG_TYPE_LWT_XMIT, "__sk_buff" }, 6361 { BPF_PROG_TYPE_NETFILTER, "bpf_nf_ctx" }, 6362 { BPF_PROG_TYPE_PERF_EVENT, "bpf_perf_event_data" }, 6363 { BPF_PROG_TYPE_RAW_TRACEPOINT, "bpf_raw_tracepoint_args" }, 6364 { BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, "bpf_raw_tracepoint_args" }, 6365 { BPF_PROG_TYPE_SCHED_ACT, "__sk_buff" }, 6366 { BPF_PROG_TYPE_SCHED_CLS, "__sk_buff" }, 6367 { BPF_PROG_TYPE_SK_LOOKUP, "bpf_sk_lookup" }, 6368 { BPF_PROG_TYPE_SK_MSG, "sk_msg_md" }, 6369 { BPF_PROG_TYPE_SK_REUSEPORT, "sk_reuseport_md" }, 6370 { BPF_PROG_TYPE_SK_SKB, "__sk_buff" }, 6371 { BPF_PROG_TYPE_SOCK_OPS, "bpf_sock_ops" }, 6372 { BPF_PROG_TYPE_SOCKET_FILTER, "__sk_buff" }, 6373 { BPF_PROG_TYPE_XDP, "xdp_md" }, 6374 /* all other program types don't have "named" context structs */ 6375 }; 6376 6377 /* forward declarations for arch-specific underlying types of bpf_user_pt_regs_t typedef, 6378 * for below __builtin_types_compatible_p() checks; 6379 * with this approach we don't need any extra arch-specific #ifdef guards 6380 */ 6381 struct pt_regs; 6382 struct user_pt_regs; 6383 struct user_regs_struct; 6384 6385 static bool need_func_arg_type_fixup(const struct btf *btf, const struct bpf_program *prog, 6386 const char *subprog_name, int arg_idx, 6387 int arg_type_id, const char *ctx_name) 6388 { 6389 const struct btf_type *t; 6390 const char *tname; 6391 6392 /* check if existing parameter already matches verifier expectations */ 6393 t = skip_mods_and_typedefs(btf, arg_type_id, NULL); 6394 if (!btf_is_ptr(t)) 6395 goto out_warn; 6396 6397 /* typedef bpf_user_pt_regs_t is a special PITA case, valid for kprobe 6398 * and perf_event programs, so check this case early on and forget 6399 * about it for subsequent checks 6400 */ 6401 while (btf_is_mod(t)) 6402 t = btf__type_by_id(btf, t->type); 6403 if (btf_is_typedef(t) && 6404 (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) { 6405 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>"; 6406 if (strcmp(tname, "bpf_user_pt_regs_t") == 0) 6407 return false; /* canonical type for kprobe/perf_event */ 6408 } 6409 6410 /* now we can ignore typedefs moving forward */ 6411 t = skip_mods_and_typedefs(btf, t->type, NULL); 6412 6413 /* if it's `void *`, definitely fix up BTF info */ 6414 if (btf_is_void(t)) 6415 return true; 6416 6417 /* if it's already proper canonical type, no need to fix up */ 6418 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>"; 6419 if (btf_is_struct(t) && strcmp(tname, ctx_name) == 0) 6420 return false; 6421 6422 /* special cases */ 6423 switch (prog->type) { 6424 case BPF_PROG_TYPE_KPROBE: 6425 /* `struct pt_regs *` is expected, but we need to fix up */ 6426 if (btf_is_struct(t) && strcmp(tname, "pt_regs") == 0) 6427 return true; 6428 break; 6429 case BPF_PROG_TYPE_PERF_EVENT: 6430 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct pt_regs) && 6431 btf_is_struct(t) && strcmp(tname, "pt_regs") == 0) 6432 return true; 6433 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_pt_regs) && 6434 btf_is_struct(t) && strcmp(tname, "user_pt_regs") == 0) 6435 return true; 6436 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_regs_struct) && 6437 btf_is_struct(t) && strcmp(tname, "user_regs_struct") == 0) 6438 return true; 6439 break; 6440 case BPF_PROG_TYPE_RAW_TRACEPOINT: 6441 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 6442 /* allow u64* as ctx */ 6443 if (btf_is_int(t) && t->size == 8) 6444 return true; 6445 break; 6446 default: 6447 break; 6448 } 6449 6450 out_warn: 6451 pr_warn("prog '%s': subprog '%s' arg#%d is expected to be of `struct %s *` type\n", 6452 prog->name, subprog_name, arg_idx, ctx_name); 6453 return false; 6454 } 6455 6456 static int clone_func_btf_info(struct btf *btf, int orig_fn_id, struct bpf_program *prog) 6457 { 6458 int fn_id, fn_proto_id, ret_type_id, orig_proto_id; 6459 int i, err, arg_cnt, fn_name_off, linkage; 6460 struct btf_type *fn_t, *fn_proto_t, *t; 6461 struct btf_param *p; 6462 6463 /* caller already validated FUNC -> FUNC_PROTO validity */ 6464 fn_t = btf_type_by_id(btf, orig_fn_id); 6465 fn_proto_t = btf_type_by_id(btf, fn_t->type); 6466 6467 /* Note that each btf__add_xxx() operation invalidates 6468 * all btf_type and string pointers, so we need to be 6469 * very careful when cloning BTF types. BTF type 6470 * pointers have to be always refetched. And to avoid 6471 * problems with invalidated string pointers, we 6472 * add empty strings initially, then just fix up 6473 * name_off offsets in place. Offsets are stable for 6474 * existing strings, so that works out. 6475 */ 6476 fn_name_off = fn_t->name_off; /* we are about to invalidate fn_t */ 6477 linkage = btf_func_linkage(fn_t); 6478 orig_proto_id = fn_t->type; /* original FUNC_PROTO ID */ 6479 ret_type_id = fn_proto_t->type; /* fn_proto_t will be invalidated */ 6480 arg_cnt = btf_vlen(fn_proto_t); 6481 6482 /* clone FUNC_PROTO and its params */ 6483 fn_proto_id = btf__add_func_proto(btf, ret_type_id); 6484 if (fn_proto_id < 0) 6485 return -EINVAL; 6486 6487 for (i = 0; i < arg_cnt; i++) { 6488 int name_off; 6489 6490 /* copy original parameter data */ 6491 t = btf_type_by_id(btf, orig_proto_id); 6492 p = &btf_params(t)[i]; 6493 name_off = p->name_off; 6494 6495 err = btf__add_func_param(btf, "", p->type); 6496 if (err) 6497 return err; 6498 6499 fn_proto_t = btf_type_by_id(btf, fn_proto_id); 6500 p = &btf_params(fn_proto_t)[i]; 6501 p->name_off = name_off; /* use remembered str offset */ 6502 } 6503 6504 /* clone FUNC now, btf__add_func() enforces non-empty name, so use 6505 * entry program's name as a placeholder, which we replace immediately 6506 * with original name_off 6507 */ 6508 fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id); 6509 if (fn_id < 0) 6510 return -EINVAL; 6511 6512 fn_t = btf_type_by_id(btf, fn_id); 6513 fn_t->name_off = fn_name_off; /* reuse original string */ 6514 6515 return fn_id; 6516 } 6517 6518 /* Check if main program or global subprog's function prototype has `arg:ctx` 6519 * argument tags, and, if necessary, substitute correct type to match what BPF 6520 * verifier would expect, taking into account specific program type. This 6521 * allows to support __arg_ctx tag transparently on old kernels that don't yet 6522 * have a native support for it in the verifier, making user's life much 6523 * easier. 6524 */ 6525 static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_program *prog) 6526 { 6527 const char *ctx_name = NULL, *ctx_tag = "arg:ctx", *fn_name; 6528 struct bpf_func_info_min *func_rec; 6529 struct btf_type *fn_t, *fn_proto_t; 6530 struct btf *btf = obj->btf; 6531 const struct btf_type *t; 6532 struct btf_param *p; 6533 int ptr_id = 0, struct_id, tag_id, orig_fn_id; 6534 int i, n, arg_idx, arg_cnt, err, rec_idx; 6535 int *orig_ids; 6536 6537 /* no .BTF.ext, no problem */ 6538 if (!obj->btf_ext || !prog->func_info) 6539 return 0; 6540 6541 /* don't do any fix ups if kernel natively supports __arg_ctx */ 6542 if (kernel_supports(obj, FEAT_ARG_CTX_TAG)) 6543 return 0; 6544 6545 /* some BPF program types just don't have named context structs, so 6546 * this fallback mechanism doesn't work for them 6547 */ 6548 for (i = 0; i < ARRAY_SIZE(global_ctx_map); i++) { 6549 if (global_ctx_map[i].prog_type != prog->type) 6550 continue; 6551 ctx_name = global_ctx_map[i].ctx_name; 6552 break; 6553 } 6554 if (!ctx_name) 6555 return 0; 6556 6557 /* remember original func BTF IDs to detect if we already cloned them */ 6558 orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids)); 6559 if (!orig_ids) 6560 return -ENOMEM; 6561 for (i = 0; i < prog->func_info_cnt; i++) { 6562 func_rec = prog->func_info + prog->func_info_rec_size * i; 6563 orig_ids[i] = func_rec->type_id; 6564 } 6565 6566 /* go through each DECL_TAG with "arg:ctx" and see if it points to one 6567 * of our subprogs; if yes and subprog is global and needs adjustment, 6568 * clone and adjust FUNC -> FUNC_PROTO combo 6569 */ 6570 for (i = 1, n = btf__type_cnt(btf); i < n; i++) { 6571 /* only DECL_TAG with "arg:ctx" value are interesting */ 6572 t = btf__type_by_id(btf, i); 6573 if (!btf_is_decl_tag(t)) 6574 continue; 6575 if (strcmp(btf__str_by_offset(btf, t->name_off), ctx_tag) != 0) 6576 continue; 6577 6578 /* only global funcs need adjustment, if at all */ 6579 orig_fn_id = t->type; 6580 fn_t = btf_type_by_id(btf, orig_fn_id); 6581 if (!btf_is_func(fn_t) || btf_func_linkage(fn_t) != BTF_FUNC_GLOBAL) 6582 continue; 6583 6584 /* sanity check FUNC -> FUNC_PROTO chain, just in case */ 6585 fn_proto_t = btf_type_by_id(btf, fn_t->type); 6586 if (!fn_proto_t || !btf_is_func_proto(fn_proto_t)) 6587 continue; 6588 6589 /* find corresponding func_info record */ 6590 func_rec = NULL; 6591 for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) { 6592 if (orig_ids[rec_idx] == t->type) { 6593 func_rec = prog->func_info + prog->func_info_rec_size * rec_idx; 6594 break; 6595 } 6596 } 6597 /* current main program doesn't call into this subprog */ 6598 if (!func_rec) 6599 continue; 6600 6601 /* some more sanity checking of DECL_TAG */ 6602 arg_cnt = btf_vlen(fn_proto_t); 6603 arg_idx = btf_decl_tag(t)->component_idx; 6604 if (arg_idx < 0 || arg_idx >= arg_cnt) 6605 continue; 6606 6607 /* check if we should fix up argument type */ 6608 p = &btf_params(fn_proto_t)[arg_idx]; 6609 fn_name = btf__str_by_offset(btf, fn_t->name_off) ?: "<anon>"; 6610 if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name)) 6611 continue; 6612 6613 /* clone fn/fn_proto, unless we already did it for another arg */ 6614 if (func_rec->type_id == orig_fn_id) { 6615 int fn_id; 6616 6617 fn_id = clone_func_btf_info(btf, orig_fn_id, prog); 6618 if (fn_id < 0) { 6619 err = fn_id; 6620 goto err_out; 6621 } 6622 6623 /* point func_info record to a cloned FUNC type */ 6624 func_rec->type_id = fn_id; 6625 } 6626 6627 /* create PTR -> STRUCT type chain to mark PTR_TO_CTX argument; 6628 * we do it just once per main BPF program, as all global 6629 * funcs share the same program type, so need only PTR -> 6630 * STRUCT type chain 6631 */ 6632 if (ptr_id == 0) { 6633 struct_id = btf__add_struct(btf, ctx_name, 0); 6634 ptr_id = btf__add_ptr(btf, struct_id); 6635 if (ptr_id < 0 || struct_id < 0) { 6636 err = -EINVAL; 6637 goto err_out; 6638 } 6639 } 6640 6641 /* for completeness, clone DECL_TAG and point it to cloned param */ 6642 tag_id = btf__add_decl_tag(btf, ctx_tag, func_rec->type_id, arg_idx); 6643 if (tag_id < 0) { 6644 err = -EINVAL; 6645 goto err_out; 6646 } 6647 6648 /* all the BTF manipulations invalidated pointers, refetch them */ 6649 fn_t = btf_type_by_id(btf, func_rec->type_id); 6650 fn_proto_t = btf_type_by_id(btf, fn_t->type); 6651 6652 /* fix up type ID pointed to by param */ 6653 p = &btf_params(fn_proto_t)[arg_idx]; 6654 p->type = ptr_id; 6655 } 6656 6657 free(orig_ids); 6658 return 0; 6659 err_out: 6660 free(orig_ids); 6661 return err; 6662 } 6663 6664 static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) 6665 { 6666 struct bpf_program *prog; 6667 size_t i, j; 6668 int err; 6669 6670 if (obj->btf_ext) { 6671 err = bpf_object__relocate_core(obj, targ_btf_path); 6672 if (err) { 6673 pr_warn("failed to perform CO-RE relocations: %d\n", 6674 err); 6675 return err; 6676 } 6677 bpf_object__sort_relos(obj); 6678 } 6679 6680 /* Before relocating calls pre-process relocations and mark 6681 * few ld_imm64 instructions that points to subprogs. 6682 * Otherwise bpf_object__reloc_code() later would have to consider 6683 * all ld_imm64 insns as relocation candidates. That would 6684 * reduce relocation speed, since amount of find_prog_insn_relo() 6685 * would increase and most of them will fail to find a relo. 6686 */ 6687 for (i = 0; i < obj->nr_programs; i++) { 6688 prog = &obj->programs[i]; 6689 for (j = 0; j < prog->nr_reloc; j++) { 6690 struct reloc_desc *relo = &prog->reloc_desc[j]; 6691 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 6692 6693 /* mark the insn, so it's recognized by insn_is_pseudo_func() */ 6694 if (relo->type == RELO_SUBPROG_ADDR) 6695 insn[0].src_reg = BPF_PSEUDO_FUNC; 6696 } 6697 } 6698 6699 /* relocate subprogram calls and append used subprograms to main 6700 * programs; each copy of subprogram code needs to be relocated 6701 * differently for each main program, because its code location might 6702 * have changed. 6703 * Append subprog relos to main programs to allow data relos to be 6704 * processed after text is completely relocated. 6705 */ 6706 for (i = 0; i < obj->nr_programs; i++) { 6707 prog = &obj->programs[i]; 6708 /* sub-program's sub-calls are relocated within the context of 6709 * its main program only 6710 */ 6711 if (prog_is_subprog(obj, prog)) 6712 continue; 6713 if (!prog->autoload) 6714 continue; 6715 6716 err = bpf_object__relocate_calls(obj, prog); 6717 if (err) { 6718 pr_warn("prog '%s': failed to relocate calls: %d\n", 6719 prog->name, err); 6720 return err; 6721 } 6722 6723 err = bpf_prog_assign_exc_cb(obj, prog); 6724 if (err) 6725 return err; 6726 /* Now, also append exception callback if it has not been done already. */ 6727 if (prog->exception_cb_idx >= 0) { 6728 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx]; 6729 6730 /* Calling exception callback directly is disallowed, which the 6731 * verifier will reject later. In case it was processed already, 6732 * we can skip this step, otherwise for all other valid cases we 6733 * have to append exception callback now. 6734 */ 6735 if (subprog->sub_insn_off == 0) { 6736 err = bpf_object__append_subprog_code(obj, prog, subprog); 6737 if (err) 6738 return err; 6739 err = bpf_object__reloc_code(obj, prog, subprog); 6740 if (err) 6741 return err; 6742 } 6743 } 6744 } 6745 for (i = 0; i < obj->nr_programs; i++) { 6746 prog = &obj->programs[i]; 6747 if (prog_is_subprog(obj, prog)) 6748 continue; 6749 if (!prog->autoload) 6750 continue; 6751 6752 /* Process data relos for main programs */ 6753 err = bpf_object__relocate_data(obj, prog); 6754 if (err) { 6755 pr_warn("prog '%s': failed to relocate data references: %d\n", 6756 prog->name, err); 6757 return err; 6758 } 6759 6760 /* Fix up .BTF.ext information, if necessary */ 6761 err = bpf_program_fixup_func_info(obj, prog); 6762 if (err) { 6763 pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %d\n", 6764 prog->name, err); 6765 return err; 6766 } 6767 } 6768 6769 return 0; 6770 } 6771 6772 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 6773 Elf64_Shdr *shdr, Elf_Data *data); 6774 6775 static int bpf_object__collect_map_relos(struct bpf_object *obj, 6776 Elf64_Shdr *shdr, Elf_Data *data) 6777 { 6778 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *); 6779 int i, j, nrels, new_sz; 6780 const struct btf_var_secinfo *vi = NULL; 6781 const struct btf_type *sec, *var, *def; 6782 struct bpf_map *map = NULL, *targ_map = NULL; 6783 struct bpf_program *targ_prog = NULL; 6784 bool is_prog_array, is_map_in_map; 6785 const struct btf_member *member; 6786 const char *name, *mname, *type; 6787 unsigned int moff; 6788 Elf64_Sym *sym; 6789 Elf64_Rel *rel; 6790 void *tmp; 6791 6792 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) 6793 return -EINVAL; 6794 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); 6795 if (!sec) 6796 return -EINVAL; 6797 6798 nrels = shdr->sh_size / shdr->sh_entsize; 6799 for (i = 0; i < nrels; i++) { 6800 rel = elf_rel_by_idx(data, i); 6801 if (!rel) { 6802 pr_warn(".maps relo #%d: failed to get ELF relo\n", i); 6803 return -LIBBPF_ERRNO__FORMAT; 6804 } 6805 6806 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); 6807 if (!sym) { 6808 pr_warn(".maps relo #%d: symbol %zx not found\n", 6809 i, (size_t)ELF64_R_SYM(rel->r_info)); 6810 return -LIBBPF_ERRNO__FORMAT; 6811 } 6812 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; 6813 6814 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n", 6815 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value, 6816 (size_t)rel->r_offset, sym->st_name, name); 6817 6818 for (j = 0; j < obj->nr_maps; j++) { 6819 map = &obj->maps[j]; 6820 if (map->sec_idx != obj->efile.btf_maps_shndx) 6821 continue; 6822 6823 vi = btf_var_secinfos(sec) + map->btf_var_idx; 6824 if (vi->offset <= rel->r_offset && 6825 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size) 6826 break; 6827 } 6828 if (j == obj->nr_maps) { 6829 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n", 6830 i, name, (size_t)rel->r_offset); 6831 return -EINVAL; 6832 } 6833 6834 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type); 6835 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY; 6836 type = is_map_in_map ? "map" : "prog"; 6837 if (is_map_in_map) { 6838 if (sym->st_shndx != obj->efile.btf_maps_shndx) { 6839 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", 6840 i, name); 6841 return -LIBBPF_ERRNO__RELOC; 6842 } 6843 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && 6844 map->def.key_size != sizeof(int)) { 6845 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", 6846 i, map->name, sizeof(int)); 6847 return -EINVAL; 6848 } 6849 targ_map = bpf_object__find_map_by_name(obj, name); 6850 if (!targ_map) { 6851 pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n", 6852 i, name); 6853 return -ESRCH; 6854 } 6855 } else if (is_prog_array) { 6856 targ_prog = bpf_object__find_program_by_name(obj, name); 6857 if (!targ_prog) { 6858 pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n", 6859 i, name); 6860 return -ESRCH; 6861 } 6862 if (targ_prog->sec_idx != sym->st_shndx || 6863 targ_prog->sec_insn_off * 8 != sym->st_value || 6864 prog_is_subprog(obj, targ_prog)) { 6865 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n", 6866 i, name); 6867 return -LIBBPF_ERRNO__RELOC; 6868 } 6869 } else { 6870 return -EINVAL; 6871 } 6872 6873 var = btf__type_by_id(obj->btf, vi->type); 6874 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 6875 if (btf_vlen(def) == 0) 6876 return -EINVAL; 6877 member = btf_members(def) + btf_vlen(def) - 1; 6878 mname = btf__name_by_offset(obj->btf, member->name_off); 6879 if (strcmp(mname, "values")) 6880 return -EINVAL; 6881 6882 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; 6883 if (rel->r_offset - vi->offset < moff) 6884 return -EINVAL; 6885 6886 moff = rel->r_offset - vi->offset - moff; 6887 /* here we use BPF pointer size, which is always 64 bit, as we 6888 * are parsing ELF that was built for BPF target 6889 */ 6890 if (moff % bpf_ptr_sz) 6891 return -EINVAL; 6892 moff /= bpf_ptr_sz; 6893 if (moff >= map->init_slots_sz) { 6894 new_sz = moff + 1; 6895 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz); 6896 if (!tmp) 6897 return -ENOMEM; 6898 map->init_slots = tmp; 6899 memset(map->init_slots + map->init_slots_sz, 0, 6900 (new_sz - map->init_slots_sz) * host_ptr_sz); 6901 map->init_slots_sz = new_sz; 6902 } 6903 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog; 6904 6905 pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n", 6906 i, map->name, moff, type, name); 6907 } 6908 6909 return 0; 6910 } 6911 6912 static int bpf_object__collect_relos(struct bpf_object *obj) 6913 { 6914 int i, err; 6915 6916 for (i = 0; i < obj->efile.sec_cnt; i++) { 6917 struct elf_sec_desc *sec_desc = &obj->efile.secs[i]; 6918 Elf64_Shdr *shdr; 6919 Elf_Data *data; 6920 int idx; 6921 6922 if (sec_desc->sec_type != SEC_RELO) 6923 continue; 6924 6925 shdr = sec_desc->shdr; 6926 data = sec_desc->data; 6927 idx = shdr->sh_info; 6928 6929 if (shdr->sh_type != SHT_REL) { 6930 pr_warn("internal error at %d\n", __LINE__); 6931 return -LIBBPF_ERRNO__INTERNAL; 6932 } 6933 6934 if (idx == obj->efile.st_ops_shndx || idx == obj->efile.st_ops_link_shndx) 6935 err = bpf_object__collect_st_ops_relos(obj, shdr, data); 6936 else if (idx == obj->efile.btf_maps_shndx) 6937 err = bpf_object__collect_map_relos(obj, shdr, data); 6938 else 6939 err = bpf_object__collect_prog_relos(obj, shdr, data); 6940 if (err) 6941 return err; 6942 } 6943 6944 bpf_object__sort_relos(obj); 6945 return 0; 6946 } 6947 6948 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id) 6949 { 6950 if (BPF_CLASS(insn->code) == BPF_JMP && 6951 BPF_OP(insn->code) == BPF_CALL && 6952 BPF_SRC(insn->code) == BPF_K && 6953 insn->src_reg == 0 && 6954 insn->dst_reg == 0) { 6955 *func_id = insn->imm; 6956 return true; 6957 } 6958 return false; 6959 } 6960 6961 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) 6962 { 6963 struct bpf_insn *insn = prog->insns; 6964 enum bpf_func_id func_id; 6965 int i; 6966 6967 if (obj->gen_loader) 6968 return 0; 6969 6970 for (i = 0; i < prog->insns_cnt; i++, insn++) { 6971 if (!insn_is_helper_call(insn, &func_id)) 6972 continue; 6973 6974 /* on kernels that don't yet support 6975 * bpf_probe_read_{kernel,user}[_str] helpers, fall back 6976 * to bpf_probe_read() which works well for old kernels 6977 */ 6978 switch (func_id) { 6979 case BPF_FUNC_probe_read_kernel: 6980 case BPF_FUNC_probe_read_user: 6981 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 6982 insn->imm = BPF_FUNC_probe_read; 6983 break; 6984 case BPF_FUNC_probe_read_kernel_str: 6985 case BPF_FUNC_probe_read_user_str: 6986 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 6987 insn->imm = BPF_FUNC_probe_read_str; 6988 break; 6989 default: 6990 break; 6991 } 6992 } 6993 return 0; 6994 } 6995 6996 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, 6997 int *btf_obj_fd, int *btf_type_id); 6998 6999 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */ 7000 static int libbpf_prepare_prog_load(struct bpf_program *prog, 7001 struct bpf_prog_load_opts *opts, long cookie) 7002 { 7003 enum sec_def_flags def = cookie; 7004 7005 /* old kernels might not support specifying expected_attach_type */ 7006 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) 7007 opts->expected_attach_type = 0; 7008 7009 if (def & SEC_SLEEPABLE) 7010 opts->prog_flags |= BPF_F_SLEEPABLE; 7011 7012 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) 7013 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; 7014 7015 /* special check for usdt to use uprobe_multi link */ 7016 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) 7017 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; 7018 7019 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { 7020 int btf_obj_fd = 0, btf_type_id = 0, err; 7021 const char *attach_name; 7022 7023 attach_name = strchr(prog->sec_name, '/'); 7024 if (!attach_name) { 7025 /* if BPF program is annotated with just SEC("fentry") 7026 * (or similar) without declaratively specifying 7027 * target, then it is expected that target will be 7028 * specified with bpf_program__set_attach_target() at 7029 * runtime before BPF object load step. If not, then 7030 * there is nothing to load into the kernel as BPF 7031 * verifier won't be able to validate BPF program 7032 * correctness anyways. 7033 */ 7034 pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n", 7035 prog->name); 7036 return -EINVAL; 7037 } 7038 attach_name++; /* skip over / */ 7039 7040 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id); 7041 if (err) 7042 return err; 7043 7044 /* cache resolved BTF FD and BTF type ID in the prog */ 7045 prog->attach_btf_obj_fd = btf_obj_fd; 7046 prog->attach_btf_id = btf_type_id; 7047 7048 /* but by now libbpf common logic is not utilizing 7049 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because 7050 * this callback is called after opts were populated by 7051 * libbpf, so this callback has to update opts explicitly here 7052 */ 7053 opts->attach_btf_obj_fd = btf_obj_fd; 7054 opts->attach_btf_id = btf_type_id; 7055 } 7056 return 0; 7057 } 7058 7059 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz); 7060 7061 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, 7062 struct bpf_insn *insns, int insns_cnt, 7063 const char *license, __u32 kern_version, int *prog_fd) 7064 { 7065 LIBBPF_OPTS(bpf_prog_load_opts, load_attr); 7066 const char *prog_name = NULL; 7067 char *cp, errmsg[STRERR_BUFSIZE]; 7068 size_t log_buf_size = 0; 7069 char *log_buf = NULL, *tmp; 7070 int btf_fd, ret, err; 7071 bool own_log_buf = true; 7072 __u32 log_level = prog->log_level; 7073 7074 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 7075 /* 7076 * The program type must be set. Most likely we couldn't find a proper 7077 * section definition at load time, and thus we didn't infer the type. 7078 */ 7079 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n", 7080 prog->name, prog->sec_name); 7081 return -EINVAL; 7082 } 7083 7084 if (!insns || !insns_cnt) 7085 return -EINVAL; 7086 7087 if (kernel_supports(obj, FEAT_PROG_NAME)) 7088 prog_name = prog->name; 7089 load_attr.attach_prog_fd = prog->attach_prog_fd; 7090 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; 7091 load_attr.attach_btf_id = prog->attach_btf_id; 7092 load_attr.kern_version = kern_version; 7093 load_attr.prog_ifindex = prog->prog_ifindex; 7094 7095 /* specify func_info/line_info only if kernel supports them */ 7096 btf_fd = btf__fd(obj->btf); 7097 if (btf_fd >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) { 7098 load_attr.prog_btf_fd = btf_fd; 7099 load_attr.func_info = prog->func_info; 7100 load_attr.func_info_rec_size = prog->func_info_rec_size; 7101 load_attr.func_info_cnt = prog->func_info_cnt; 7102 load_attr.line_info = prog->line_info; 7103 load_attr.line_info_rec_size = prog->line_info_rec_size; 7104 load_attr.line_info_cnt = prog->line_info_cnt; 7105 } 7106 load_attr.log_level = log_level; 7107 load_attr.prog_flags = prog->prog_flags; 7108 load_attr.fd_array = obj->fd_array; 7109 7110 load_attr.token_fd = obj->token_fd; 7111 if (obj->token_fd) 7112 load_attr.prog_flags |= BPF_F_TOKEN_FD; 7113 7114 /* adjust load_attr if sec_def provides custom preload callback */ 7115 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) { 7116 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie); 7117 if (err < 0) { 7118 pr_warn("prog '%s': failed to prepare load attributes: %d\n", 7119 prog->name, err); 7120 return err; 7121 } 7122 insns = prog->insns; 7123 insns_cnt = prog->insns_cnt; 7124 } 7125 7126 /* allow prog_prepare_load_fn to change expected_attach_type */ 7127 load_attr.expected_attach_type = prog->expected_attach_type; 7128 7129 if (obj->gen_loader) { 7130 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, 7131 license, insns, insns_cnt, &load_attr, 7132 prog - obj->programs); 7133 *prog_fd = -1; 7134 return 0; 7135 } 7136 7137 retry_load: 7138 /* if log_level is zero, we don't request logs initially even if 7139 * custom log_buf is specified; if the program load fails, then we'll 7140 * bump log_level to 1 and use either custom log_buf or we'll allocate 7141 * our own and retry the load to get details on what failed 7142 */ 7143 if (log_level) { 7144 if (prog->log_buf) { 7145 log_buf = prog->log_buf; 7146 log_buf_size = prog->log_size; 7147 own_log_buf = false; 7148 } else if (obj->log_buf) { 7149 log_buf = obj->log_buf; 7150 log_buf_size = obj->log_size; 7151 own_log_buf = false; 7152 } else { 7153 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2); 7154 tmp = realloc(log_buf, log_buf_size); 7155 if (!tmp) { 7156 ret = -ENOMEM; 7157 goto out; 7158 } 7159 log_buf = tmp; 7160 log_buf[0] = '\0'; 7161 own_log_buf = true; 7162 } 7163 } 7164 7165 load_attr.log_buf = log_buf; 7166 load_attr.log_size = log_buf_size; 7167 load_attr.log_level = log_level; 7168 7169 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); 7170 if (ret >= 0) { 7171 if (log_level && own_log_buf) { 7172 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", 7173 prog->name, log_buf); 7174 } 7175 7176 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) { 7177 struct bpf_map *map; 7178 int i; 7179 7180 for (i = 0; i < obj->nr_maps; i++) { 7181 map = &prog->obj->maps[i]; 7182 if (map->libbpf_type != LIBBPF_MAP_RODATA) 7183 continue; 7184 7185 if (bpf_prog_bind_map(ret, map->fd, NULL)) { 7186 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 7187 pr_warn("prog '%s': failed to bind map '%s': %s\n", 7188 prog->name, map->real_name, cp); 7189 /* Don't fail hard if can't bind rodata. */ 7190 } 7191 } 7192 } 7193 7194 *prog_fd = ret; 7195 ret = 0; 7196 goto out; 7197 } 7198 7199 if (log_level == 0) { 7200 log_level = 1; 7201 goto retry_load; 7202 } 7203 /* On ENOSPC, increase log buffer size and retry, unless custom 7204 * log_buf is specified. 7205 * Be careful to not overflow u32, though. Kernel's log buf size limit 7206 * isn't part of UAPI so it can always be bumped to full 4GB. So don't 7207 * multiply by 2 unless we are sure we'll fit within 32 bits. 7208 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2). 7209 */ 7210 if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2) 7211 goto retry_load; 7212 7213 ret = -errno; 7214 7215 /* post-process verifier log to improve error descriptions */ 7216 fixup_verifier_log(prog, log_buf, log_buf_size); 7217 7218 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 7219 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp); 7220 pr_perm_msg(ret); 7221 7222 if (own_log_buf && log_buf && log_buf[0] != '\0') { 7223 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", 7224 prog->name, log_buf); 7225 } 7226 7227 out: 7228 if (own_log_buf) 7229 free(log_buf); 7230 return ret; 7231 } 7232 7233 static char *find_prev_line(char *buf, char *cur) 7234 { 7235 char *p; 7236 7237 if (cur == buf) /* end of a log buf */ 7238 return NULL; 7239 7240 p = cur - 1; 7241 while (p - 1 >= buf && *(p - 1) != '\n') 7242 p--; 7243 7244 return p; 7245 } 7246 7247 static void patch_log(char *buf, size_t buf_sz, size_t log_sz, 7248 char *orig, size_t orig_sz, const char *patch) 7249 { 7250 /* size of the remaining log content to the right from the to-be-replaced part */ 7251 size_t rem_sz = (buf + log_sz) - (orig + orig_sz); 7252 size_t patch_sz = strlen(patch); 7253 7254 if (patch_sz != orig_sz) { 7255 /* If patch line(s) are longer than original piece of verifier log, 7256 * shift log contents by (patch_sz - orig_sz) bytes to the right 7257 * starting from after to-be-replaced part of the log. 7258 * 7259 * If patch line(s) are shorter than original piece of verifier log, 7260 * shift log contents by (orig_sz - patch_sz) bytes to the left 7261 * starting from after to-be-replaced part of the log 7262 * 7263 * We need to be careful about not overflowing available 7264 * buf_sz capacity. If that's the case, we'll truncate the end 7265 * of the original log, as necessary. 7266 */ 7267 if (patch_sz > orig_sz) { 7268 if (orig + patch_sz >= buf + buf_sz) { 7269 /* patch is big enough to cover remaining space completely */ 7270 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1; 7271 rem_sz = 0; 7272 } else if (patch_sz - orig_sz > buf_sz - log_sz) { 7273 /* patch causes part of remaining log to be truncated */ 7274 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz); 7275 } 7276 } 7277 /* shift remaining log to the right by calculated amount */ 7278 memmove(orig + patch_sz, orig + orig_sz, rem_sz); 7279 } 7280 7281 memcpy(orig, patch, patch_sz); 7282 } 7283 7284 static void fixup_log_failed_core_relo(struct bpf_program *prog, 7285 char *buf, size_t buf_sz, size_t log_sz, 7286 char *line1, char *line2, char *line3) 7287 { 7288 /* Expected log for failed and not properly guarded CO-RE relocation: 7289 * line1 -> 123: (85) call unknown#195896080 7290 * line2 -> invalid func unknown#195896080 7291 * line3 -> <anything else or end of buffer> 7292 * 7293 * "123" is the index of the instruction that was poisoned. We extract 7294 * instruction index to find corresponding CO-RE relocation and 7295 * replace this part of the log with more relevant information about 7296 * failed CO-RE relocation. 7297 */ 7298 const struct bpf_core_relo *relo; 7299 struct bpf_core_spec spec; 7300 char patch[512], spec_buf[256]; 7301 int insn_idx, err, spec_len; 7302 7303 if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1) 7304 return; 7305 7306 relo = find_relo_core(prog, insn_idx); 7307 if (!relo) 7308 return; 7309 7310 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec); 7311 if (err) 7312 return; 7313 7314 spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec); 7315 snprintf(patch, sizeof(patch), 7316 "%d: <invalid CO-RE relocation>\n" 7317 "failed to resolve CO-RE relocation %s%s\n", 7318 insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : ""); 7319 7320 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 7321 } 7322 7323 static void fixup_log_missing_map_load(struct bpf_program *prog, 7324 char *buf, size_t buf_sz, size_t log_sz, 7325 char *line1, char *line2, char *line3) 7326 { 7327 /* Expected log for failed and not properly guarded map reference: 7328 * line1 -> 123: (85) call unknown#2001000345 7329 * line2 -> invalid func unknown#2001000345 7330 * line3 -> <anything else or end of buffer> 7331 * 7332 * "123" is the index of the instruction that was poisoned. 7333 * "345" in "2001000345" is a map index in obj->maps to fetch map name. 7334 */ 7335 struct bpf_object *obj = prog->obj; 7336 const struct bpf_map *map; 7337 int insn_idx, map_idx; 7338 char patch[128]; 7339 7340 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2) 7341 return; 7342 7343 map_idx -= POISON_LDIMM64_MAP_BASE; 7344 if (map_idx < 0 || map_idx >= obj->nr_maps) 7345 return; 7346 map = &obj->maps[map_idx]; 7347 7348 snprintf(patch, sizeof(patch), 7349 "%d: <invalid BPF map reference>\n" 7350 "BPF map '%s' is referenced but wasn't created\n", 7351 insn_idx, map->name); 7352 7353 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 7354 } 7355 7356 static void fixup_log_missing_kfunc_call(struct bpf_program *prog, 7357 char *buf, size_t buf_sz, size_t log_sz, 7358 char *line1, char *line2, char *line3) 7359 { 7360 /* Expected log for failed and not properly guarded kfunc call: 7361 * line1 -> 123: (85) call unknown#2002000345 7362 * line2 -> invalid func unknown#2002000345 7363 * line3 -> <anything else or end of buffer> 7364 * 7365 * "123" is the index of the instruction that was poisoned. 7366 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name. 7367 */ 7368 struct bpf_object *obj = prog->obj; 7369 const struct extern_desc *ext; 7370 int insn_idx, ext_idx; 7371 char patch[128]; 7372 7373 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &ext_idx) != 2) 7374 return; 7375 7376 ext_idx -= POISON_CALL_KFUNC_BASE; 7377 if (ext_idx < 0 || ext_idx >= obj->nr_extern) 7378 return; 7379 ext = &obj->externs[ext_idx]; 7380 7381 snprintf(patch, sizeof(patch), 7382 "%d: <invalid kfunc call>\n" 7383 "kfunc '%s' is referenced but wasn't resolved\n", 7384 insn_idx, ext->name); 7385 7386 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 7387 } 7388 7389 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz) 7390 { 7391 /* look for familiar error patterns in last N lines of the log */ 7392 const size_t max_last_line_cnt = 10; 7393 char *prev_line, *cur_line, *next_line; 7394 size_t log_sz; 7395 int i; 7396 7397 if (!buf) 7398 return; 7399 7400 log_sz = strlen(buf) + 1; 7401 next_line = buf + log_sz - 1; 7402 7403 for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) { 7404 cur_line = find_prev_line(buf, next_line); 7405 if (!cur_line) 7406 return; 7407 7408 if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) { 7409 prev_line = find_prev_line(buf, cur_line); 7410 if (!prev_line) 7411 continue; 7412 7413 /* failed CO-RE relocation case */ 7414 fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz, 7415 prev_line, cur_line, next_line); 7416 return; 7417 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_LDIMM64_MAP_PFX)) { 7418 prev_line = find_prev_line(buf, cur_line); 7419 if (!prev_line) 7420 continue; 7421 7422 /* reference to uncreated BPF map */ 7423 fixup_log_missing_map_load(prog, buf, buf_sz, log_sz, 7424 prev_line, cur_line, next_line); 7425 return; 7426 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_CALL_KFUNC_PFX)) { 7427 prev_line = find_prev_line(buf, cur_line); 7428 if (!prev_line) 7429 continue; 7430 7431 /* reference to unresolved kfunc */ 7432 fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz, 7433 prev_line, cur_line, next_line); 7434 return; 7435 } 7436 } 7437 } 7438 7439 static int bpf_program_record_relos(struct bpf_program *prog) 7440 { 7441 struct bpf_object *obj = prog->obj; 7442 int i; 7443 7444 for (i = 0; i < prog->nr_reloc; i++) { 7445 struct reloc_desc *relo = &prog->reloc_desc[i]; 7446 struct extern_desc *ext = &obj->externs[relo->ext_idx]; 7447 int kind; 7448 7449 switch (relo->type) { 7450 case RELO_EXTERN_LD64: 7451 if (ext->type != EXT_KSYM) 7452 continue; 7453 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ? 7454 BTF_KIND_VAR : BTF_KIND_FUNC; 7455 bpf_gen__record_extern(obj->gen_loader, ext->name, 7456 ext->is_weak, !ext->ksym.type_id, 7457 true, kind, relo->insn_idx); 7458 break; 7459 case RELO_EXTERN_CALL: 7460 bpf_gen__record_extern(obj->gen_loader, ext->name, 7461 ext->is_weak, false, false, BTF_KIND_FUNC, 7462 relo->insn_idx); 7463 break; 7464 case RELO_CORE: { 7465 struct bpf_core_relo cr = { 7466 .insn_off = relo->insn_idx * 8, 7467 .type_id = relo->core_relo->type_id, 7468 .access_str_off = relo->core_relo->access_str_off, 7469 .kind = relo->core_relo->kind, 7470 }; 7471 7472 bpf_gen__record_relo_core(obj->gen_loader, &cr); 7473 break; 7474 } 7475 default: 7476 continue; 7477 } 7478 } 7479 return 0; 7480 } 7481 7482 static int 7483 bpf_object__load_progs(struct bpf_object *obj, int log_level) 7484 { 7485 struct bpf_program *prog; 7486 size_t i; 7487 int err; 7488 7489 for (i = 0; i < obj->nr_programs; i++) { 7490 prog = &obj->programs[i]; 7491 err = bpf_object__sanitize_prog(obj, prog); 7492 if (err) 7493 return err; 7494 } 7495 7496 for (i = 0; i < obj->nr_programs; i++) { 7497 prog = &obj->programs[i]; 7498 if (prog_is_subprog(obj, prog)) 7499 continue; 7500 if (!prog->autoload) { 7501 pr_debug("prog '%s': skipped loading\n", prog->name); 7502 continue; 7503 } 7504 prog->log_level |= log_level; 7505 7506 if (obj->gen_loader) 7507 bpf_program_record_relos(prog); 7508 7509 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, 7510 obj->license, obj->kern_version, &prog->fd); 7511 if (err) { 7512 pr_warn("prog '%s': failed to load: %d\n", prog->name, err); 7513 return err; 7514 } 7515 } 7516 7517 bpf_object__free_relocs(obj); 7518 return 0; 7519 } 7520 7521 static const struct bpf_sec_def *find_sec_def(const char *sec_name); 7522 7523 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts) 7524 { 7525 struct bpf_program *prog; 7526 int err; 7527 7528 bpf_object__for_each_program(prog, obj) { 7529 prog->sec_def = find_sec_def(prog->sec_name); 7530 if (!prog->sec_def) { 7531 /* couldn't guess, but user might manually specify */ 7532 pr_debug("prog '%s': unrecognized ELF section name '%s'\n", 7533 prog->name, prog->sec_name); 7534 continue; 7535 } 7536 7537 prog->type = prog->sec_def->prog_type; 7538 prog->expected_attach_type = prog->sec_def->expected_attach_type; 7539 7540 /* sec_def can have custom callback which should be called 7541 * after bpf_program is initialized to adjust its properties 7542 */ 7543 if (prog->sec_def->prog_setup_fn) { 7544 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie); 7545 if (err < 0) { 7546 pr_warn("prog '%s': failed to initialize: %d\n", 7547 prog->name, err); 7548 return err; 7549 } 7550 } 7551 } 7552 7553 return 0; 7554 } 7555 7556 static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz, 7557 const struct bpf_object_open_opts *opts) 7558 { 7559 const char *obj_name, *kconfig, *btf_tmp_path, *token_path; 7560 struct bpf_object *obj; 7561 char tmp_name[64]; 7562 int err; 7563 char *log_buf; 7564 size_t log_size; 7565 __u32 log_level; 7566 7567 if (elf_version(EV_CURRENT) == EV_NONE) { 7568 pr_warn("failed to init libelf for %s\n", 7569 path ? : "(mem buf)"); 7570 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 7571 } 7572 7573 if (!OPTS_VALID(opts, bpf_object_open_opts)) 7574 return ERR_PTR(-EINVAL); 7575 7576 obj_name = OPTS_GET(opts, object_name, NULL); 7577 if (obj_buf) { 7578 if (!obj_name) { 7579 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 7580 (unsigned long)obj_buf, 7581 (unsigned long)obj_buf_sz); 7582 obj_name = tmp_name; 7583 } 7584 path = obj_name; 7585 pr_debug("loading object '%s' from buffer\n", obj_name); 7586 } 7587 7588 log_buf = OPTS_GET(opts, kernel_log_buf, NULL); 7589 log_size = OPTS_GET(opts, kernel_log_size, 0); 7590 log_level = OPTS_GET(opts, kernel_log_level, 0); 7591 if (log_size > UINT_MAX) 7592 return ERR_PTR(-EINVAL); 7593 if (log_size && !log_buf) 7594 return ERR_PTR(-EINVAL); 7595 7596 token_path = OPTS_GET(opts, bpf_token_path, NULL); 7597 /* if user didn't specify bpf_token_path explicitly, check if 7598 * LIBBPF_BPF_TOKEN_PATH envvar was set and treat it as bpf_token_path 7599 * option 7600 */ 7601 if (!token_path) 7602 token_path = getenv("LIBBPF_BPF_TOKEN_PATH"); 7603 if (token_path && strlen(token_path) >= PATH_MAX) 7604 return ERR_PTR(-ENAMETOOLONG); 7605 7606 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); 7607 if (IS_ERR(obj)) 7608 return obj; 7609 7610 obj->log_buf = log_buf; 7611 obj->log_size = log_size; 7612 obj->log_level = log_level; 7613 7614 if (token_path) { 7615 obj->token_path = strdup(token_path); 7616 if (!obj->token_path) { 7617 err = -ENOMEM; 7618 goto out; 7619 } 7620 } 7621 7622 btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL); 7623 if (btf_tmp_path) { 7624 if (strlen(btf_tmp_path) >= PATH_MAX) { 7625 err = -ENAMETOOLONG; 7626 goto out; 7627 } 7628 obj->btf_custom_path = strdup(btf_tmp_path); 7629 if (!obj->btf_custom_path) { 7630 err = -ENOMEM; 7631 goto out; 7632 } 7633 } 7634 7635 kconfig = OPTS_GET(opts, kconfig, NULL); 7636 if (kconfig) { 7637 obj->kconfig = strdup(kconfig); 7638 if (!obj->kconfig) { 7639 err = -ENOMEM; 7640 goto out; 7641 } 7642 } 7643 7644 err = bpf_object__elf_init(obj); 7645 err = err ? : bpf_object__check_endianness(obj); 7646 err = err ? : bpf_object__elf_collect(obj); 7647 err = err ? : bpf_object__collect_externs(obj); 7648 err = err ? : bpf_object_fixup_btf(obj); 7649 err = err ? : bpf_object__init_maps(obj, opts); 7650 err = err ? : bpf_object_init_progs(obj, opts); 7651 err = err ? : bpf_object__collect_relos(obj); 7652 if (err) 7653 goto out; 7654 7655 bpf_object__elf_finish(obj); 7656 7657 return obj; 7658 out: 7659 bpf_object__close(obj); 7660 return ERR_PTR(err); 7661 } 7662 7663 struct bpf_object * 7664 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) 7665 { 7666 if (!path) 7667 return libbpf_err_ptr(-EINVAL); 7668 7669 pr_debug("loading %s\n", path); 7670 7671 return libbpf_ptr(bpf_object_open(path, NULL, 0, opts)); 7672 } 7673 7674 struct bpf_object *bpf_object__open(const char *path) 7675 { 7676 return bpf_object__open_file(path, NULL); 7677 } 7678 7679 struct bpf_object * 7680 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, 7681 const struct bpf_object_open_opts *opts) 7682 { 7683 if (!obj_buf || obj_buf_sz == 0) 7684 return libbpf_err_ptr(-EINVAL); 7685 7686 return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts)); 7687 } 7688 7689 static int bpf_object_unload(struct bpf_object *obj) 7690 { 7691 size_t i; 7692 7693 if (!obj) 7694 return libbpf_err(-EINVAL); 7695 7696 for (i = 0; i < obj->nr_maps; i++) { 7697 zclose(obj->maps[i].fd); 7698 if (obj->maps[i].st_ops) 7699 zfree(&obj->maps[i].st_ops->kern_vdata); 7700 } 7701 7702 for (i = 0; i < obj->nr_programs; i++) 7703 bpf_program__unload(&obj->programs[i]); 7704 7705 return 0; 7706 } 7707 7708 static int bpf_object__sanitize_maps(struct bpf_object *obj) 7709 { 7710 struct bpf_map *m; 7711 7712 bpf_object__for_each_map(m, obj) { 7713 if (!bpf_map__is_internal(m)) 7714 continue; 7715 if (!kernel_supports(obj, FEAT_ARRAY_MMAP)) 7716 m->def.map_flags &= ~BPF_F_MMAPABLE; 7717 } 7718 7719 return 0; 7720 } 7721 7722 int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx) 7723 { 7724 char sym_type, sym_name[500]; 7725 unsigned long long sym_addr; 7726 int ret, err = 0; 7727 FILE *f; 7728 7729 f = fopen("/proc/kallsyms", "re"); 7730 if (!f) { 7731 err = -errno; 7732 pr_warn("failed to open /proc/kallsyms: %d\n", err); 7733 return err; 7734 } 7735 7736 while (true) { 7737 ret = fscanf(f, "%llx %c %499s%*[^\n]\n", 7738 &sym_addr, &sym_type, sym_name); 7739 if (ret == EOF && feof(f)) 7740 break; 7741 if (ret != 3) { 7742 pr_warn("failed to read kallsyms entry: %d\n", ret); 7743 err = -EINVAL; 7744 break; 7745 } 7746 7747 err = cb(sym_addr, sym_type, sym_name, ctx); 7748 if (err) 7749 break; 7750 } 7751 7752 fclose(f); 7753 return err; 7754 } 7755 7756 static int kallsyms_cb(unsigned long long sym_addr, char sym_type, 7757 const char *sym_name, void *ctx) 7758 { 7759 struct bpf_object *obj = ctx; 7760 const struct btf_type *t; 7761 struct extern_desc *ext; 7762 7763 ext = find_extern_by_name(obj, sym_name); 7764 if (!ext || ext->type != EXT_KSYM) 7765 return 0; 7766 7767 t = btf__type_by_id(obj->btf, ext->btf_id); 7768 if (!btf_is_var(t)) 7769 return 0; 7770 7771 if (ext->is_set && ext->ksym.addr != sym_addr) { 7772 pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n", 7773 sym_name, ext->ksym.addr, sym_addr); 7774 return -EINVAL; 7775 } 7776 if (!ext->is_set) { 7777 ext->is_set = true; 7778 ext->ksym.addr = sym_addr; 7779 pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr); 7780 } 7781 return 0; 7782 } 7783 7784 static int bpf_object__read_kallsyms_file(struct bpf_object *obj) 7785 { 7786 return libbpf_kallsyms_parse(kallsyms_cb, obj); 7787 } 7788 7789 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, 7790 __u16 kind, struct btf **res_btf, 7791 struct module_btf **res_mod_btf) 7792 { 7793 struct module_btf *mod_btf; 7794 struct btf *btf; 7795 int i, id, err; 7796 7797 btf = obj->btf_vmlinux; 7798 mod_btf = NULL; 7799 id = btf__find_by_name_kind(btf, ksym_name, kind); 7800 7801 if (id == -ENOENT) { 7802 err = load_module_btfs(obj); 7803 if (err) 7804 return err; 7805 7806 for (i = 0; i < obj->btf_module_cnt; i++) { 7807 /* we assume module_btf's BTF FD is always >0 */ 7808 mod_btf = &obj->btf_modules[i]; 7809 btf = mod_btf->btf; 7810 id = btf__find_by_name_kind_own(btf, ksym_name, kind); 7811 if (id != -ENOENT) 7812 break; 7813 } 7814 } 7815 if (id <= 0) 7816 return -ESRCH; 7817 7818 *res_btf = btf; 7819 *res_mod_btf = mod_btf; 7820 return id; 7821 } 7822 7823 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, 7824 struct extern_desc *ext) 7825 { 7826 const struct btf_type *targ_var, *targ_type; 7827 __u32 targ_type_id, local_type_id; 7828 struct module_btf *mod_btf = NULL; 7829 const char *targ_var_name; 7830 struct btf *btf = NULL; 7831 int id, err; 7832 7833 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf); 7834 if (id < 0) { 7835 if (id == -ESRCH && ext->is_weak) 7836 return 0; 7837 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n", 7838 ext->name); 7839 return id; 7840 } 7841 7842 /* find local type_id */ 7843 local_type_id = ext->ksym.type_id; 7844 7845 /* find target type_id */ 7846 targ_var = btf__type_by_id(btf, id); 7847 targ_var_name = btf__name_by_offset(btf, targ_var->name_off); 7848 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id); 7849 7850 err = bpf_core_types_are_compat(obj->btf, local_type_id, 7851 btf, targ_type_id); 7852 if (err <= 0) { 7853 const struct btf_type *local_type; 7854 const char *targ_name, *local_name; 7855 7856 local_type = btf__type_by_id(obj->btf, local_type_id); 7857 local_name = btf__name_by_offset(obj->btf, local_type->name_off); 7858 targ_name = btf__name_by_offset(btf, targ_type->name_off); 7859 7860 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", 7861 ext->name, local_type_id, 7862 btf_kind_str(local_type), local_name, targ_type_id, 7863 btf_kind_str(targ_type), targ_name); 7864 return -EINVAL; 7865 } 7866 7867 ext->is_set = true; 7868 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; 7869 ext->ksym.kernel_btf_id = id; 7870 pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n", 7871 ext->name, id, btf_kind_str(targ_var), targ_var_name); 7872 7873 return 0; 7874 } 7875 7876 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, 7877 struct extern_desc *ext) 7878 { 7879 int local_func_proto_id, kfunc_proto_id, kfunc_id; 7880 struct module_btf *mod_btf = NULL; 7881 const struct btf_type *kern_func; 7882 struct btf *kern_btf = NULL; 7883 int ret; 7884 7885 local_func_proto_id = ext->ksym.type_id; 7886 7887 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf, 7888 &mod_btf); 7889 if (kfunc_id < 0) { 7890 if (kfunc_id == -ESRCH && ext->is_weak) 7891 return 0; 7892 pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n", 7893 ext->name); 7894 return kfunc_id; 7895 } 7896 7897 kern_func = btf__type_by_id(kern_btf, kfunc_id); 7898 kfunc_proto_id = kern_func->type; 7899 7900 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, 7901 kern_btf, kfunc_proto_id); 7902 if (ret <= 0) { 7903 if (ext->is_weak) 7904 return 0; 7905 7906 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with %s [%d]\n", 7907 ext->name, local_func_proto_id, 7908 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id); 7909 return -EINVAL; 7910 } 7911 7912 /* set index for module BTF fd in fd_array, if unset */ 7913 if (mod_btf && !mod_btf->fd_array_idx) { 7914 /* insn->off is s16 */ 7915 if (obj->fd_array_cnt == INT16_MAX) { 7916 pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n", 7917 ext->name, mod_btf->fd_array_idx); 7918 return -E2BIG; 7919 } 7920 /* Cannot use index 0 for module BTF fd */ 7921 if (!obj->fd_array_cnt) 7922 obj->fd_array_cnt = 1; 7923 7924 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), 7925 obj->fd_array_cnt + 1); 7926 if (ret) 7927 return ret; 7928 mod_btf->fd_array_idx = obj->fd_array_cnt; 7929 /* we assume module BTF FD is always >0 */ 7930 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd; 7931 } 7932 7933 ext->is_set = true; 7934 ext->ksym.kernel_btf_id = kfunc_id; 7935 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0; 7936 /* Also set kernel_btf_obj_fd to make sure that bpf_object__relocate_data() 7937 * populates FD into ld_imm64 insn when it's used to point to kfunc. 7938 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call. 7939 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64. 7940 */ 7941 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; 7942 pr_debug("extern (func ksym) '%s': resolved to %s [%d]\n", 7943 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id); 7944 7945 return 0; 7946 } 7947 7948 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj) 7949 { 7950 const struct btf_type *t; 7951 struct extern_desc *ext; 7952 int i, err; 7953 7954 for (i = 0; i < obj->nr_extern; i++) { 7955 ext = &obj->externs[i]; 7956 if (ext->type != EXT_KSYM || !ext->ksym.type_id) 7957 continue; 7958 7959 if (obj->gen_loader) { 7960 ext->is_set = true; 7961 ext->ksym.kernel_btf_obj_fd = 0; 7962 ext->ksym.kernel_btf_id = 0; 7963 continue; 7964 } 7965 t = btf__type_by_id(obj->btf, ext->btf_id); 7966 if (btf_is_var(t)) 7967 err = bpf_object__resolve_ksym_var_btf_id(obj, ext); 7968 else 7969 err = bpf_object__resolve_ksym_func_btf_id(obj, ext); 7970 if (err) 7971 return err; 7972 } 7973 return 0; 7974 } 7975 7976 static int bpf_object__resolve_externs(struct bpf_object *obj, 7977 const char *extra_kconfig) 7978 { 7979 bool need_config = false, need_kallsyms = false; 7980 bool need_vmlinux_btf = false; 7981 struct extern_desc *ext; 7982 void *kcfg_data = NULL; 7983 int err, i; 7984 7985 if (obj->nr_extern == 0) 7986 return 0; 7987 7988 if (obj->kconfig_map_idx >= 0) 7989 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; 7990 7991 for (i = 0; i < obj->nr_extern; i++) { 7992 ext = &obj->externs[i]; 7993 7994 if (ext->type == EXT_KSYM) { 7995 if (ext->ksym.type_id) 7996 need_vmlinux_btf = true; 7997 else 7998 need_kallsyms = true; 7999 continue; 8000 } else if (ext->type == EXT_KCFG) { 8001 void *ext_ptr = kcfg_data + ext->kcfg.data_off; 8002 __u64 value = 0; 8003 8004 /* Kconfig externs need actual /proc/config.gz */ 8005 if (str_has_pfx(ext->name, "CONFIG_")) { 8006 need_config = true; 8007 continue; 8008 } 8009 8010 /* Virtual kcfg externs are customly handled by libbpf */ 8011 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { 8012 value = get_kernel_version(); 8013 if (!value) { 8014 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name); 8015 return -EINVAL; 8016 } 8017 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) { 8018 value = kernel_supports(obj, FEAT_BPF_COOKIE); 8019 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) { 8020 value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER); 8021 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) { 8022 /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed 8023 * __kconfig externs, where LINUX_ ones are virtual and filled out 8024 * customly by libbpf (their values don't come from Kconfig). 8025 * If LINUX_xxx variable is not recognized by libbpf, but is marked 8026 * __weak, it defaults to zero value, just like for CONFIG_xxx 8027 * externs. 8028 */ 8029 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name); 8030 return -EINVAL; 8031 } 8032 8033 err = set_kcfg_value_num(ext, ext_ptr, value); 8034 if (err) 8035 return err; 8036 pr_debug("extern (kcfg) '%s': set to 0x%llx\n", 8037 ext->name, (long long)value); 8038 } else { 8039 pr_warn("extern '%s': unrecognized extern kind\n", ext->name); 8040 return -EINVAL; 8041 } 8042 } 8043 if (need_config && extra_kconfig) { 8044 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data); 8045 if (err) 8046 return -EINVAL; 8047 need_config = false; 8048 for (i = 0; i < obj->nr_extern; i++) { 8049 ext = &obj->externs[i]; 8050 if (ext->type == EXT_KCFG && !ext->is_set) { 8051 need_config = true; 8052 break; 8053 } 8054 } 8055 } 8056 if (need_config) { 8057 err = bpf_object__read_kconfig_file(obj, kcfg_data); 8058 if (err) 8059 return -EINVAL; 8060 } 8061 if (need_kallsyms) { 8062 err = bpf_object__read_kallsyms_file(obj); 8063 if (err) 8064 return -EINVAL; 8065 } 8066 if (need_vmlinux_btf) { 8067 err = bpf_object__resolve_ksyms_btf_id(obj); 8068 if (err) 8069 return -EINVAL; 8070 } 8071 for (i = 0; i < obj->nr_extern; i++) { 8072 ext = &obj->externs[i]; 8073 8074 if (!ext->is_set && !ext->is_weak) { 8075 pr_warn("extern '%s' (strong): not resolved\n", ext->name); 8076 return -ESRCH; 8077 } else if (!ext->is_set) { 8078 pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n", 8079 ext->name); 8080 } 8081 } 8082 8083 return 0; 8084 } 8085 8086 static void bpf_map_prepare_vdata(const struct bpf_map *map) 8087 { 8088 struct bpf_struct_ops *st_ops; 8089 __u32 i; 8090 8091 st_ops = map->st_ops; 8092 for (i = 0; i < btf_vlen(st_ops->type); i++) { 8093 struct bpf_program *prog = st_ops->progs[i]; 8094 void *kern_data; 8095 int prog_fd; 8096 8097 if (!prog) 8098 continue; 8099 8100 prog_fd = bpf_program__fd(prog); 8101 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; 8102 *(unsigned long *)kern_data = prog_fd; 8103 } 8104 } 8105 8106 static int bpf_object_prepare_struct_ops(struct bpf_object *obj) 8107 { 8108 int i; 8109 8110 for (i = 0; i < obj->nr_maps; i++) 8111 if (bpf_map__is_struct_ops(&obj->maps[i])) 8112 bpf_map_prepare_vdata(&obj->maps[i]); 8113 8114 return 0; 8115 } 8116 8117 static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path) 8118 { 8119 int err, i; 8120 8121 if (!obj) 8122 return libbpf_err(-EINVAL); 8123 8124 if (obj->loaded) { 8125 pr_warn("object '%s': load can't be attempted twice\n", obj->name); 8126 return libbpf_err(-EINVAL); 8127 } 8128 8129 if (obj->gen_loader) 8130 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); 8131 8132 err = bpf_object_prepare_token(obj); 8133 err = err ? : bpf_object__probe_loading(obj); 8134 err = err ? : bpf_object__load_vmlinux_btf(obj, false); 8135 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); 8136 err = err ? : bpf_object__sanitize_maps(obj); 8137 err = err ? : bpf_object__init_kern_struct_ops_maps(obj); 8138 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path); 8139 err = err ? : bpf_object__sanitize_and_load_btf(obj); 8140 err = err ? : bpf_object__create_maps(obj); 8141 err = err ? : bpf_object__load_progs(obj, extra_log_level); 8142 err = err ? : bpf_object_init_prog_arrays(obj); 8143 err = err ? : bpf_object_prepare_struct_ops(obj); 8144 8145 if (obj->gen_loader) { 8146 /* reset FDs */ 8147 if (obj->btf) 8148 btf__set_fd(obj->btf, -1); 8149 if (!err) 8150 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps); 8151 } 8152 8153 /* clean up fd_array */ 8154 zfree(&obj->fd_array); 8155 8156 /* clean up module BTFs */ 8157 for (i = 0; i < obj->btf_module_cnt; i++) { 8158 close(obj->btf_modules[i].fd); 8159 btf__free(obj->btf_modules[i].btf); 8160 free(obj->btf_modules[i].name); 8161 } 8162 free(obj->btf_modules); 8163 8164 /* clean up vmlinux BTF */ 8165 btf__free(obj->btf_vmlinux); 8166 obj->btf_vmlinux = NULL; 8167 8168 obj->loaded = true; /* doesn't matter if successfully or not */ 8169 8170 if (err) 8171 goto out; 8172 8173 return 0; 8174 out: 8175 /* unpin any maps that were auto-pinned during load */ 8176 for (i = 0; i < obj->nr_maps; i++) 8177 if (obj->maps[i].pinned && !obj->maps[i].reused) 8178 bpf_map__unpin(&obj->maps[i], NULL); 8179 8180 bpf_object_unload(obj); 8181 pr_warn("failed to load object '%s'\n", obj->path); 8182 return libbpf_err(err); 8183 } 8184 8185 int bpf_object__load(struct bpf_object *obj) 8186 { 8187 return bpf_object_load(obj, 0, NULL); 8188 } 8189 8190 static int make_parent_dir(const char *path) 8191 { 8192 char *cp, errmsg[STRERR_BUFSIZE]; 8193 char *dname, *dir; 8194 int err = 0; 8195 8196 dname = strdup(path); 8197 if (dname == NULL) 8198 return -ENOMEM; 8199 8200 dir = dirname(dname); 8201 if (mkdir(dir, 0700) && errno != EEXIST) 8202 err = -errno; 8203 8204 free(dname); 8205 if (err) { 8206 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 8207 pr_warn("failed to mkdir %s: %s\n", path, cp); 8208 } 8209 return err; 8210 } 8211 8212 static int check_path(const char *path) 8213 { 8214 char *cp, errmsg[STRERR_BUFSIZE]; 8215 struct statfs st_fs; 8216 char *dname, *dir; 8217 int err = 0; 8218 8219 if (path == NULL) 8220 return -EINVAL; 8221 8222 dname = strdup(path); 8223 if (dname == NULL) 8224 return -ENOMEM; 8225 8226 dir = dirname(dname); 8227 if (statfs(dir, &st_fs)) { 8228 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 8229 pr_warn("failed to statfs %s: %s\n", dir, cp); 8230 err = -errno; 8231 } 8232 free(dname); 8233 8234 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 8235 pr_warn("specified path %s is not on BPF FS\n", path); 8236 err = -EINVAL; 8237 } 8238 8239 return err; 8240 } 8241 8242 int bpf_program__pin(struct bpf_program *prog, const char *path) 8243 { 8244 char *cp, errmsg[STRERR_BUFSIZE]; 8245 int err; 8246 8247 if (prog->fd < 0) { 8248 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name); 8249 return libbpf_err(-EINVAL); 8250 } 8251 8252 err = make_parent_dir(path); 8253 if (err) 8254 return libbpf_err(err); 8255 8256 err = check_path(path); 8257 if (err) 8258 return libbpf_err(err); 8259 8260 if (bpf_obj_pin(prog->fd, path)) { 8261 err = -errno; 8262 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 8263 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp); 8264 return libbpf_err(err); 8265 } 8266 8267 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path); 8268 return 0; 8269 } 8270 8271 int bpf_program__unpin(struct bpf_program *prog, const char *path) 8272 { 8273 int err; 8274 8275 if (prog->fd < 0) { 8276 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name); 8277 return libbpf_err(-EINVAL); 8278 } 8279 8280 err = check_path(path); 8281 if (err) 8282 return libbpf_err(err); 8283 8284 err = unlink(path); 8285 if (err) 8286 return libbpf_err(-errno); 8287 8288 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path); 8289 return 0; 8290 } 8291 8292 int bpf_map__pin(struct bpf_map *map, const char *path) 8293 { 8294 char *cp, errmsg[STRERR_BUFSIZE]; 8295 int err; 8296 8297 if (map == NULL) { 8298 pr_warn("invalid map pointer\n"); 8299 return libbpf_err(-EINVAL); 8300 } 8301 8302 if (map->pin_path) { 8303 if (path && strcmp(path, map->pin_path)) { 8304 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 8305 bpf_map__name(map), map->pin_path, path); 8306 return libbpf_err(-EINVAL); 8307 } else if (map->pinned) { 8308 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", 8309 bpf_map__name(map), map->pin_path); 8310 return 0; 8311 } 8312 } else { 8313 if (!path) { 8314 pr_warn("missing a path to pin map '%s' at\n", 8315 bpf_map__name(map)); 8316 return libbpf_err(-EINVAL); 8317 } else if (map->pinned) { 8318 pr_warn("map '%s' already pinned\n", bpf_map__name(map)); 8319 return libbpf_err(-EEXIST); 8320 } 8321 8322 map->pin_path = strdup(path); 8323 if (!map->pin_path) { 8324 err = -errno; 8325 goto out_err; 8326 } 8327 } 8328 8329 err = make_parent_dir(map->pin_path); 8330 if (err) 8331 return libbpf_err(err); 8332 8333 err = check_path(map->pin_path); 8334 if (err) 8335 return libbpf_err(err); 8336 8337 if (bpf_obj_pin(map->fd, map->pin_path)) { 8338 err = -errno; 8339 goto out_err; 8340 } 8341 8342 map->pinned = true; 8343 pr_debug("pinned map '%s'\n", map->pin_path); 8344 8345 return 0; 8346 8347 out_err: 8348 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 8349 pr_warn("failed to pin map: %s\n", cp); 8350 return libbpf_err(err); 8351 } 8352 8353 int bpf_map__unpin(struct bpf_map *map, const char *path) 8354 { 8355 int err; 8356 8357 if (map == NULL) { 8358 pr_warn("invalid map pointer\n"); 8359 return libbpf_err(-EINVAL); 8360 } 8361 8362 if (map->pin_path) { 8363 if (path && strcmp(path, map->pin_path)) { 8364 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 8365 bpf_map__name(map), map->pin_path, path); 8366 return libbpf_err(-EINVAL); 8367 } 8368 path = map->pin_path; 8369 } else if (!path) { 8370 pr_warn("no path to unpin map '%s' from\n", 8371 bpf_map__name(map)); 8372 return libbpf_err(-EINVAL); 8373 } 8374 8375 err = check_path(path); 8376 if (err) 8377 return libbpf_err(err); 8378 8379 err = unlink(path); 8380 if (err != 0) 8381 return libbpf_err(-errno); 8382 8383 map->pinned = false; 8384 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); 8385 8386 return 0; 8387 } 8388 8389 int bpf_map__set_pin_path(struct bpf_map *map, const char *path) 8390 { 8391 char *new = NULL; 8392 8393 if (path) { 8394 new = strdup(path); 8395 if (!new) 8396 return libbpf_err(-errno); 8397 } 8398 8399 free(map->pin_path); 8400 map->pin_path = new; 8401 return 0; 8402 } 8403 8404 __alias(bpf_map__pin_path) 8405 const char *bpf_map__get_pin_path(const struct bpf_map *map); 8406 8407 const char *bpf_map__pin_path(const struct bpf_map *map) 8408 { 8409 return map->pin_path; 8410 } 8411 8412 bool bpf_map__is_pinned(const struct bpf_map *map) 8413 { 8414 return map->pinned; 8415 } 8416 8417 static void sanitize_pin_path(char *s) 8418 { 8419 /* bpffs disallows periods in path names */ 8420 while (*s) { 8421 if (*s == '.') 8422 *s = '_'; 8423 s++; 8424 } 8425 } 8426 8427 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 8428 { 8429 struct bpf_map *map; 8430 int err; 8431 8432 if (!obj) 8433 return libbpf_err(-ENOENT); 8434 8435 if (!obj->loaded) { 8436 pr_warn("object not yet loaded; load it first\n"); 8437 return libbpf_err(-ENOENT); 8438 } 8439 8440 bpf_object__for_each_map(map, obj) { 8441 char *pin_path = NULL; 8442 char buf[PATH_MAX]; 8443 8444 if (!map->autocreate) 8445 continue; 8446 8447 if (path) { 8448 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 8449 if (err) 8450 goto err_unpin_maps; 8451 sanitize_pin_path(buf); 8452 pin_path = buf; 8453 } else if (!map->pin_path) { 8454 continue; 8455 } 8456 8457 err = bpf_map__pin(map, pin_path); 8458 if (err) 8459 goto err_unpin_maps; 8460 } 8461 8462 return 0; 8463 8464 err_unpin_maps: 8465 while ((map = bpf_object__prev_map(obj, map))) { 8466 if (!map->pin_path) 8467 continue; 8468 8469 bpf_map__unpin(map, NULL); 8470 } 8471 8472 return libbpf_err(err); 8473 } 8474 8475 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 8476 { 8477 struct bpf_map *map; 8478 int err; 8479 8480 if (!obj) 8481 return libbpf_err(-ENOENT); 8482 8483 bpf_object__for_each_map(map, obj) { 8484 char *pin_path = NULL; 8485 char buf[PATH_MAX]; 8486 8487 if (path) { 8488 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 8489 if (err) 8490 return libbpf_err(err); 8491 sanitize_pin_path(buf); 8492 pin_path = buf; 8493 } else if (!map->pin_path) { 8494 continue; 8495 } 8496 8497 err = bpf_map__unpin(map, pin_path); 8498 if (err) 8499 return libbpf_err(err); 8500 } 8501 8502 return 0; 8503 } 8504 8505 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 8506 { 8507 struct bpf_program *prog; 8508 char buf[PATH_MAX]; 8509 int err; 8510 8511 if (!obj) 8512 return libbpf_err(-ENOENT); 8513 8514 if (!obj->loaded) { 8515 pr_warn("object not yet loaded; load it first\n"); 8516 return libbpf_err(-ENOENT); 8517 } 8518 8519 bpf_object__for_each_program(prog, obj) { 8520 err = pathname_concat(buf, sizeof(buf), path, prog->name); 8521 if (err) 8522 goto err_unpin_programs; 8523 8524 err = bpf_program__pin(prog, buf); 8525 if (err) 8526 goto err_unpin_programs; 8527 } 8528 8529 return 0; 8530 8531 err_unpin_programs: 8532 while ((prog = bpf_object__prev_program(obj, prog))) { 8533 if (pathname_concat(buf, sizeof(buf), path, prog->name)) 8534 continue; 8535 8536 bpf_program__unpin(prog, buf); 8537 } 8538 8539 return libbpf_err(err); 8540 } 8541 8542 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 8543 { 8544 struct bpf_program *prog; 8545 int err; 8546 8547 if (!obj) 8548 return libbpf_err(-ENOENT); 8549 8550 bpf_object__for_each_program(prog, obj) { 8551 char buf[PATH_MAX]; 8552 8553 err = pathname_concat(buf, sizeof(buf), path, prog->name); 8554 if (err) 8555 return libbpf_err(err); 8556 8557 err = bpf_program__unpin(prog, buf); 8558 if (err) 8559 return libbpf_err(err); 8560 } 8561 8562 return 0; 8563 } 8564 8565 int bpf_object__pin(struct bpf_object *obj, const char *path) 8566 { 8567 int err; 8568 8569 err = bpf_object__pin_maps(obj, path); 8570 if (err) 8571 return libbpf_err(err); 8572 8573 err = bpf_object__pin_programs(obj, path); 8574 if (err) { 8575 bpf_object__unpin_maps(obj, path); 8576 return libbpf_err(err); 8577 } 8578 8579 return 0; 8580 } 8581 8582 int bpf_object__unpin(struct bpf_object *obj, const char *path) 8583 { 8584 int err; 8585 8586 err = bpf_object__unpin_programs(obj, path); 8587 if (err) 8588 return libbpf_err(err); 8589 8590 err = bpf_object__unpin_maps(obj, path); 8591 if (err) 8592 return libbpf_err(err); 8593 8594 return 0; 8595 } 8596 8597 static void bpf_map__destroy(struct bpf_map *map) 8598 { 8599 if (map->inner_map) { 8600 bpf_map__destroy(map->inner_map); 8601 zfree(&map->inner_map); 8602 } 8603 8604 zfree(&map->init_slots); 8605 map->init_slots_sz = 0; 8606 8607 if (map->mmaped) { 8608 size_t mmap_sz; 8609 8610 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 8611 munmap(map->mmaped, mmap_sz); 8612 map->mmaped = NULL; 8613 } 8614 8615 if (map->st_ops) { 8616 zfree(&map->st_ops->data); 8617 zfree(&map->st_ops->progs); 8618 zfree(&map->st_ops->kern_func_off); 8619 zfree(&map->st_ops); 8620 } 8621 8622 zfree(&map->name); 8623 zfree(&map->real_name); 8624 zfree(&map->pin_path); 8625 8626 if (map->fd >= 0) 8627 zclose(map->fd); 8628 } 8629 8630 void bpf_object__close(struct bpf_object *obj) 8631 { 8632 size_t i; 8633 8634 if (IS_ERR_OR_NULL(obj)) 8635 return; 8636 8637 usdt_manager_free(obj->usdt_man); 8638 obj->usdt_man = NULL; 8639 8640 bpf_gen__free(obj->gen_loader); 8641 bpf_object__elf_finish(obj); 8642 bpf_object_unload(obj); 8643 btf__free(obj->btf); 8644 btf__free(obj->btf_vmlinux); 8645 btf_ext__free(obj->btf_ext); 8646 8647 for (i = 0; i < obj->nr_maps; i++) 8648 bpf_map__destroy(&obj->maps[i]); 8649 8650 zfree(&obj->btf_custom_path); 8651 zfree(&obj->kconfig); 8652 8653 for (i = 0; i < obj->nr_extern; i++) 8654 zfree(&obj->externs[i].essent_name); 8655 8656 zfree(&obj->externs); 8657 obj->nr_extern = 0; 8658 8659 zfree(&obj->maps); 8660 obj->nr_maps = 0; 8661 8662 if (obj->programs && obj->nr_programs) { 8663 for (i = 0; i < obj->nr_programs; i++) 8664 bpf_program__exit(&obj->programs[i]); 8665 } 8666 zfree(&obj->programs); 8667 8668 zfree(&obj->feat_cache); 8669 zfree(&obj->token_path); 8670 if (obj->token_fd > 0) 8671 close(obj->token_fd); 8672 8673 free(obj); 8674 } 8675 8676 const char *bpf_object__name(const struct bpf_object *obj) 8677 { 8678 return obj ? obj->name : libbpf_err_ptr(-EINVAL); 8679 } 8680 8681 unsigned int bpf_object__kversion(const struct bpf_object *obj) 8682 { 8683 return obj ? obj->kern_version : 0; 8684 } 8685 8686 struct btf *bpf_object__btf(const struct bpf_object *obj) 8687 { 8688 return obj ? obj->btf : NULL; 8689 } 8690 8691 int bpf_object__btf_fd(const struct bpf_object *obj) 8692 { 8693 return obj->btf ? btf__fd(obj->btf) : -1; 8694 } 8695 8696 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version) 8697 { 8698 if (obj->loaded) 8699 return libbpf_err(-EINVAL); 8700 8701 obj->kern_version = kern_version; 8702 8703 return 0; 8704 } 8705 8706 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) 8707 { 8708 struct bpf_gen *gen; 8709 8710 if (!opts) 8711 return -EFAULT; 8712 if (!OPTS_VALID(opts, gen_loader_opts)) 8713 return -EINVAL; 8714 gen = calloc(sizeof(*gen), 1); 8715 if (!gen) 8716 return -ENOMEM; 8717 gen->opts = opts; 8718 obj->gen_loader = gen; 8719 return 0; 8720 } 8721 8722 static struct bpf_program * 8723 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 8724 bool forward) 8725 { 8726 size_t nr_programs = obj->nr_programs; 8727 ssize_t idx; 8728 8729 if (!nr_programs) 8730 return NULL; 8731 8732 if (!p) 8733 /* Iter from the beginning */ 8734 return forward ? &obj->programs[0] : 8735 &obj->programs[nr_programs - 1]; 8736 8737 if (p->obj != obj) { 8738 pr_warn("error: program handler doesn't match object\n"); 8739 return errno = EINVAL, NULL; 8740 } 8741 8742 idx = (p - obj->programs) + (forward ? 1 : -1); 8743 if (idx >= obj->nr_programs || idx < 0) 8744 return NULL; 8745 return &obj->programs[idx]; 8746 } 8747 8748 struct bpf_program * 8749 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev) 8750 { 8751 struct bpf_program *prog = prev; 8752 8753 do { 8754 prog = __bpf_program__iter(prog, obj, true); 8755 } while (prog && prog_is_subprog(obj, prog)); 8756 8757 return prog; 8758 } 8759 8760 struct bpf_program * 8761 bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next) 8762 { 8763 struct bpf_program *prog = next; 8764 8765 do { 8766 prog = __bpf_program__iter(prog, obj, false); 8767 } while (prog && prog_is_subprog(obj, prog)); 8768 8769 return prog; 8770 } 8771 8772 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 8773 { 8774 prog->prog_ifindex = ifindex; 8775 } 8776 8777 const char *bpf_program__name(const struct bpf_program *prog) 8778 { 8779 return prog->name; 8780 } 8781 8782 const char *bpf_program__section_name(const struct bpf_program *prog) 8783 { 8784 return prog->sec_name; 8785 } 8786 8787 bool bpf_program__autoload(const struct bpf_program *prog) 8788 { 8789 return prog->autoload; 8790 } 8791 8792 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) 8793 { 8794 if (prog->obj->loaded) 8795 return libbpf_err(-EINVAL); 8796 8797 prog->autoload = autoload; 8798 return 0; 8799 } 8800 8801 bool bpf_program__autoattach(const struct bpf_program *prog) 8802 { 8803 return prog->autoattach; 8804 } 8805 8806 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach) 8807 { 8808 prog->autoattach = autoattach; 8809 } 8810 8811 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog) 8812 { 8813 return prog->insns; 8814 } 8815 8816 size_t bpf_program__insn_cnt(const struct bpf_program *prog) 8817 { 8818 return prog->insns_cnt; 8819 } 8820 8821 int bpf_program__set_insns(struct bpf_program *prog, 8822 struct bpf_insn *new_insns, size_t new_insn_cnt) 8823 { 8824 struct bpf_insn *insns; 8825 8826 if (prog->obj->loaded) 8827 return -EBUSY; 8828 8829 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); 8830 /* NULL is a valid return from reallocarray if the new count is zero */ 8831 if (!insns && new_insn_cnt) { 8832 pr_warn("prog '%s': failed to realloc prog code\n", prog->name); 8833 return -ENOMEM; 8834 } 8835 memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns)); 8836 8837 prog->insns = insns; 8838 prog->insns_cnt = new_insn_cnt; 8839 return 0; 8840 } 8841 8842 int bpf_program__fd(const struct bpf_program *prog) 8843 { 8844 if (!prog) 8845 return libbpf_err(-EINVAL); 8846 8847 if (prog->fd < 0) 8848 return libbpf_err(-ENOENT); 8849 8850 return prog->fd; 8851 } 8852 8853 __alias(bpf_program__type) 8854 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog); 8855 8856 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog) 8857 { 8858 return prog->type; 8859 } 8860 8861 static size_t custom_sec_def_cnt; 8862 static struct bpf_sec_def *custom_sec_defs; 8863 static struct bpf_sec_def custom_fallback_def; 8864 static bool has_custom_fallback_def; 8865 static int last_custom_sec_def_handler_id; 8866 8867 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 8868 { 8869 if (prog->obj->loaded) 8870 return libbpf_err(-EBUSY); 8871 8872 /* if type is not changed, do nothing */ 8873 if (prog->type == type) 8874 return 0; 8875 8876 prog->type = type; 8877 8878 /* If a program type was changed, we need to reset associated SEC() 8879 * handler, as it will be invalid now. The only exception is a generic 8880 * fallback handler, which by definition is program type-agnostic and 8881 * is a catch-all custom handler, optionally set by the application, 8882 * so should be able to handle any type of BPF program. 8883 */ 8884 if (prog->sec_def != &custom_fallback_def) 8885 prog->sec_def = NULL; 8886 return 0; 8887 } 8888 8889 __alias(bpf_program__expected_attach_type) 8890 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog); 8891 8892 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog) 8893 { 8894 return prog->expected_attach_type; 8895 } 8896 8897 int bpf_program__set_expected_attach_type(struct bpf_program *prog, 8898 enum bpf_attach_type type) 8899 { 8900 if (prog->obj->loaded) 8901 return libbpf_err(-EBUSY); 8902 8903 prog->expected_attach_type = type; 8904 return 0; 8905 } 8906 8907 __u32 bpf_program__flags(const struct bpf_program *prog) 8908 { 8909 return prog->prog_flags; 8910 } 8911 8912 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags) 8913 { 8914 if (prog->obj->loaded) 8915 return libbpf_err(-EBUSY); 8916 8917 prog->prog_flags = flags; 8918 return 0; 8919 } 8920 8921 __u32 bpf_program__log_level(const struct bpf_program *prog) 8922 { 8923 return prog->log_level; 8924 } 8925 8926 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level) 8927 { 8928 if (prog->obj->loaded) 8929 return libbpf_err(-EBUSY); 8930 8931 prog->log_level = log_level; 8932 return 0; 8933 } 8934 8935 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size) 8936 { 8937 *log_size = prog->log_size; 8938 return prog->log_buf; 8939 } 8940 8941 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size) 8942 { 8943 if (log_size && !log_buf) 8944 return -EINVAL; 8945 if (prog->log_size > UINT_MAX) 8946 return -EINVAL; 8947 if (prog->obj->loaded) 8948 return -EBUSY; 8949 8950 prog->log_buf = log_buf; 8951 prog->log_size = log_size; 8952 return 0; 8953 } 8954 8955 #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \ 8956 .sec = (char *)sec_pfx, \ 8957 .prog_type = BPF_PROG_TYPE_##ptype, \ 8958 .expected_attach_type = atype, \ 8959 .cookie = (long)(flags), \ 8960 .prog_prepare_load_fn = libbpf_prepare_prog_load, \ 8961 __VA_ARGS__ \ 8962 } 8963 8964 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8965 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8966 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8967 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8968 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8969 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8970 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8971 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8972 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8973 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8974 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link); 8975 8976 static const struct bpf_sec_def section_defs[] = { 8977 SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE), 8978 SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE), 8979 SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE), 8980 SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), 8981 SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), 8982 SEC_DEF("uprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), 8983 SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), 8984 SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), 8985 SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), 8986 SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), 8987 SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), 8988 SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi), 8989 SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi), 8990 SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), 8991 SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), 8992 SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), 8993 SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), 8994 SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt), 8995 SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT | SEC_SLEEPABLE, attach_usdt), 8996 SEC_DEF("tc/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */ 8997 SEC_DEF("tc/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), /* alias for tcx */ 8998 SEC_DEF("tcx/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), 8999 SEC_DEF("tcx/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), 9000 SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 9001 SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 9002 SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 9003 SEC_DEF("netkit/primary", SCHED_CLS, BPF_NETKIT_PRIMARY, SEC_NONE), 9004 SEC_DEF("netkit/peer", SCHED_CLS, BPF_NETKIT_PEER, SEC_NONE), 9005 SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp), 9006 SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp), 9007 SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), 9008 SEC_DEF("raw_tp+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), 9009 SEC_DEF("raw_tracepoint.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), 9010 SEC_DEF("raw_tp.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), 9011 SEC_DEF("tp_btf+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace), 9012 SEC_DEF("fentry+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace), 9013 SEC_DEF("fmod_ret+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace), 9014 SEC_DEF("fexit+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace), 9015 SEC_DEF("fentry.s+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 9016 SEC_DEF("fmod_ret.s+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 9017 SEC_DEF("fexit.s+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 9018 SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace), 9019 SEC_DEF("lsm+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm), 9020 SEC_DEF("lsm.s+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm), 9021 SEC_DEF("lsm_cgroup+", LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF), 9022 SEC_DEF("iter+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter), 9023 SEC_DEF("iter.s+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter), 9024 SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE), 9025 SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS), 9026 SEC_DEF("xdp/devmap", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE), 9027 SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS), 9028 SEC_DEF("xdp/cpumap", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE), 9029 SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS), 9030 SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT), 9031 SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE), 9032 SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE), 9033 SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE), 9034 SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE), 9035 SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE), 9036 SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT), 9037 SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT), 9038 SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT), 9039 SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE), 9040 SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT), 9041 SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT), 9042 SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT), 9043 SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT), 9044 SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT), 9045 SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE), 9046 SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE), 9047 SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE), 9048 SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT), 9049 SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE), 9050 SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE), 9051 SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE), 9052 SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE), 9053 SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE), 9054 SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE), 9055 SEC_DEF("cgroup/connect_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_CONNECT, SEC_ATTACHABLE), 9056 SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE), 9057 SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE), 9058 SEC_DEF("cgroup/sendmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_SENDMSG, SEC_ATTACHABLE), 9059 SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE), 9060 SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE), 9061 SEC_DEF("cgroup/recvmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_RECVMSG, SEC_ATTACHABLE), 9062 SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE), 9063 SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE), 9064 SEC_DEF("cgroup/getpeername_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETPEERNAME, SEC_ATTACHABLE), 9065 SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE), 9066 SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE), 9067 SEC_DEF("cgroup/getsockname_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETSOCKNAME, SEC_ATTACHABLE), 9068 SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE), 9069 SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE), 9070 SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE), 9071 SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT), 9072 SEC_DEF("struct_ops+", STRUCT_OPS, 0, SEC_NONE), 9073 SEC_DEF("struct_ops.s+", STRUCT_OPS, 0, SEC_SLEEPABLE), 9074 SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE), 9075 SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE), 9076 }; 9077 9078 int libbpf_register_prog_handler(const char *sec, 9079 enum bpf_prog_type prog_type, 9080 enum bpf_attach_type exp_attach_type, 9081 const struct libbpf_prog_handler_opts *opts) 9082 { 9083 struct bpf_sec_def *sec_def; 9084 9085 if (!OPTS_VALID(opts, libbpf_prog_handler_opts)) 9086 return libbpf_err(-EINVAL); 9087 9088 if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */ 9089 return libbpf_err(-E2BIG); 9090 9091 if (sec) { 9092 sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1, 9093 sizeof(*sec_def)); 9094 if (!sec_def) 9095 return libbpf_err(-ENOMEM); 9096 9097 custom_sec_defs = sec_def; 9098 sec_def = &custom_sec_defs[custom_sec_def_cnt]; 9099 } else { 9100 if (has_custom_fallback_def) 9101 return libbpf_err(-EBUSY); 9102 9103 sec_def = &custom_fallback_def; 9104 } 9105 9106 sec_def->sec = sec ? strdup(sec) : NULL; 9107 if (sec && !sec_def->sec) 9108 return libbpf_err(-ENOMEM); 9109 9110 sec_def->prog_type = prog_type; 9111 sec_def->expected_attach_type = exp_attach_type; 9112 sec_def->cookie = OPTS_GET(opts, cookie, 0); 9113 9114 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL); 9115 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL); 9116 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL); 9117 9118 sec_def->handler_id = ++last_custom_sec_def_handler_id; 9119 9120 if (sec) 9121 custom_sec_def_cnt++; 9122 else 9123 has_custom_fallback_def = true; 9124 9125 return sec_def->handler_id; 9126 } 9127 9128 int libbpf_unregister_prog_handler(int handler_id) 9129 { 9130 struct bpf_sec_def *sec_defs; 9131 int i; 9132 9133 if (handler_id <= 0) 9134 return libbpf_err(-EINVAL); 9135 9136 if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) { 9137 memset(&custom_fallback_def, 0, sizeof(custom_fallback_def)); 9138 has_custom_fallback_def = false; 9139 return 0; 9140 } 9141 9142 for (i = 0; i < custom_sec_def_cnt; i++) { 9143 if (custom_sec_defs[i].handler_id == handler_id) 9144 break; 9145 } 9146 9147 if (i == custom_sec_def_cnt) 9148 return libbpf_err(-ENOENT); 9149 9150 free(custom_sec_defs[i].sec); 9151 for (i = i + 1; i < custom_sec_def_cnt; i++) 9152 custom_sec_defs[i - 1] = custom_sec_defs[i]; 9153 custom_sec_def_cnt--; 9154 9155 /* try to shrink the array, but it's ok if we couldn't */ 9156 sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs)); 9157 /* if new count is zero, reallocarray can return a valid NULL result; 9158 * in this case the previous pointer will be freed, so we *have to* 9159 * reassign old pointer to the new value (even if it's NULL) 9160 */ 9161 if (sec_defs || custom_sec_def_cnt == 0) 9162 custom_sec_defs = sec_defs; 9163 9164 return 0; 9165 } 9166 9167 static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name) 9168 { 9169 size_t len = strlen(sec_def->sec); 9170 9171 /* "type/" always has to have proper SEC("type/extras") form */ 9172 if (sec_def->sec[len - 1] == '/') { 9173 if (str_has_pfx(sec_name, sec_def->sec)) 9174 return true; 9175 return false; 9176 } 9177 9178 /* "type+" means it can be either exact SEC("type") or 9179 * well-formed SEC("type/extras") with proper '/' separator 9180 */ 9181 if (sec_def->sec[len - 1] == '+') { 9182 len--; 9183 /* not even a prefix */ 9184 if (strncmp(sec_name, sec_def->sec, len) != 0) 9185 return false; 9186 /* exact match or has '/' separator */ 9187 if (sec_name[len] == '\0' || sec_name[len] == '/') 9188 return true; 9189 return false; 9190 } 9191 9192 return strcmp(sec_name, sec_def->sec) == 0; 9193 } 9194 9195 static const struct bpf_sec_def *find_sec_def(const char *sec_name) 9196 { 9197 const struct bpf_sec_def *sec_def; 9198 int i, n; 9199 9200 n = custom_sec_def_cnt; 9201 for (i = 0; i < n; i++) { 9202 sec_def = &custom_sec_defs[i]; 9203 if (sec_def_matches(sec_def, sec_name)) 9204 return sec_def; 9205 } 9206 9207 n = ARRAY_SIZE(section_defs); 9208 for (i = 0; i < n; i++) { 9209 sec_def = §ion_defs[i]; 9210 if (sec_def_matches(sec_def, sec_name)) 9211 return sec_def; 9212 } 9213 9214 if (has_custom_fallback_def) 9215 return &custom_fallback_def; 9216 9217 return NULL; 9218 } 9219 9220 #define MAX_TYPE_NAME_SIZE 32 9221 9222 static char *libbpf_get_type_names(bool attach_type) 9223 { 9224 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; 9225 char *buf; 9226 9227 buf = malloc(len); 9228 if (!buf) 9229 return NULL; 9230 9231 buf[0] = '\0'; 9232 /* Forge string buf with all available names */ 9233 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 9234 const struct bpf_sec_def *sec_def = §ion_defs[i]; 9235 9236 if (attach_type) { 9237 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) 9238 continue; 9239 9240 if (!(sec_def->cookie & SEC_ATTACHABLE)) 9241 continue; 9242 } 9243 9244 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { 9245 free(buf); 9246 return NULL; 9247 } 9248 strcat(buf, " "); 9249 strcat(buf, section_defs[i].sec); 9250 } 9251 9252 return buf; 9253 } 9254 9255 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 9256 enum bpf_attach_type *expected_attach_type) 9257 { 9258 const struct bpf_sec_def *sec_def; 9259 char *type_names; 9260 9261 if (!name) 9262 return libbpf_err(-EINVAL); 9263 9264 sec_def = find_sec_def(name); 9265 if (sec_def) { 9266 *prog_type = sec_def->prog_type; 9267 *expected_attach_type = sec_def->expected_attach_type; 9268 return 0; 9269 } 9270 9271 pr_debug("failed to guess program type from ELF section '%s'\n", name); 9272 type_names = libbpf_get_type_names(false); 9273 if (type_names != NULL) { 9274 pr_debug("supported section(type) names are:%s\n", type_names); 9275 free(type_names); 9276 } 9277 9278 return libbpf_err(-ESRCH); 9279 } 9280 9281 const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t) 9282 { 9283 if (t < 0 || t >= ARRAY_SIZE(attach_type_name)) 9284 return NULL; 9285 9286 return attach_type_name[t]; 9287 } 9288 9289 const char *libbpf_bpf_link_type_str(enum bpf_link_type t) 9290 { 9291 if (t < 0 || t >= ARRAY_SIZE(link_type_name)) 9292 return NULL; 9293 9294 return link_type_name[t]; 9295 } 9296 9297 const char *libbpf_bpf_map_type_str(enum bpf_map_type t) 9298 { 9299 if (t < 0 || t >= ARRAY_SIZE(map_type_name)) 9300 return NULL; 9301 9302 return map_type_name[t]; 9303 } 9304 9305 const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t) 9306 { 9307 if (t < 0 || t >= ARRAY_SIZE(prog_type_name)) 9308 return NULL; 9309 9310 return prog_type_name[t]; 9311 } 9312 9313 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, 9314 int sec_idx, 9315 size_t offset) 9316 { 9317 struct bpf_map *map; 9318 size_t i; 9319 9320 for (i = 0; i < obj->nr_maps; i++) { 9321 map = &obj->maps[i]; 9322 if (!bpf_map__is_struct_ops(map)) 9323 continue; 9324 if (map->sec_idx == sec_idx && 9325 map->sec_offset <= offset && 9326 offset - map->sec_offset < map->def.value_size) 9327 return map; 9328 } 9329 9330 return NULL; 9331 } 9332 9333 /* Collect the reloc from ELF, populate the st_ops->progs[], and update 9334 * st_ops->data for shadow type. 9335 */ 9336 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 9337 Elf64_Shdr *shdr, Elf_Data *data) 9338 { 9339 const struct btf_member *member; 9340 struct bpf_struct_ops *st_ops; 9341 struct bpf_program *prog; 9342 unsigned int shdr_idx; 9343 const struct btf *btf; 9344 struct bpf_map *map; 9345 unsigned int moff, insn_idx; 9346 const char *name; 9347 __u32 member_idx; 9348 Elf64_Sym *sym; 9349 Elf64_Rel *rel; 9350 int i, nrels; 9351 9352 btf = obj->btf; 9353 nrels = shdr->sh_size / shdr->sh_entsize; 9354 for (i = 0; i < nrels; i++) { 9355 rel = elf_rel_by_idx(data, i); 9356 if (!rel) { 9357 pr_warn("struct_ops reloc: failed to get %d reloc\n", i); 9358 return -LIBBPF_ERRNO__FORMAT; 9359 } 9360 9361 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); 9362 if (!sym) { 9363 pr_warn("struct_ops reloc: symbol %zx not found\n", 9364 (size_t)ELF64_R_SYM(rel->r_info)); 9365 return -LIBBPF_ERRNO__FORMAT; 9366 } 9367 9368 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; 9369 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset); 9370 if (!map) { 9371 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n", 9372 (size_t)rel->r_offset); 9373 return -EINVAL; 9374 } 9375 9376 moff = rel->r_offset - map->sec_offset; 9377 shdr_idx = sym->st_shndx; 9378 st_ops = map->st_ops; 9379 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", 9380 map->name, 9381 (long long)(rel->r_info >> 32), 9382 (long long)sym->st_value, 9383 shdr_idx, (size_t)rel->r_offset, 9384 map->sec_offset, sym->st_name, name); 9385 9386 if (shdr_idx >= SHN_LORESERVE) { 9387 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n", 9388 map->name, (size_t)rel->r_offset, shdr_idx); 9389 return -LIBBPF_ERRNO__RELOC; 9390 } 9391 if (sym->st_value % BPF_INSN_SZ) { 9392 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n", 9393 map->name, (unsigned long long)sym->st_value); 9394 return -LIBBPF_ERRNO__FORMAT; 9395 } 9396 insn_idx = sym->st_value / BPF_INSN_SZ; 9397 9398 member = find_member_by_offset(st_ops->type, moff * 8); 9399 if (!member) { 9400 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", 9401 map->name, moff); 9402 return -EINVAL; 9403 } 9404 member_idx = member - btf_members(st_ops->type); 9405 name = btf__name_by_offset(btf, member->name_off); 9406 9407 if (!resolve_func_ptr(btf, member->type, NULL)) { 9408 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", 9409 map->name, name); 9410 return -EINVAL; 9411 } 9412 9413 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); 9414 if (!prog) { 9415 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", 9416 map->name, shdr_idx, name); 9417 return -EINVAL; 9418 } 9419 9420 /* prevent the use of BPF prog with invalid type */ 9421 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) { 9422 pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n", 9423 map->name, prog->name); 9424 return -EINVAL; 9425 } 9426 9427 /* if we haven't yet processed this BPF program, record proper 9428 * attach_btf_id and member_idx 9429 */ 9430 if (!prog->attach_btf_id) { 9431 prog->attach_btf_id = st_ops->type_id; 9432 prog->expected_attach_type = member_idx; 9433 } 9434 9435 /* struct_ops BPF prog can be re-used between multiple 9436 * .struct_ops & .struct_ops.link as long as it's the 9437 * same struct_ops struct definition and the same 9438 * function pointer field 9439 */ 9440 if (prog->attach_btf_id != st_ops->type_id || 9441 prog->expected_attach_type != member_idx) { 9442 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n", 9443 map->name, prog->name, prog->sec_name, prog->type, 9444 prog->attach_btf_id, prog->expected_attach_type, name); 9445 return -EINVAL; 9446 } 9447 9448 st_ops->progs[member_idx] = prog; 9449 9450 /* st_ops->data will be exposed to users, being returned by 9451 * bpf_map__initial_value() as a pointer to the shadow 9452 * type. All function pointers in the original struct type 9453 * should be converted to a pointer to struct bpf_program 9454 * in the shadow type. 9455 */ 9456 *((struct bpf_program **)(st_ops->data + moff)) = prog; 9457 } 9458 9459 return 0; 9460 } 9461 9462 #define BTF_TRACE_PREFIX "btf_trace_" 9463 #define BTF_LSM_PREFIX "bpf_lsm_" 9464 #define BTF_ITER_PREFIX "bpf_iter_" 9465 #define BTF_MAX_NAME_SIZE 128 9466 9467 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, 9468 const char **prefix, int *kind) 9469 { 9470 switch (attach_type) { 9471 case BPF_TRACE_RAW_TP: 9472 *prefix = BTF_TRACE_PREFIX; 9473 *kind = BTF_KIND_TYPEDEF; 9474 break; 9475 case BPF_LSM_MAC: 9476 case BPF_LSM_CGROUP: 9477 *prefix = BTF_LSM_PREFIX; 9478 *kind = BTF_KIND_FUNC; 9479 break; 9480 case BPF_TRACE_ITER: 9481 *prefix = BTF_ITER_PREFIX; 9482 *kind = BTF_KIND_FUNC; 9483 break; 9484 default: 9485 *prefix = ""; 9486 *kind = BTF_KIND_FUNC; 9487 } 9488 } 9489 9490 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 9491 const char *name, __u32 kind) 9492 { 9493 char btf_type_name[BTF_MAX_NAME_SIZE]; 9494 int ret; 9495 9496 ret = snprintf(btf_type_name, sizeof(btf_type_name), 9497 "%s%s", prefix, name); 9498 /* snprintf returns the number of characters written excluding the 9499 * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it 9500 * indicates truncation. 9501 */ 9502 if (ret < 0 || ret >= sizeof(btf_type_name)) 9503 return -ENAMETOOLONG; 9504 return btf__find_by_name_kind(btf, btf_type_name, kind); 9505 } 9506 9507 static inline int find_attach_btf_id(struct btf *btf, const char *name, 9508 enum bpf_attach_type attach_type) 9509 { 9510 const char *prefix; 9511 int kind; 9512 9513 btf_get_kernel_prefix_kind(attach_type, &prefix, &kind); 9514 return find_btf_by_prefix_kind(btf, prefix, name, kind); 9515 } 9516 9517 int libbpf_find_vmlinux_btf_id(const char *name, 9518 enum bpf_attach_type attach_type) 9519 { 9520 struct btf *btf; 9521 int err; 9522 9523 btf = btf__load_vmlinux_btf(); 9524 err = libbpf_get_error(btf); 9525 if (err) { 9526 pr_warn("vmlinux BTF is not found\n"); 9527 return libbpf_err(err); 9528 } 9529 9530 err = find_attach_btf_id(btf, name, attach_type); 9531 if (err <= 0) 9532 pr_warn("%s is not found in vmlinux BTF\n", name); 9533 9534 btf__free(btf); 9535 return libbpf_err(err); 9536 } 9537 9538 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) 9539 { 9540 struct bpf_prog_info info; 9541 __u32 info_len = sizeof(info); 9542 struct btf *btf; 9543 int err; 9544 9545 memset(&info, 0, info_len); 9546 err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len); 9547 if (err) { 9548 pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %d\n", 9549 attach_prog_fd, err); 9550 return err; 9551 } 9552 9553 err = -EINVAL; 9554 if (!info.btf_id) { 9555 pr_warn("The target program doesn't have BTF\n"); 9556 goto out; 9557 } 9558 btf = btf__load_from_kernel_by_id(info.btf_id); 9559 err = libbpf_get_error(btf); 9560 if (err) { 9561 pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err); 9562 goto out; 9563 } 9564 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 9565 btf__free(btf); 9566 if (err <= 0) { 9567 pr_warn("%s is not found in prog's BTF\n", name); 9568 goto out; 9569 } 9570 out: 9571 return err; 9572 } 9573 9574 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name, 9575 enum bpf_attach_type attach_type, 9576 int *btf_obj_fd, int *btf_type_id) 9577 { 9578 int ret, i; 9579 9580 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type); 9581 if (ret > 0) { 9582 *btf_obj_fd = 0; /* vmlinux BTF */ 9583 *btf_type_id = ret; 9584 return 0; 9585 } 9586 if (ret != -ENOENT) 9587 return ret; 9588 9589 ret = load_module_btfs(obj); 9590 if (ret) 9591 return ret; 9592 9593 for (i = 0; i < obj->btf_module_cnt; i++) { 9594 const struct module_btf *mod = &obj->btf_modules[i]; 9595 9596 ret = find_attach_btf_id(mod->btf, attach_name, attach_type); 9597 if (ret > 0) { 9598 *btf_obj_fd = mod->fd; 9599 *btf_type_id = ret; 9600 return 0; 9601 } 9602 if (ret == -ENOENT) 9603 continue; 9604 9605 return ret; 9606 } 9607 9608 return -ESRCH; 9609 } 9610 9611 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, 9612 int *btf_obj_fd, int *btf_type_id) 9613 { 9614 enum bpf_attach_type attach_type = prog->expected_attach_type; 9615 __u32 attach_prog_fd = prog->attach_prog_fd; 9616 int err = 0; 9617 9618 /* BPF program's BTF ID */ 9619 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) { 9620 if (!attach_prog_fd) { 9621 pr_warn("prog '%s': attach program FD is not set\n", prog->name); 9622 return -EINVAL; 9623 } 9624 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd); 9625 if (err < 0) { 9626 pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n", 9627 prog->name, attach_prog_fd, attach_name, err); 9628 return err; 9629 } 9630 *btf_obj_fd = 0; 9631 *btf_type_id = err; 9632 return 0; 9633 } 9634 9635 /* kernel/module BTF ID */ 9636 if (prog->obj->gen_loader) { 9637 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); 9638 *btf_obj_fd = 0; 9639 *btf_type_id = 1; 9640 } else { 9641 err = find_kernel_btf_id(prog->obj, attach_name, 9642 attach_type, btf_obj_fd, 9643 btf_type_id); 9644 } 9645 if (err) { 9646 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n", 9647 prog->name, attach_name, err); 9648 return err; 9649 } 9650 return 0; 9651 } 9652 9653 int libbpf_attach_type_by_name(const char *name, 9654 enum bpf_attach_type *attach_type) 9655 { 9656 char *type_names; 9657 const struct bpf_sec_def *sec_def; 9658 9659 if (!name) 9660 return libbpf_err(-EINVAL); 9661 9662 sec_def = find_sec_def(name); 9663 if (!sec_def) { 9664 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); 9665 type_names = libbpf_get_type_names(true); 9666 if (type_names != NULL) { 9667 pr_debug("attachable section(type) names are:%s\n", type_names); 9668 free(type_names); 9669 } 9670 9671 return libbpf_err(-EINVAL); 9672 } 9673 9674 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) 9675 return libbpf_err(-EINVAL); 9676 if (!(sec_def->cookie & SEC_ATTACHABLE)) 9677 return libbpf_err(-EINVAL); 9678 9679 *attach_type = sec_def->expected_attach_type; 9680 return 0; 9681 } 9682 9683 int bpf_map__fd(const struct bpf_map *map) 9684 { 9685 if (!map) 9686 return libbpf_err(-EINVAL); 9687 if (!map_is_created(map)) 9688 return -1; 9689 return map->fd; 9690 } 9691 9692 static bool map_uses_real_name(const struct bpf_map *map) 9693 { 9694 /* Since libbpf started to support custom .data.* and .rodata.* maps, 9695 * their user-visible name differs from kernel-visible name. Users see 9696 * such map's corresponding ELF section name as a map name. 9697 * This check distinguishes .data/.rodata from .data.* and .rodata.* 9698 * maps to know which name has to be returned to the user. 9699 */ 9700 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0) 9701 return true; 9702 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0) 9703 return true; 9704 return false; 9705 } 9706 9707 const char *bpf_map__name(const struct bpf_map *map) 9708 { 9709 if (!map) 9710 return NULL; 9711 9712 if (map_uses_real_name(map)) 9713 return map->real_name; 9714 9715 return map->name; 9716 } 9717 9718 enum bpf_map_type bpf_map__type(const struct bpf_map *map) 9719 { 9720 return map->def.type; 9721 } 9722 9723 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type) 9724 { 9725 if (map_is_created(map)) 9726 return libbpf_err(-EBUSY); 9727 map->def.type = type; 9728 return 0; 9729 } 9730 9731 __u32 bpf_map__map_flags(const struct bpf_map *map) 9732 { 9733 return map->def.map_flags; 9734 } 9735 9736 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags) 9737 { 9738 if (map_is_created(map)) 9739 return libbpf_err(-EBUSY); 9740 map->def.map_flags = flags; 9741 return 0; 9742 } 9743 9744 __u64 bpf_map__map_extra(const struct bpf_map *map) 9745 { 9746 return map->map_extra; 9747 } 9748 9749 int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra) 9750 { 9751 if (map_is_created(map)) 9752 return libbpf_err(-EBUSY); 9753 map->map_extra = map_extra; 9754 return 0; 9755 } 9756 9757 __u32 bpf_map__numa_node(const struct bpf_map *map) 9758 { 9759 return map->numa_node; 9760 } 9761 9762 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node) 9763 { 9764 if (map_is_created(map)) 9765 return libbpf_err(-EBUSY); 9766 map->numa_node = numa_node; 9767 return 0; 9768 } 9769 9770 __u32 bpf_map__key_size(const struct bpf_map *map) 9771 { 9772 return map->def.key_size; 9773 } 9774 9775 int bpf_map__set_key_size(struct bpf_map *map, __u32 size) 9776 { 9777 if (map_is_created(map)) 9778 return libbpf_err(-EBUSY); 9779 map->def.key_size = size; 9780 return 0; 9781 } 9782 9783 __u32 bpf_map__value_size(const struct bpf_map *map) 9784 { 9785 return map->def.value_size; 9786 } 9787 9788 static int map_btf_datasec_resize(struct bpf_map *map, __u32 size) 9789 { 9790 struct btf *btf; 9791 struct btf_type *datasec_type, *var_type; 9792 struct btf_var_secinfo *var; 9793 const struct btf_type *array_type; 9794 const struct btf_array *array; 9795 int vlen, element_sz, new_array_id; 9796 __u32 nr_elements; 9797 9798 /* check btf existence */ 9799 btf = bpf_object__btf(map->obj); 9800 if (!btf) 9801 return -ENOENT; 9802 9803 /* verify map is datasec */ 9804 datasec_type = btf_type_by_id(btf, bpf_map__btf_value_type_id(map)); 9805 if (!btf_is_datasec(datasec_type)) { 9806 pr_warn("map '%s': cannot be resized, map value type is not a datasec\n", 9807 bpf_map__name(map)); 9808 return -EINVAL; 9809 } 9810 9811 /* verify datasec has at least one var */ 9812 vlen = btf_vlen(datasec_type); 9813 if (vlen == 0) { 9814 pr_warn("map '%s': cannot be resized, map value datasec is empty\n", 9815 bpf_map__name(map)); 9816 return -EINVAL; 9817 } 9818 9819 /* verify last var in the datasec is an array */ 9820 var = &btf_var_secinfos(datasec_type)[vlen - 1]; 9821 var_type = btf_type_by_id(btf, var->type); 9822 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL); 9823 if (!btf_is_array(array_type)) { 9824 pr_warn("map '%s': cannot be resized, last var must be an array\n", 9825 bpf_map__name(map)); 9826 return -EINVAL; 9827 } 9828 9829 /* verify request size aligns with array */ 9830 array = btf_array(array_type); 9831 element_sz = btf__resolve_size(btf, array->type); 9832 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) { 9833 pr_warn("map '%s': cannot be resized, element size (%d) doesn't align with new total size (%u)\n", 9834 bpf_map__name(map), element_sz, size); 9835 return -EINVAL; 9836 } 9837 9838 /* create a new array based on the existing array, but with new length */ 9839 nr_elements = (size - var->offset) / element_sz; 9840 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements); 9841 if (new_array_id < 0) 9842 return new_array_id; 9843 9844 /* adding a new btf type invalidates existing pointers to btf objects, 9845 * so refresh pointers before proceeding 9846 */ 9847 datasec_type = btf_type_by_id(btf, map->btf_value_type_id); 9848 var = &btf_var_secinfos(datasec_type)[vlen - 1]; 9849 var_type = btf_type_by_id(btf, var->type); 9850 9851 /* finally update btf info */ 9852 datasec_type->size = size; 9853 var->size = size - var->offset; 9854 var_type->type = new_array_id; 9855 9856 return 0; 9857 } 9858 9859 int bpf_map__set_value_size(struct bpf_map *map, __u32 size) 9860 { 9861 if (map->obj->loaded || map->reused) 9862 return libbpf_err(-EBUSY); 9863 9864 if (map->mmaped) { 9865 int err; 9866 size_t mmap_old_sz, mmap_new_sz; 9867 9868 mmap_old_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 9869 mmap_new_sz = bpf_map_mmap_sz(size, map->def.max_entries); 9870 err = bpf_map_mmap_resize(map, mmap_old_sz, mmap_new_sz); 9871 if (err) { 9872 pr_warn("map '%s': failed to resize memory-mapped region: %d\n", 9873 bpf_map__name(map), err); 9874 return err; 9875 } 9876 err = map_btf_datasec_resize(map, size); 9877 if (err && err != -ENOENT) { 9878 pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %d\n", 9879 bpf_map__name(map), err); 9880 map->btf_value_type_id = 0; 9881 map->btf_key_type_id = 0; 9882 } 9883 } 9884 9885 map->def.value_size = size; 9886 return 0; 9887 } 9888 9889 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 9890 { 9891 return map ? map->btf_key_type_id : 0; 9892 } 9893 9894 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 9895 { 9896 return map ? map->btf_value_type_id : 0; 9897 } 9898 9899 int bpf_map__set_initial_value(struct bpf_map *map, 9900 const void *data, size_t size) 9901 { 9902 if (map->obj->loaded || map->reused) 9903 return libbpf_err(-EBUSY); 9904 9905 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG || 9906 size != map->def.value_size) 9907 return libbpf_err(-EINVAL); 9908 9909 memcpy(map->mmaped, data, size); 9910 return 0; 9911 } 9912 9913 void *bpf_map__initial_value(struct bpf_map *map, size_t *psize) 9914 { 9915 if (bpf_map__is_struct_ops(map)) { 9916 if (psize) 9917 *psize = map->def.value_size; 9918 return map->st_ops->data; 9919 } 9920 9921 if (!map->mmaped) 9922 return NULL; 9923 *psize = map->def.value_size; 9924 return map->mmaped; 9925 } 9926 9927 bool bpf_map__is_internal(const struct bpf_map *map) 9928 { 9929 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 9930 } 9931 9932 __u32 bpf_map__ifindex(const struct bpf_map *map) 9933 { 9934 return map->map_ifindex; 9935 } 9936 9937 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 9938 { 9939 if (map_is_created(map)) 9940 return libbpf_err(-EBUSY); 9941 map->map_ifindex = ifindex; 9942 return 0; 9943 } 9944 9945 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 9946 { 9947 if (!bpf_map_type__is_map_in_map(map->def.type)) { 9948 pr_warn("error: unsupported map type\n"); 9949 return libbpf_err(-EINVAL); 9950 } 9951 if (map->inner_map_fd != -1) { 9952 pr_warn("error: inner_map_fd already specified\n"); 9953 return libbpf_err(-EINVAL); 9954 } 9955 if (map->inner_map) { 9956 bpf_map__destroy(map->inner_map); 9957 zfree(&map->inner_map); 9958 } 9959 map->inner_map_fd = fd; 9960 return 0; 9961 } 9962 9963 static struct bpf_map * 9964 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 9965 { 9966 ssize_t idx; 9967 struct bpf_map *s, *e; 9968 9969 if (!obj || !obj->maps) 9970 return errno = EINVAL, NULL; 9971 9972 s = obj->maps; 9973 e = obj->maps + obj->nr_maps; 9974 9975 if ((m < s) || (m >= e)) { 9976 pr_warn("error in %s: map handler doesn't belong to object\n", 9977 __func__); 9978 return errno = EINVAL, NULL; 9979 } 9980 9981 idx = (m - obj->maps) + i; 9982 if (idx >= obj->nr_maps || idx < 0) 9983 return NULL; 9984 return &obj->maps[idx]; 9985 } 9986 9987 struct bpf_map * 9988 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev) 9989 { 9990 if (prev == NULL) 9991 return obj->maps; 9992 9993 return __bpf_map__iter(prev, obj, 1); 9994 } 9995 9996 struct bpf_map * 9997 bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next) 9998 { 9999 if (next == NULL) { 10000 if (!obj->nr_maps) 10001 return NULL; 10002 return obj->maps + obj->nr_maps - 1; 10003 } 10004 10005 return __bpf_map__iter(next, obj, -1); 10006 } 10007 10008 struct bpf_map * 10009 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 10010 { 10011 struct bpf_map *pos; 10012 10013 bpf_object__for_each_map(pos, obj) { 10014 /* if it's a special internal map name (which always starts 10015 * with dot) then check if that special name matches the 10016 * real map name (ELF section name) 10017 */ 10018 if (name[0] == '.') { 10019 if (pos->real_name && strcmp(pos->real_name, name) == 0) 10020 return pos; 10021 continue; 10022 } 10023 /* otherwise map name has to be an exact match */ 10024 if (map_uses_real_name(pos)) { 10025 if (strcmp(pos->real_name, name) == 0) 10026 return pos; 10027 continue; 10028 } 10029 if (strcmp(pos->name, name) == 0) 10030 return pos; 10031 } 10032 return errno = ENOENT, NULL; 10033 } 10034 10035 int 10036 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 10037 { 10038 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 10039 } 10040 10041 static int validate_map_op(const struct bpf_map *map, size_t key_sz, 10042 size_t value_sz, bool check_value_sz) 10043 { 10044 if (!map_is_created(map)) /* map is not yet created */ 10045 return -ENOENT; 10046 10047 if (map->def.key_size != key_sz) { 10048 pr_warn("map '%s': unexpected key size %zu provided, expected %u\n", 10049 map->name, key_sz, map->def.key_size); 10050 return -EINVAL; 10051 } 10052 10053 if (!check_value_sz) 10054 return 0; 10055 10056 switch (map->def.type) { 10057 case BPF_MAP_TYPE_PERCPU_ARRAY: 10058 case BPF_MAP_TYPE_PERCPU_HASH: 10059 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 10060 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: { 10061 int num_cpu = libbpf_num_possible_cpus(); 10062 size_t elem_sz = roundup(map->def.value_size, 8); 10063 10064 if (value_sz != num_cpu * elem_sz) { 10065 pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n", 10066 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); 10067 return -EINVAL; 10068 } 10069 break; 10070 } 10071 default: 10072 if (map->def.value_size != value_sz) { 10073 pr_warn("map '%s': unexpected value size %zu provided, expected %u\n", 10074 map->name, value_sz, map->def.value_size); 10075 return -EINVAL; 10076 } 10077 break; 10078 } 10079 return 0; 10080 } 10081 10082 int bpf_map__lookup_elem(const struct bpf_map *map, 10083 const void *key, size_t key_sz, 10084 void *value, size_t value_sz, __u64 flags) 10085 { 10086 int err; 10087 10088 err = validate_map_op(map, key_sz, value_sz, true); 10089 if (err) 10090 return libbpf_err(err); 10091 10092 return bpf_map_lookup_elem_flags(map->fd, key, value, flags); 10093 } 10094 10095 int bpf_map__update_elem(const struct bpf_map *map, 10096 const void *key, size_t key_sz, 10097 const void *value, size_t value_sz, __u64 flags) 10098 { 10099 int err; 10100 10101 err = validate_map_op(map, key_sz, value_sz, true); 10102 if (err) 10103 return libbpf_err(err); 10104 10105 return bpf_map_update_elem(map->fd, key, value, flags); 10106 } 10107 10108 int bpf_map__delete_elem(const struct bpf_map *map, 10109 const void *key, size_t key_sz, __u64 flags) 10110 { 10111 int err; 10112 10113 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */); 10114 if (err) 10115 return libbpf_err(err); 10116 10117 return bpf_map_delete_elem_flags(map->fd, key, flags); 10118 } 10119 10120 int bpf_map__lookup_and_delete_elem(const struct bpf_map *map, 10121 const void *key, size_t key_sz, 10122 void *value, size_t value_sz, __u64 flags) 10123 { 10124 int err; 10125 10126 err = validate_map_op(map, key_sz, value_sz, true); 10127 if (err) 10128 return libbpf_err(err); 10129 10130 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags); 10131 } 10132 10133 int bpf_map__get_next_key(const struct bpf_map *map, 10134 const void *cur_key, void *next_key, size_t key_sz) 10135 { 10136 int err; 10137 10138 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */); 10139 if (err) 10140 return libbpf_err(err); 10141 10142 return bpf_map_get_next_key(map->fd, cur_key, next_key); 10143 } 10144 10145 long libbpf_get_error(const void *ptr) 10146 { 10147 if (!IS_ERR_OR_NULL(ptr)) 10148 return 0; 10149 10150 if (IS_ERR(ptr)) 10151 errno = -PTR_ERR(ptr); 10152 10153 /* If ptr == NULL, then errno should be already set by the failing 10154 * API, because libbpf never returns NULL on success and it now always 10155 * sets errno on error. So no extra errno handling for ptr == NULL 10156 * case. 10157 */ 10158 return -errno; 10159 } 10160 10161 /* Replace link's underlying BPF program with the new one */ 10162 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) 10163 { 10164 int ret; 10165 10166 ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL); 10167 return libbpf_err_errno(ret); 10168 } 10169 10170 /* Release "ownership" of underlying BPF resource (typically, BPF program 10171 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected 10172 * link, when destructed through bpf_link__destroy() call won't attempt to 10173 * detach/unregisted that BPF resource. This is useful in situations where, 10174 * say, attached BPF program has to outlive userspace program that attached it 10175 * in the system. Depending on type of BPF program, though, there might be 10176 * additional steps (like pinning BPF program in BPF FS) necessary to ensure 10177 * exit of userspace program doesn't trigger automatic detachment and clean up 10178 * inside the kernel. 10179 */ 10180 void bpf_link__disconnect(struct bpf_link *link) 10181 { 10182 link->disconnected = true; 10183 } 10184 10185 int bpf_link__destroy(struct bpf_link *link) 10186 { 10187 int err = 0; 10188 10189 if (IS_ERR_OR_NULL(link)) 10190 return 0; 10191 10192 if (!link->disconnected && link->detach) 10193 err = link->detach(link); 10194 if (link->pin_path) 10195 free(link->pin_path); 10196 if (link->dealloc) 10197 link->dealloc(link); 10198 else 10199 free(link); 10200 10201 return libbpf_err(err); 10202 } 10203 10204 int bpf_link__fd(const struct bpf_link *link) 10205 { 10206 return link->fd; 10207 } 10208 10209 const char *bpf_link__pin_path(const struct bpf_link *link) 10210 { 10211 return link->pin_path; 10212 } 10213 10214 static int bpf_link__detach_fd(struct bpf_link *link) 10215 { 10216 return libbpf_err_errno(close(link->fd)); 10217 } 10218 10219 struct bpf_link *bpf_link__open(const char *path) 10220 { 10221 struct bpf_link *link; 10222 int fd; 10223 10224 fd = bpf_obj_get(path); 10225 if (fd < 0) { 10226 fd = -errno; 10227 pr_warn("failed to open link at %s: %d\n", path, fd); 10228 return libbpf_err_ptr(fd); 10229 } 10230 10231 link = calloc(1, sizeof(*link)); 10232 if (!link) { 10233 close(fd); 10234 return libbpf_err_ptr(-ENOMEM); 10235 } 10236 link->detach = &bpf_link__detach_fd; 10237 link->fd = fd; 10238 10239 link->pin_path = strdup(path); 10240 if (!link->pin_path) { 10241 bpf_link__destroy(link); 10242 return libbpf_err_ptr(-ENOMEM); 10243 } 10244 10245 return link; 10246 } 10247 10248 int bpf_link__detach(struct bpf_link *link) 10249 { 10250 return bpf_link_detach(link->fd) ? -errno : 0; 10251 } 10252 10253 int bpf_link__pin(struct bpf_link *link, const char *path) 10254 { 10255 int err; 10256 10257 if (link->pin_path) 10258 return libbpf_err(-EBUSY); 10259 err = make_parent_dir(path); 10260 if (err) 10261 return libbpf_err(err); 10262 err = check_path(path); 10263 if (err) 10264 return libbpf_err(err); 10265 10266 link->pin_path = strdup(path); 10267 if (!link->pin_path) 10268 return libbpf_err(-ENOMEM); 10269 10270 if (bpf_obj_pin(link->fd, link->pin_path)) { 10271 err = -errno; 10272 zfree(&link->pin_path); 10273 return libbpf_err(err); 10274 } 10275 10276 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); 10277 return 0; 10278 } 10279 10280 int bpf_link__unpin(struct bpf_link *link) 10281 { 10282 int err; 10283 10284 if (!link->pin_path) 10285 return libbpf_err(-EINVAL); 10286 10287 err = unlink(link->pin_path); 10288 if (err != 0) 10289 return -errno; 10290 10291 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); 10292 zfree(&link->pin_path); 10293 return 0; 10294 } 10295 10296 struct bpf_link_perf { 10297 struct bpf_link link; 10298 int perf_event_fd; 10299 /* legacy kprobe support: keep track of probe identifier and type */ 10300 char *legacy_probe_name; 10301 bool legacy_is_kprobe; 10302 bool legacy_is_retprobe; 10303 }; 10304 10305 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe); 10306 static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe); 10307 10308 static int bpf_link_perf_detach(struct bpf_link *link) 10309 { 10310 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 10311 int err = 0; 10312 10313 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0) 10314 err = -errno; 10315 10316 if (perf_link->perf_event_fd != link->fd) 10317 close(perf_link->perf_event_fd); 10318 close(link->fd); 10319 10320 /* legacy uprobe/kprobe needs to be removed after perf event fd closure */ 10321 if (perf_link->legacy_probe_name) { 10322 if (perf_link->legacy_is_kprobe) { 10323 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name, 10324 perf_link->legacy_is_retprobe); 10325 } else { 10326 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name, 10327 perf_link->legacy_is_retprobe); 10328 } 10329 } 10330 10331 return err; 10332 } 10333 10334 static void bpf_link_perf_dealloc(struct bpf_link *link) 10335 { 10336 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 10337 10338 free(perf_link->legacy_probe_name); 10339 free(perf_link); 10340 } 10341 10342 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd, 10343 const struct bpf_perf_event_opts *opts) 10344 { 10345 char errmsg[STRERR_BUFSIZE]; 10346 struct bpf_link_perf *link; 10347 int prog_fd, link_fd = -1, err; 10348 bool force_ioctl_attach; 10349 10350 if (!OPTS_VALID(opts, bpf_perf_event_opts)) 10351 return libbpf_err_ptr(-EINVAL); 10352 10353 if (pfd < 0) { 10354 pr_warn("prog '%s': invalid perf event FD %d\n", 10355 prog->name, pfd); 10356 return libbpf_err_ptr(-EINVAL); 10357 } 10358 prog_fd = bpf_program__fd(prog); 10359 if (prog_fd < 0) { 10360 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n", 10361 prog->name); 10362 return libbpf_err_ptr(-EINVAL); 10363 } 10364 10365 link = calloc(1, sizeof(*link)); 10366 if (!link) 10367 return libbpf_err_ptr(-ENOMEM); 10368 link->link.detach = &bpf_link_perf_detach; 10369 link->link.dealloc = &bpf_link_perf_dealloc; 10370 link->perf_event_fd = pfd; 10371 10372 force_ioctl_attach = OPTS_GET(opts, force_ioctl_attach, false); 10373 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) { 10374 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts, 10375 .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0)); 10376 10377 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts); 10378 if (link_fd < 0) { 10379 err = -errno; 10380 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n", 10381 prog->name, pfd, 10382 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10383 goto err_out; 10384 } 10385 link->link.fd = link_fd; 10386 } else { 10387 if (OPTS_GET(opts, bpf_cookie, 0)) { 10388 pr_warn("prog '%s': user context value is not supported\n", prog->name); 10389 err = -EOPNOTSUPP; 10390 goto err_out; 10391 } 10392 10393 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 10394 err = -errno; 10395 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n", 10396 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10397 if (err == -EPROTO) 10398 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n", 10399 prog->name, pfd); 10400 goto err_out; 10401 } 10402 link->link.fd = pfd; 10403 } 10404 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 10405 err = -errno; 10406 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n", 10407 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10408 goto err_out; 10409 } 10410 10411 return &link->link; 10412 err_out: 10413 if (link_fd >= 0) 10414 close(link_fd); 10415 free(link); 10416 return libbpf_err_ptr(err); 10417 } 10418 10419 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd) 10420 { 10421 return bpf_program__attach_perf_event_opts(prog, pfd, NULL); 10422 } 10423 10424 /* 10425 * this function is expected to parse integer in the range of [0, 2^31-1] from 10426 * given file using scanf format string fmt. If actual parsed value is 10427 * negative, the result might be indistinguishable from error 10428 */ 10429 static int parse_uint_from_file(const char *file, const char *fmt) 10430 { 10431 char buf[STRERR_BUFSIZE]; 10432 int err, ret; 10433 FILE *f; 10434 10435 f = fopen(file, "re"); 10436 if (!f) { 10437 err = -errno; 10438 pr_debug("failed to open '%s': %s\n", file, 10439 libbpf_strerror_r(err, buf, sizeof(buf))); 10440 return err; 10441 } 10442 err = fscanf(f, fmt, &ret); 10443 if (err != 1) { 10444 err = err == EOF ? -EIO : -errno; 10445 pr_debug("failed to parse '%s': %s\n", file, 10446 libbpf_strerror_r(err, buf, sizeof(buf))); 10447 fclose(f); 10448 return err; 10449 } 10450 fclose(f); 10451 return ret; 10452 } 10453 10454 static int determine_kprobe_perf_type(void) 10455 { 10456 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 10457 10458 return parse_uint_from_file(file, "%d\n"); 10459 } 10460 10461 static int determine_uprobe_perf_type(void) 10462 { 10463 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 10464 10465 return parse_uint_from_file(file, "%d\n"); 10466 } 10467 10468 static int determine_kprobe_retprobe_bit(void) 10469 { 10470 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 10471 10472 return parse_uint_from_file(file, "config:%d\n"); 10473 } 10474 10475 static int determine_uprobe_retprobe_bit(void) 10476 { 10477 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 10478 10479 return parse_uint_from_file(file, "config:%d\n"); 10480 } 10481 10482 #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32 10483 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32 10484 10485 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 10486 uint64_t offset, int pid, size_t ref_ctr_off) 10487 { 10488 const size_t attr_sz = sizeof(struct perf_event_attr); 10489 struct perf_event_attr attr; 10490 char errmsg[STRERR_BUFSIZE]; 10491 int type, pfd; 10492 10493 if ((__u64)ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS)) 10494 return -EINVAL; 10495 10496 memset(&attr, 0, attr_sz); 10497 10498 type = uprobe ? determine_uprobe_perf_type() 10499 : determine_kprobe_perf_type(); 10500 if (type < 0) { 10501 pr_warn("failed to determine %s perf type: %s\n", 10502 uprobe ? "uprobe" : "kprobe", 10503 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 10504 return type; 10505 } 10506 if (retprobe) { 10507 int bit = uprobe ? determine_uprobe_retprobe_bit() 10508 : determine_kprobe_retprobe_bit(); 10509 10510 if (bit < 0) { 10511 pr_warn("failed to determine %s retprobe bit: %s\n", 10512 uprobe ? "uprobe" : "kprobe", 10513 libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); 10514 return bit; 10515 } 10516 attr.config |= 1 << bit; 10517 } 10518 attr.size = attr_sz; 10519 attr.type = type; 10520 attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT; 10521 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 10522 attr.config2 = offset; /* kprobe_addr or probe_offset */ 10523 10524 /* pid filter is meaningful only for uprobes */ 10525 pfd = syscall(__NR_perf_event_open, &attr, 10526 pid < 0 ? -1 : pid /* pid */, 10527 pid == -1 ? 0 : -1 /* cpu */, 10528 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 10529 return pfd >= 0 ? pfd : -errno; 10530 } 10531 10532 static int append_to_file(const char *file, const char *fmt, ...) 10533 { 10534 int fd, n, err = 0; 10535 va_list ap; 10536 char buf[1024]; 10537 10538 va_start(ap, fmt); 10539 n = vsnprintf(buf, sizeof(buf), fmt, ap); 10540 va_end(ap); 10541 10542 if (n < 0 || n >= sizeof(buf)) 10543 return -EINVAL; 10544 10545 fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0); 10546 if (fd < 0) 10547 return -errno; 10548 10549 if (write(fd, buf, n) < 0) 10550 err = -errno; 10551 10552 close(fd); 10553 return err; 10554 } 10555 10556 #define DEBUGFS "/sys/kernel/debug/tracing" 10557 #define TRACEFS "/sys/kernel/tracing" 10558 10559 static bool use_debugfs(void) 10560 { 10561 static int has_debugfs = -1; 10562 10563 if (has_debugfs < 0) 10564 has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0; 10565 10566 return has_debugfs == 1; 10567 } 10568 10569 static const char *tracefs_path(void) 10570 { 10571 return use_debugfs() ? DEBUGFS : TRACEFS; 10572 } 10573 10574 static const char *tracefs_kprobe_events(void) 10575 { 10576 return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events"; 10577 } 10578 10579 static const char *tracefs_uprobe_events(void) 10580 { 10581 return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events"; 10582 } 10583 10584 static const char *tracefs_available_filter_functions(void) 10585 { 10586 return use_debugfs() ? DEBUGFS"/available_filter_functions" 10587 : TRACEFS"/available_filter_functions"; 10588 } 10589 10590 static const char *tracefs_available_filter_functions_addrs(void) 10591 { 10592 return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" 10593 : TRACEFS"/available_filter_functions_addrs"; 10594 } 10595 10596 static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz, 10597 const char *kfunc_name, size_t offset) 10598 { 10599 static int index = 0; 10600 int i; 10601 10602 snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset, 10603 __sync_fetch_and_add(&index, 1)); 10604 10605 /* sanitize binary_path in the probe name */ 10606 for (i = 0; buf[i]; i++) { 10607 if (!isalnum(buf[i])) 10608 buf[i] = '_'; 10609 } 10610 } 10611 10612 static int add_kprobe_event_legacy(const char *probe_name, bool retprobe, 10613 const char *kfunc_name, size_t offset) 10614 { 10615 return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx", 10616 retprobe ? 'r' : 'p', 10617 retprobe ? "kretprobes" : "kprobes", 10618 probe_name, kfunc_name, offset); 10619 } 10620 10621 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe) 10622 { 10623 return append_to_file(tracefs_kprobe_events(), "-:%s/%s", 10624 retprobe ? "kretprobes" : "kprobes", probe_name); 10625 } 10626 10627 static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe) 10628 { 10629 char file[256]; 10630 10631 snprintf(file, sizeof(file), "%s/events/%s/%s/id", 10632 tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name); 10633 10634 return parse_uint_from_file(file, "%d\n"); 10635 } 10636 10637 static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, 10638 const char *kfunc_name, size_t offset, int pid) 10639 { 10640 const size_t attr_sz = sizeof(struct perf_event_attr); 10641 struct perf_event_attr attr; 10642 char errmsg[STRERR_BUFSIZE]; 10643 int type, pfd, err; 10644 10645 err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset); 10646 if (err < 0) { 10647 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n", 10648 kfunc_name, offset, 10649 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10650 return err; 10651 } 10652 type = determine_kprobe_perf_type_legacy(probe_name, retprobe); 10653 if (type < 0) { 10654 err = type; 10655 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n", 10656 kfunc_name, offset, 10657 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10658 goto err_clean_legacy; 10659 } 10660 10661 memset(&attr, 0, attr_sz); 10662 attr.size = attr_sz; 10663 attr.config = type; 10664 attr.type = PERF_TYPE_TRACEPOINT; 10665 10666 pfd = syscall(__NR_perf_event_open, &attr, 10667 pid < 0 ? -1 : pid, /* pid */ 10668 pid == -1 ? 0 : -1, /* cpu */ 10669 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 10670 if (pfd < 0) { 10671 err = -errno; 10672 pr_warn("legacy kprobe perf_event_open() failed: %s\n", 10673 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10674 goto err_clean_legacy; 10675 } 10676 return pfd; 10677 10678 err_clean_legacy: 10679 /* Clear the newly added legacy kprobe_event */ 10680 remove_kprobe_event_legacy(probe_name, retprobe); 10681 return err; 10682 } 10683 10684 static const char *arch_specific_syscall_pfx(void) 10685 { 10686 #if defined(__x86_64__) 10687 return "x64"; 10688 #elif defined(__i386__) 10689 return "ia32"; 10690 #elif defined(__s390x__) 10691 return "s390x"; 10692 #elif defined(__s390__) 10693 return "s390"; 10694 #elif defined(__arm__) 10695 return "arm"; 10696 #elif defined(__aarch64__) 10697 return "arm64"; 10698 #elif defined(__mips__) 10699 return "mips"; 10700 #elif defined(__riscv) 10701 return "riscv"; 10702 #elif defined(__powerpc__) 10703 return "powerpc"; 10704 #elif defined(__powerpc64__) 10705 return "powerpc64"; 10706 #else 10707 return NULL; 10708 #endif 10709 } 10710 10711 int probe_kern_syscall_wrapper(int token_fd) 10712 { 10713 char syscall_name[64]; 10714 const char *ksys_pfx; 10715 10716 ksys_pfx = arch_specific_syscall_pfx(); 10717 if (!ksys_pfx) 10718 return 0; 10719 10720 snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx); 10721 10722 if (determine_kprobe_perf_type() >= 0) { 10723 int pfd; 10724 10725 pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0); 10726 if (pfd >= 0) 10727 close(pfd); 10728 10729 return pfd >= 0 ? 1 : 0; 10730 } else { /* legacy mode */ 10731 char probe_name[128]; 10732 10733 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0); 10734 if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0) 10735 return 0; 10736 10737 (void)remove_kprobe_event_legacy(probe_name, false); 10738 return 1; 10739 } 10740 } 10741 10742 struct bpf_link * 10743 bpf_program__attach_kprobe_opts(const struct bpf_program *prog, 10744 const char *func_name, 10745 const struct bpf_kprobe_opts *opts) 10746 { 10747 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 10748 enum probe_attach_mode attach_mode; 10749 char errmsg[STRERR_BUFSIZE]; 10750 char *legacy_probe = NULL; 10751 struct bpf_link *link; 10752 size_t offset; 10753 bool retprobe, legacy; 10754 int pfd, err; 10755 10756 if (!OPTS_VALID(opts, bpf_kprobe_opts)) 10757 return libbpf_err_ptr(-EINVAL); 10758 10759 attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT); 10760 retprobe = OPTS_GET(opts, retprobe, false); 10761 offset = OPTS_GET(opts, offset, 0); 10762 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 10763 10764 legacy = determine_kprobe_perf_type() < 0; 10765 switch (attach_mode) { 10766 case PROBE_ATTACH_MODE_LEGACY: 10767 legacy = true; 10768 pe_opts.force_ioctl_attach = true; 10769 break; 10770 case PROBE_ATTACH_MODE_PERF: 10771 if (legacy) 10772 return libbpf_err_ptr(-ENOTSUP); 10773 pe_opts.force_ioctl_attach = true; 10774 break; 10775 case PROBE_ATTACH_MODE_LINK: 10776 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) 10777 return libbpf_err_ptr(-ENOTSUP); 10778 break; 10779 case PROBE_ATTACH_MODE_DEFAULT: 10780 break; 10781 default: 10782 return libbpf_err_ptr(-EINVAL); 10783 } 10784 10785 if (!legacy) { 10786 pfd = perf_event_open_probe(false /* uprobe */, retprobe, 10787 func_name, offset, 10788 -1 /* pid */, 0 /* ref_ctr_off */); 10789 } else { 10790 char probe_name[256]; 10791 10792 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), 10793 func_name, offset); 10794 10795 legacy_probe = strdup(probe_name); 10796 if (!legacy_probe) 10797 return libbpf_err_ptr(-ENOMEM); 10798 10799 pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name, 10800 offset, -1 /* pid */); 10801 } 10802 if (pfd < 0) { 10803 err = -errno; 10804 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n", 10805 prog->name, retprobe ? "kretprobe" : "kprobe", 10806 func_name, offset, 10807 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10808 goto err_out; 10809 } 10810 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 10811 err = libbpf_get_error(link); 10812 if (err) { 10813 close(pfd); 10814 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n", 10815 prog->name, retprobe ? "kretprobe" : "kprobe", 10816 func_name, offset, 10817 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10818 goto err_clean_legacy; 10819 } 10820 if (legacy) { 10821 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 10822 10823 perf_link->legacy_probe_name = legacy_probe; 10824 perf_link->legacy_is_kprobe = true; 10825 perf_link->legacy_is_retprobe = retprobe; 10826 } 10827 10828 return link; 10829 10830 err_clean_legacy: 10831 if (legacy) 10832 remove_kprobe_event_legacy(legacy_probe, retprobe); 10833 err_out: 10834 free(legacy_probe); 10835 return libbpf_err_ptr(err); 10836 } 10837 10838 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog, 10839 bool retprobe, 10840 const char *func_name) 10841 { 10842 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts, 10843 .retprobe = retprobe, 10844 ); 10845 10846 return bpf_program__attach_kprobe_opts(prog, func_name, &opts); 10847 } 10848 10849 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog, 10850 const char *syscall_name, 10851 const struct bpf_ksyscall_opts *opts) 10852 { 10853 LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); 10854 char func_name[128]; 10855 10856 if (!OPTS_VALID(opts, bpf_ksyscall_opts)) 10857 return libbpf_err_ptr(-EINVAL); 10858 10859 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { 10860 /* arch_specific_syscall_pfx() should never return NULL here 10861 * because it is guarded by kernel_supports(). However, since 10862 * compiler does not know that we have an explicit conditional 10863 * as well. 10864 */ 10865 snprintf(func_name, sizeof(func_name), "__%s_sys_%s", 10866 arch_specific_syscall_pfx() ? : "", syscall_name); 10867 } else { 10868 snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name); 10869 } 10870 10871 kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false); 10872 kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 10873 10874 return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts); 10875 } 10876 10877 /* Adapted from perf/util/string.c */ 10878 bool glob_match(const char *str, const char *pat) 10879 { 10880 while (*str && *pat && *pat != '*') { 10881 if (*pat == '?') { /* Matches any single character */ 10882 str++; 10883 pat++; 10884 continue; 10885 } 10886 if (*str != *pat) 10887 return false; 10888 str++; 10889 pat++; 10890 } 10891 /* Check wild card */ 10892 if (*pat == '*') { 10893 while (*pat == '*') 10894 pat++; 10895 if (!*pat) /* Tail wild card matches all */ 10896 return true; 10897 while (*str) 10898 if (glob_match(str++, pat)) 10899 return true; 10900 } 10901 return !*str && !*pat; 10902 } 10903 10904 struct kprobe_multi_resolve { 10905 const char *pattern; 10906 unsigned long *addrs; 10907 size_t cap; 10908 size_t cnt; 10909 }; 10910 10911 struct avail_kallsyms_data { 10912 char **syms; 10913 size_t cnt; 10914 struct kprobe_multi_resolve *res; 10915 }; 10916 10917 static int avail_func_cmp(const void *a, const void *b) 10918 { 10919 return strcmp(*(const char **)a, *(const char **)b); 10920 } 10921 10922 static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type, 10923 const char *sym_name, void *ctx) 10924 { 10925 struct avail_kallsyms_data *data = ctx; 10926 struct kprobe_multi_resolve *res = data->res; 10927 int err; 10928 10929 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) 10930 return 0; 10931 10932 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1); 10933 if (err) 10934 return err; 10935 10936 res->addrs[res->cnt++] = (unsigned long)sym_addr; 10937 return 0; 10938 } 10939 10940 static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res) 10941 { 10942 const char *available_functions_file = tracefs_available_filter_functions(); 10943 struct avail_kallsyms_data data; 10944 char sym_name[500]; 10945 FILE *f; 10946 int err = 0, ret, i; 10947 char **syms = NULL; 10948 size_t cap = 0, cnt = 0; 10949 10950 f = fopen(available_functions_file, "re"); 10951 if (!f) { 10952 err = -errno; 10953 pr_warn("failed to open %s: %d\n", available_functions_file, err); 10954 return err; 10955 } 10956 10957 while (true) { 10958 char *name; 10959 10960 ret = fscanf(f, "%499s%*[^\n]\n", sym_name); 10961 if (ret == EOF && feof(f)) 10962 break; 10963 10964 if (ret != 1) { 10965 pr_warn("failed to parse available_filter_functions entry: %d\n", ret); 10966 err = -EINVAL; 10967 goto cleanup; 10968 } 10969 10970 if (!glob_match(sym_name, res->pattern)) 10971 continue; 10972 10973 err = libbpf_ensure_mem((void **)&syms, &cap, sizeof(*syms), cnt + 1); 10974 if (err) 10975 goto cleanup; 10976 10977 name = strdup(sym_name); 10978 if (!name) { 10979 err = -errno; 10980 goto cleanup; 10981 } 10982 10983 syms[cnt++] = name; 10984 } 10985 10986 /* no entries found, bail out */ 10987 if (cnt == 0) { 10988 err = -ENOENT; 10989 goto cleanup; 10990 } 10991 10992 /* sort available functions */ 10993 qsort(syms, cnt, sizeof(*syms), avail_func_cmp); 10994 10995 data.syms = syms; 10996 data.res = res; 10997 data.cnt = cnt; 10998 libbpf_kallsyms_parse(avail_kallsyms_cb, &data); 10999 11000 if (res->cnt == 0) 11001 err = -ENOENT; 11002 11003 cleanup: 11004 for (i = 0; i < cnt; i++) 11005 free((char *)syms[i]); 11006 free(syms); 11007 11008 fclose(f); 11009 return err; 11010 } 11011 11012 static bool has_available_filter_functions_addrs(void) 11013 { 11014 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1; 11015 } 11016 11017 static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res) 11018 { 11019 const char *available_path = tracefs_available_filter_functions_addrs(); 11020 char sym_name[500]; 11021 FILE *f; 11022 int ret, err = 0; 11023 unsigned long long sym_addr; 11024 11025 f = fopen(available_path, "re"); 11026 if (!f) { 11027 err = -errno; 11028 pr_warn("failed to open %s: %d\n", available_path, err); 11029 return err; 11030 } 11031 11032 while (true) { 11033 ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name); 11034 if (ret == EOF && feof(f)) 11035 break; 11036 11037 if (ret != 2) { 11038 pr_warn("failed to parse available_filter_functions_addrs entry: %d\n", 11039 ret); 11040 err = -EINVAL; 11041 goto cleanup; 11042 } 11043 11044 if (!glob_match(sym_name, res->pattern)) 11045 continue; 11046 11047 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, 11048 sizeof(*res->addrs), res->cnt + 1); 11049 if (err) 11050 goto cleanup; 11051 11052 res->addrs[res->cnt++] = (unsigned long)sym_addr; 11053 } 11054 11055 if (res->cnt == 0) 11056 err = -ENOENT; 11057 11058 cleanup: 11059 fclose(f); 11060 return err; 11061 } 11062 11063 struct bpf_link * 11064 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, 11065 const char *pattern, 11066 const struct bpf_kprobe_multi_opts *opts) 11067 { 11068 LIBBPF_OPTS(bpf_link_create_opts, lopts); 11069 struct kprobe_multi_resolve res = { 11070 .pattern = pattern, 11071 }; 11072 struct bpf_link *link = NULL; 11073 char errmsg[STRERR_BUFSIZE]; 11074 const unsigned long *addrs; 11075 int err, link_fd, prog_fd; 11076 const __u64 *cookies; 11077 const char **syms; 11078 bool retprobe; 11079 size_t cnt; 11080 11081 if (!OPTS_VALID(opts, bpf_kprobe_multi_opts)) 11082 return libbpf_err_ptr(-EINVAL); 11083 11084 syms = OPTS_GET(opts, syms, false); 11085 addrs = OPTS_GET(opts, addrs, false); 11086 cnt = OPTS_GET(opts, cnt, false); 11087 cookies = OPTS_GET(opts, cookies, false); 11088 11089 if (!pattern && !addrs && !syms) 11090 return libbpf_err_ptr(-EINVAL); 11091 if (pattern && (addrs || syms || cookies || cnt)) 11092 return libbpf_err_ptr(-EINVAL); 11093 if (!pattern && !cnt) 11094 return libbpf_err_ptr(-EINVAL); 11095 if (addrs && syms) 11096 return libbpf_err_ptr(-EINVAL); 11097 11098 if (pattern) { 11099 if (has_available_filter_functions_addrs()) 11100 err = libbpf_available_kprobes_parse(&res); 11101 else 11102 err = libbpf_available_kallsyms_parse(&res); 11103 if (err) 11104 goto error; 11105 addrs = res.addrs; 11106 cnt = res.cnt; 11107 } 11108 11109 retprobe = OPTS_GET(opts, retprobe, false); 11110 11111 lopts.kprobe_multi.syms = syms; 11112 lopts.kprobe_multi.addrs = addrs; 11113 lopts.kprobe_multi.cookies = cookies; 11114 lopts.kprobe_multi.cnt = cnt; 11115 lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0; 11116 11117 link = calloc(1, sizeof(*link)); 11118 if (!link) { 11119 err = -ENOMEM; 11120 goto error; 11121 } 11122 link->detach = &bpf_link__detach_fd; 11123 11124 prog_fd = bpf_program__fd(prog); 11125 link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, &lopts); 11126 if (link_fd < 0) { 11127 err = -errno; 11128 pr_warn("prog '%s': failed to attach: %s\n", 11129 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11130 goto error; 11131 } 11132 link->fd = link_fd; 11133 free(res.addrs); 11134 return link; 11135 11136 error: 11137 free(link); 11138 free(res.addrs); 11139 return libbpf_err_ptr(err); 11140 } 11141 11142 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11143 { 11144 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts); 11145 unsigned long offset = 0; 11146 const char *func_name; 11147 char *func; 11148 int n; 11149 11150 *link = NULL; 11151 11152 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */ 11153 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0) 11154 return 0; 11155 11156 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/"); 11157 if (opts.retprobe) 11158 func_name = prog->sec_name + sizeof("kretprobe/") - 1; 11159 else 11160 func_name = prog->sec_name + sizeof("kprobe/") - 1; 11161 11162 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset); 11163 if (n < 1) { 11164 pr_warn("kprobe name is invalid: %s\n", func_name); 11165 return -EINVAL; 11166 } 11167 if (opts.retprobe && offset != 0) { 11168 free(func); 11169 pr_warn("kretprobes do not support offset specification\n"); 11170 return -EINVAL; 11171 } 11172 11173 opts.offset = offset; 11174 *link = bpf_program__attach_kprobe_opts(prog, func, &opts); 11175 free(func); 11176 return libbpf_get_error(*link); 11177 } 11178 11179 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11180 { 11181 LIBBPF_OPTS(bpf_ksyscall_opts, opts); 11182 const char *syscall_name; 11183 11184 *link = NULL; 11185 11186 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */ 11187 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0) 11188 return 0; 11189 11190 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/"); 11191 if (opts.retprobe) 11192 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1; 11193 else 11194 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1; 11195 11196 *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts); 11197 return *link ? 0 : -errno; 11198 } 11199 11200 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11201 { 11202 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); 11203 const char *spec; 11204 char *pattern; 11205 int n; 11206 11207 *link = NULL; 11208 11209 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */ 11210 if (strcmp(prog->sec_name, "kprobe.multi") == 0 || 11211 strcmp(prog->sec_name, "kretprobe.multi") == 0) 11212 return 0; 11213 11214 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/"); 11215 if (opts.retprobe) 11216 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1; 11217 else 11218 spec = prog->sec_name + sizeof("kprobe.multi/") - 1; 11219 11220 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern); 11221 if (n < 1) { 11222 pr_warn("kprobe multi pattern is invalid: %s\n", pattern); 11223 return -EINVAL; 11224 } 11225 11226 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts); 11227 free(pattern); 11228 return libbpf_get_error(*link); 11229 } 11230 11231 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11232 { 11233 char *probe_type = NULL, *binary_path = NULL, *func_name = NULL; 11234 LIBBPF_OPTS(bpf_uprobe_multi_opts, opts); 11235 int n, ret = -EINVAL; 11236 11237 *link = NULL; 11238 11239 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", 11240 &probe_type, &binary_path, &func_name); 11241 switch (n) { 11242 case 1: 11243 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ 11244 ret = 0; 11245 break; 11246 case 3: 11247 opts.retprobe = strcmp(probe_type, "uretprobe.multi") == 0; 11248 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts); 11249 ret = libbpf_get_error(*link); 11250 break; 11251 default: 11252 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, 11253 prog->sec_name); 11254 break; 11255 } 11256 free(probe_type); 11257 free(binary_path); 11258 free(func_name); 11259 return ret; 11260 } 11261 11262 static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz, 11263 const char *binary_path, uint64_t offset) 11264 { 11265 int i; 11266 11267 snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset); 11268 11269 /* sanitize binary_path in the probe name */ 11270 for (i = 0; buf[i]; i++) { 11271 if (!isalnum(buf[i])) 11272 buf[i] = '_'; 11273 } 11274 } 11275 11276 static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe, 11277 const char *binary_path, size_t offset) 11278 { 11279 return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx", 11280 retprobe ? 'r' : 'p', 11281 retprobe ? "uretprobes" : "uprobes", 11282 probe_name, binary_path, offset); 11283 } 11284 11285 static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe) 11286 { 11287 return append_to_file(tracefs_uprobe_events(), "-:%s/%s", 11288 retprobe ? "uretprobes" : "uprobes", probe_name); 11289 } 11290 11291 static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe) 11292 { 11293 char file[512]; 11294 11295 snprintf(file, sizeof(file), "%s/events/%s/%s/id", 11296 tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name); 11297 11298 return parse_uint_from_file(file, "%d\n"); 11299 } 11300 11301 static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, 11302 const char *binary_path, size_t offset, int pid) 11303 { 11304 const size_t attr_sz = sizeof(struct perf_event_attr); 11305 struct perf_event_attr attr; 11306 int type, pfd, err; 11307 11308 err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset); 11309 if (err < 0) { 11310 pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n", 11311 binary_path, (size_t)offset, err); 11312 return err; 11313 } 11314 type = determine_uprobe_perf_type_legacy(probe_name, retprobe); 11315 if (type < 0) { 11316 err = type; 11317 pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n", 11318 binary_path, offset, err); 11319 goto err_clean_legacy; 11320 } 11321 11322 memset(&attr, 0, attr_sz); 11323 attr.size = attr_sz; 11324 attr.config = type; 11325 attr.type = PERF_TYPE_TRACEPOINT; 11326 11327 pfd = syscall(__NR_perf_event_open, &attr, 11328 pid < 0 ? -1 : pid, /* pid */ 11329 pid == -1 ? 0 : -1, /* cpu */ 11330 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 11331 if (pfd < 0) { 11332 err = -errno; 11333 pr_warn("legacy uprobe perf_event_open() failed: %d\n", err); 11334 goto err_clean_legacy; 11335 } 11336 return pfd; 11337 11338 err_clean_legacy: 11339 /* Clear the newly added legacy uprobe_event */ 11340 remove_uprobe_event_legacy(probe_name, retprobe); 11341 return err; 11342 } 11343 11344 /* Find offset of function name in archive specified by path. Currently 11345 * supported are .zip files that do not compress their contents, as used on 11346 * Android in the form of APKs, for example. "file_name" is the name of the ELF 11347 * file inside the archive. "func_name" matches symbol name or name@@LIB for 11348 * library functions. 11349 * 11350 * An overview of the APK format specifically provided here: 11351 * https://en.wikipedia.org/w/index.php?title=Apk_(file_format)&oldid=1139099120#Package_contents 11352 */ 11353 static long elf_find_func_offset_from_archive(const char *archive_path, const char *file_name, 11354 const char *func_name) 11355 { 11356 struct zip_archive *archive; 11357 struct zip_entry entry; 11358 long ret; 11359 Elf *elf; 11360 11361 archive = zip_archive_open(archive_path); 11362 if (IS_ERR(archive)) { 11363 ret = PTR_ERR(archive); 11364 pr_warn("zip: failed to open %s: %ld\n", archive_path, ret); 11365 return ret; 11366 } 11367 11368 ret = zip_archive_find_entry(archive, file_name, &entry); 11369 if (ret) { 11370 pr_warn("zip: could not find archive member %s in %s: %ld\n", file_name, 11371 archive_path, ret); 11372 goto out; 11373 } 11374 pr_debug("zip: found entry for %s in %s at 0x%lx\n", file_name, archive_path, 11375 (unsigned long)entry.data_offset); 11376 11377 if (entry.compression) { 11378 pr_warn("zip: entry %s of %s is compressed and cannot be handled\n", file_name, 11379 archive_path); 11380 ret = -LIBBPF_ERRNO__FORMAT; 11381 goto out; 11382 } 11383 11384 elf = elf_memory((void *)entry.data, entry.data_length); 11385 if (!elf) { 11386 pr_warn("elf: could not read elf file %s from %s: %s\n", file_name, archive_path, 11387 elf_errmsg(-1)); 11388 ret = -LIBBPF_ERRNO__LIBELF; 11389 goto out; 11390 } 11391 11392 ret = elf_find_func_offset(elf, file_name, func_name); 11393 if (ret > 0) { 11394 pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n", 11395 func_name, file_name, archive_path, entry.data_offset, ret, 11396 ret + entry.data_offset); 11397 ret += entry.data_offset; 11398 } 11399 elf_end(elf); 11400 11401 out: 11402 zip_archive_close(archive); 11403 return ret; 11404 } 11405 11406 static const char *arch_specific_lib_paths(void) 11407 { 11408 /* 11409 * Based on https://packages.debian.org/sid/libc6. 11410 * 11411 * Assume that the traced program is built for the same architecture 11412 * as libbpf, which should cover the vast majority of cases. 11413 */ 11414 #if defined(__x86_64__) 11415 return "/lib/x86_64-linux-gnu"; 11416 #elif defined(__i386__) 11417 return "/lib/i386-linux-gnu"; 11418 #elif defined(__s390x__) 11419 return "/lib/s390x-linux-gnu"; 11420 #elif defined(__s390__) 11421 return "/lib/s390-linux-gnu"; 11422 #elif defined(__arm__) && defined(__SOFTFP__) 11423 return "/lib/arm-linux-gnueabi"; 11424 #elif defined(__arm__) && !defined(__SOFTFP__) 11425 return "/lib/arm-linux-gnueabihf"; 11426 #elif defined(__aarch64__) 11427 return "/lib/aarch64-linux-gnu"; 11428 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64 11429 return "/lib/mips64el-linux-gnuabi64"; 11430 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32 11431 return "/lib/mipsel-linux-gnu"; 11432 #elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 11433 return "/lib/powerpc64le-linux-gnu"; 11434 #elif defined(__sparc__) && defined(__arch64__) 11435 return "/lib/sparc64-linux-gnu"; 11436 #elif defined(__riscv) && __riscv_xlen == 64 11437 return "/lib/riscv64-linux-gnu"; 11438 #else 11439 return NULL; 11440 #endif 11441 } 11442 11443 /* Get full path to program/shared library. */ 11444 static int resolve_full_path(const char *file, char *result, size_t result_sz) 11445 { 11446 const char *search_paths[3] = {}; 11447 int i, perm; 11448 11449 if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { 11450 search_paths[0] = getenv("LD_LIBRARY_PATH"); 11451 search_paths[1] = "/usr/lib64:/usr/lib"; 11452 search_paths[2] = arch_specific_lib_paths(); 11453 perm = R_OK; 11454 } else { 11455 search_paths[0] = getenv("PATH"); 11456 search_paths[1] = "/usr/bin:/usr/sbin"; 11457 perm = R_OK | X_OK; 11458 } 11459 11460 for (i = 0; i < ARRAY_SIZE(search_paths); i++) { 11461 const char *s; 11462 11463 if (!search_paths[i]) 11464 continue; 11465 for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) { 11466 char *next_path; 11467 int seg_len; 11468 11469 if (s[0] == ':') 11470 s++; 11471 next_path = strchr(s, ':'); 11472 seg_len = next_path ? next_path - s : strlen(s); 11473 if (!seg_len) 11474 continue; 11475 snprintf(result, result_sz, "%.*s/%s", seg_len, s, file); 11476 /* ensure it has required permissions */ 11477 if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0) 11478 continue; 11479 pr_debug("resolved '%s' to '%s'\n", file, result); 11480 return 0; 11481 } 11482 } 11483 return -ENOENT; 11484 } 11485 11486 struct bpf_link * 11487 bpf_program__attach_uprobe_multi(const struct bpf_program *prog, 11488 pid_t pid, 11489 const char *path, 11490 const char *func_pattern, 11491 const struct bpf_uprobe_multi_opts *opts) 11492 { 11493 const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL; 11494 LIBBPF_OPTS(bpf_link_create_opts, lopts); 11495 unsigned long *resolved_offsets = NULL; 11496 int err = 0, link_fd, prog_fd; 11497 struct bpf_link *link = NULL; 11498 char errmsg[STRERR_BUFSIZE]; 11499 char full_path[PATH_MAX]; 11500 const __u64 *cookies; 11501 const char **syms; 11502 size_t cnt; 11503 11504 if (!OPTS_VALID(opts, bpf_uprobe_multi_opts)) 11505 return libbpf_err_ptr(-EINVAL); 11506 11507 syms = OPTS_GET(opts, syms, NULL); 11508 offsets = OPTS_GET(opts, offsets, NULL); 11509 ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL); 11510 cookies = OPTS_GET(opts, cookies, NULL); 11511 cnt = OPTS_GET(opts, cnt, 0); 11512 11513 /* 11514 * User can specify 2 mutually exclusive set of inputs: 11515 * 11516 * 1) use only path/func_pattern/pid arguments 11517 * 11518 * 2) use path/pid with allowed combinations of: 11519 * syms/offsets/ref_ctr_offsets/cookies/cnt 11520 * 11521 * - syms and offsets are mutually exclusive 11522 * - ref_ctr_offsets and cookies are optional 11523 * 11524 * Any other usage results in error. 11525 */ 11526 11527 if (!path) 11528 return libbpf_err_ptr(-EINVAL); 11529 if (!func_pattern && cnt == 0) 11530 return libbpf_err_ptr(-EINVAL); 11531 11532 if (func_pattern) { 11533 if (syms || offsets || ref_ctr_offsets || cookies || cnt) 11534 return libbpf_err_ptr(-EINVAL); 11535 } else { 11536 if (!!syms == !!offsets) 11537 return libbpf_err_ptr(-EINVAL); 11538 } 11539 11540 if (func_pattern) { 11541 if (!strchr(path, '/')) { 11542 err = resolve_full_path(path, full_path, sizeof(full_path)); 11543 if (err) { 11544 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", 11545 prog->name, path, err); 11546 return libbpf_err_ptr(err); 11547 } 11548 path = full_path; 11549 } 11550 11551 err = elf_resolve_pattern_offsets(path, func_pattern, 11552 &resolved_offsets, &cnt); 11553 if (err < 0) 11554 return libbpf_err_ptr(err); 11555 offsets = resolved_offsets; 11556 } else if (syms) { 11557 err = elf_resolve_syms_offsets(path, cnt, syms, &resolved_offsets, STT_FUNC); 11558 if (err < 0) 11559 return libbpf_err_ptr(err); 11560 offsets = resolved_offsets; 11561 } 11562 11563 lopts.uprobe_multi.path = path; 11564 lopts.uprobe_multi.offsets = offsets; 11565 lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets; 11566 lopts.uprobe_multi.cookies = cookies; 11567 lopts.uprobe_multi.cnt = cnt; 11568 lopts.uprobe_multi.flags = OPTS_GET(opts, retprobe, false) ? BPF_F_UPROBE_MULTI_RETURN : 0; 11569 11570 if (pid == 0) 11571 pid = getpid(); 11572 if (pid > 0) 11573 lopts.uprobe_multi.pid = pid; 11574 11575 link = calloc(1, sizeof(*link)); 11576 if (!link) { 11577 err = -ENOMEM; 11578 goto error; 11579 } 11580 link->detach = &bpf_link__detach_fd; 11581 11582 prog_fd = bpf_program__fd(prog); 11583 link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts); 11584 if (link_fd < 0) { 11585 err = -errno; 11586 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n", 11587 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11588 goto error; 11589 } 11590 link->fd = link_fd; 11591 free(resolved_offsets); 11592 return link; 11593 11594 error: 11595 free(resolved_offsets); 11596 free(link); 11597 return libbpf_err_ptr(err); 11598 } 11599 11600 LIBBPF_API struct bpf_link * 11601 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, 11602 const char *binary_path, size_t func_offset, 11603 const struct bpf_uprobe_opts *opts) 11604 { 11605 const char *archive_path = NULL, *archive_sep = NULL; 11606 char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL; 11607 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 11608 enum probe_attach_mode attach_mode; 11609 char full_path[PATH_MAX]; 11610 struct bpf_link *link; 11611 size_t ref_ctr_off; 11612 int pfd, err; 11613 bool retprobe, legacy; 11614 const char *func_name; 11615 11616 if (!OPTS_VALID(opts, bpf_uprobe_opts)) 11617 return libbpf_err_ptr(-EINVAL); 11618 11619 attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT); 11620 retprobe = OPTS_GET(opts, retprobe, false); 11621 ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0); 11622 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 11623 11624 if (!binary_path) 11625 return libbpf_err_ptr(-EINVAL); 11626 11627 /* Check if "binary_path" refers to an archive. */ 11628 archive_sep = strstr(binary_path, "!/"); 11629 if (archive_sep) { 11630 full_path[0] = '\0'; 11631 libbpf_strlcpy(full_path, binary_path, 11632 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1))); 11633 archive_path = full_path; 11634 binary_path = archive_sep + 2; 11635 } else if (!strchr(binary_path, '/')) { 11636 err = resolve_full_path(binary_path, full_path, sizeof(full_path)); 11637 if (err) { 11638 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", 11639 prog->name, binary_path, err); 11640 return libbpf_err_ptr(err); 11641 } 11642 binary_path = full_path; 11643 } 11644 func_name = OPTS_GET(opts, func_name, NULL); 11645 if (func_name) { 11646 long sym_off; 11647 11648 if (archive_path) { 11649 sym_off = elf_find_func_offset_from_archive(archive_path, binary_path, 11650 func_name); 11651 binary_path = archive_path; 11652 } else { 11653 sym_off = elf_find_func_offset_from_file(binary_path, func_name); 11654 } 11655 if (sym_off < 0) 11656 return libbpf_err_ptr(sym_off); 11657 func_offset += sym_off; 11658 } 11659 11660 legacy = determine_uprobe_perf_type() < 0; 11661 switch (attach_mode) { 11662 case PROBE_ATTACH_MODE_LEGACY: 11663 legacy = true; 11664 pe_opts.force_ioctl_attach = true; 11665 break; 11666 case PROBE_ATTACH_MODE_PERF: 11667 if (legacy) 11668 return libbpf_err_ptr(-ENOTSUP); 11669 pe_opts.force_ioctl_attach = true; 11670 break; 11671 case PROBE_ATTACH_MODE_LINK: 11672 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) 11673 return libbpf_err_ptr(-ENOTSUP); 11674 break; 11675 case PROBE_ATTACH_MODE_DEFAULT: 11676 break; 11677 default: 11678 return libbpf_err_ptr(-EINVAL); 11679 } 11680 11681 if (!legacy) { 11682 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path, 11683 func_offset, pid, ref_ctr_off); 11684 } else { 11685 char probe_name[PATH_MAX + 64]; 11686 11687 if (ref_ctr_off) 11688 return libbpf_err_ptr(-EINVAL); 11689 11690 gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name), 11691 binary_path, func_offset); 11692 11693 legacy_probe = strdup(probe_name); 11694 if (!legacy_probe) 11695 return libbpf_err_ptr(-ENOMEM); 11696 11697 pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe, 11698 binary_path, func_offset, pid); 11699 } 11700 if (pfd < 0) { 11701 err = -errno; 11702 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 11703 prog->name, retprobe ? "uretprobe" : "uprobe", 11704 binary_path, func_offset, 11705 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11706 goto err_out; 11707 } 11708 11709 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 11710 err = libbpf_get_error(link); 11711 if (err) { 11712 close(pfd); 11713 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n", 11714 prog->name, retprobe ? "uretprobe" : "uprobe", 11715 binary_path, func_offset, 11716 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11717 goto err_clean_legacy; 11718 } 11719 if (legacy) { 11720 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 11721 11722 perf_link->legacy_probe_name = legacy_probe; 11723 perf_link->legacy_is_kprobe = false; 11724 perf_link->legacy_is_retprobe = retprobe; 11725 } 11726 return link; 11727 11728 err_clean_legacy: 11729 if (legacy) 11730 remove_uprobe_event_legacy(legacy_probe, retprobe); 11731 err_out: 11732 free(legacy_probe); 11733 return libbpf_err_ptr(err); 11734 } 11735 11736 /* Format of u[ret]probe section definition supporting auto-attach: 11737 * u[ret]probe/binary:function[+offset] 11738 * 11739 * binary can be an absolute/relative path or a filename; the latter is resolved to a 11740 * full binary path via bpf_program__attach_uprobe_opts. 11741 * 11742 * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be 11743 * specified (and auto-attach is not possible) or the above format is specified for 11744 * auto-attach. 11745 */ 11746 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11747 { 11748 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts); 11749 char *probe_type = NULL, *binary_path = NULL, *func_name = NULL, *func_off; 11750 int n, c, ret = -EINVAL; 11751 long offset = 0; 11752 11753 *link = NULL; 11754 11755 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", 11756 &probe_type, &binary_path, &func_name); 11757 switch (n) { 11758 case 1: 11759 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ 11760 ret = 0; 11761 break; 11762 case 2: 11763 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n", 11764 prog->name, prog->sec_name); 11765 break; 11766 case 3: 11767 /* check if user specifies `+offset`, if yes, this should be 11768 * the last part of the string, make sure sscanf read to EOL 11769 */ 11770 func_off = strrchr(func_name, '+'); 11771 if (func_off) { 11772 n = sscanf(func_off, "+%li%n", &offset, &c); 11773 if (n == 1 && *(func_off + c) == '\0') 11774 func_off[0] = '\0'; 11775 else 11776 offset = 0; 11777 } 11778 opts.retprobe = strcmp(probe_type, "uretprobe") == 0 || 11779 strcmp(probe_type, "uretprobe.s") == 0; 11780 if (opts.retprobe && offset != 0) { 11781 pr_warn("prog '%s': uretprobes do not support offset specification\n", 11782 prog->name); 11783 break; 11784 } 11785 opts.func_name = func_name; 11786 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts); 11787 ret = libbpf_get_error(*link); 11788 break; 11789 default: 11790 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, 11791 prog->sec_name); 11792 break; 11793 } 11794 free(probe_type); 11795 free(binary_path); 11796 free(func_name); 11797 11798 return ret; 11799 } 11800 11801 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog, 11802 bool retprobe, pid_t pid, 11803 const char *binary_path, 11804 size_t func_offset) 11805 { 11806 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe); 11807 11808 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts); 11809 } 11810 11811 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog, 11812 pid_t pid, const char *binary_path, 11813 const char *usdt_provider, const char *usdt_name, 11814 const struct bpf_usdt_opts *opts) 11815 { 11816 char resolved_path[512]; 11817 struct bpf_object *obj = prog->obj; 11818 struct bpf_link *link; 11819 __u64 usdt_cookie; 11820 int err; 11821 11822 if (!OPTS_VALID(opts, bpf_uprobe_opts)) 11823 return libbpf_err_ptr(-EINVAL); 11824 11825 if (bpf_program__fd(prog) < 0) { 11826 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n", 11827 prog->name); 11828 return libbpf_err_ptr(-EINVAL); 11829 } 11830 11831 if (!binary_path) 11832 return libbpf_err_ptr(-EINVAL); 11833 11834 if (!strchr(binary_path, '/')) { 11835 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path)); 11836 if (err) { 11837 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", 11838 prog->name, binary_path, err); 11839 return libbpf_err_ptr(err); 11840 } 11841 binary_path = resolved_path; 11842 } 11843 11844 /* USDT manager is instantiated lazily on first USDT attach. It will 11845 * be destroyed together with BPF object in bpf_object__close(). 11846 */ 11847 if (IS_ERR(obj->usdt_man)) 11848 return libbpf_ptr(obj->usdt_man); 11849 if (!obj->usdt_man) { 11850 obj->usdt_man = usdt_manager_new(obj); 11851 if (IS_ERR(obj->usdt_man)) 11852 return libbpf_ptr(obj->usdt_man); 11853 } 11854 11855 usdt_cookie = OPTS_GET(opts, usdt_cookie, 0); 11856 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path, 11857 usdt_provider, usdt_name, usdt_cookie); 11858 err = libbpf_get_error(link); 11859 if (err) 11860 return libbpf_err_ptr(err); 11861 return link; 11862 } 11863 11864 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11865 { 11866 char *path = NULL, *provider = NULL, *name = NULL; 11867 const char *sec_name; 11868 int n, err; 11869 11870 sec_name = bpf_program__section_name(prog); 11871 if (strcmp(sec_name, "usdt") == 0) { 11872 /* no auto-attach for just SEC("usdt") */ 11873 *link = NULL; 11874 return 0; 11875 } 11876 11877 n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name); 11878 if (n != 3) { 11879 pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n", 11880 sec_name); 11881 err = -EINVAL; 11882 } else { 11883 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path, 11884 provider, name, NULL); 11885 err = libbpf_get_error(*link); 11886 } 11887 free(path); 11888 free(provider); 11889 free(name); 11890 return err; 11891 } 11892 11893 static int determine_tracepoint_id(const char *tp_category, 11894 const char *tp_name) 11895 { 11896 char file[PATH_MAX]; 11897 int ret; 11898 11899 ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id", 11900 tracefs_path(), tp_category, tp_name); 11901 if (ret < 0) 11902 return -errno; 11903 if (ret >= sizeof(file)) { 11904 pr_debug("tracepoint %s/%s path is too long\n", 11905 tp_category, tp_name); 11906 return -E2BIG; 11907 } 11908 return parse_uint_from_file(file, "%d\n"); 11909 } 11910 11911 static int perf_event_open_tracepoint(const char *tp_category, 11912 const char *tp_name) 11913 { 11914 const size_t attr_sz = sizeof(struct perf_event_attr); 11915 struct perf_event_attr attr; 11916 char errmsg[STRERR_BUFSIZE]; 11917 int tp_id, pfd, err; 11918 11919 tp_id = determine_tracepoint_id(tp_category, tp_name); 11920 if (tp_id < 0) { 11921 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 11922 tp_category, tp_name, 11923 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 11924 return tp_id; 11925 } 11926 11927 memset(&attr, 0, attr_sz); 11928 attr.type = PERF_TYPE_TRACEPOINT; 11929 attr.size = attr_sz; 11930 attr.config = tp_id; 11931 11932 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 11933 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 11934 if (pfd < 0) { 11935 err = -errno; 11936 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", 11937 tp_category, tp_name, 11938 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11939 return err; 11940 } 11941 return pfd; 11942 } 11943 11944 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, 11945 const char *tp_category, 11946 const char *tp_name, 11947 const struct bpf_tracepoint_opts *opts) 11948 { 11949 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 11950 char errmsg[STRERR_BUFSIZE]; 11951 struct bpf_link *link; 11952 int pfd, err; 11953 11954 if (!OPTS_VALID(opts, bpf_tracepoint_opts)) 11955 return libbpf_err_ptr(-EINVAL); 11956 11957 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 11958 11959 pfd = perf_event_open_tracepoint(tp_category, tp_name); 11960 if (pfd < 0) { 11961 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 11962 prog->name, tp_category, tp_name, 11963 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 11964 return libbpf_err_ptr(pfd); 11965 } 11966 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 11967 err = libbpf_get_error(link); 11968 if (err) { 11969 close(pfd); 11970 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n", 11971 prog->name, tp_category, tp_name, 11972 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11973 return libbpf_err_ptr(err); 11974 } 11975 return link; 11976 } 11977 11978 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog, 11979 const char *tp_category, 11980 const char *tp_name) 11981 { 11982 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL); 11983 } 11984 11985 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11986 { 11987 char *sec_name, *tp_cat, *tp_name; 11988 11989 *link = NULL; 11990 11991 /* no auto-attach for SEC("tp") or SEC("tracepoint") */ 11992 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0) 11993 return 0; 11994 11995 sec_name = strdup(prog->sec_name); 11996 if (!sec_name) 11997 return -ENOMEM; 11998 11999 /* extract "tp/<category>/<name>" or "tracepoint/<category>/<name>" */ 12000 if (str_has_pfx(prog->sec_name, "tp/")) 12001 tp_cat = sec_name + sizeof("tp/") - 1; 12002 else 12003 tp_cat = sec_name + sizeof("tracepoint/") - 1; 12004 tp_name = strchr(tp_cat, '/'); 12005 if (!tp_name) { 12006 free(sec_name); 12007 return -EINVAL; 12008 } 12009 *tp_name = '\0'; 12010 tp_name++; 12011 12012 *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); 12013 free(sec_name); 12014 return libbpf_get_error(*link); 12015 } 12016 12017 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog, 12018 const char *tp_name) 12019 { 12020 char errmsg[STRERR_BUFSIZE]; 12021 struct bpf_link *link; 12022 int prog_fd, pfd; 12023 12024 prog_fd = bpf_program__fd(prog); 12025 if (prog_fd < 0) { 12026 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12027 return libbpf_err_ptr(-EINVAL); 12028 } 12029 12030 link = calloc(1, sizeof(*link)); 12031 if (!link) 12032 return libbpf_err_ptr(-ENOMEM); 12033 link->detach = &bpf_link__detach_fd; 12034 12035 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 12036 if (pfd < 0) { 12037 pfd = -errno; 12038 free(link); 12039 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n", 12040 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 12041 return libbpf_err_ptr(pfd); 12042 } 12043 link->fd = pfd; 12044 return link; 12045 } 12046 12047 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12048 { 12049 static const char *const prefixes[] = { 12050 "raw_tp", 12051 "raw_tracepoint", 12052 "raw_tp.w", 12053 "raw_tracepoint.w", 12054 }; 12055 size_t i; 12056 const char *tp_name = NULL; 12057 12058 *link = NULL; 12059 12060 for (i = 0; i < ARRAY_SIZE(prefixes); i++) { 12061 size_t pfx_len; 12062 12063 if (!str_has_pfx(prog->sec_name, prefixes[i])) 12064 continue; 12065 12066 pfx_len = strlen(prefixes[i]); 12067 /* no auto-attach case of, e.g., SEC("raw_tp") */ 12068 if (prog->sec_name[pfx_len] == '\0') 12069 return 0; 12070 12071 if (prog->sec_name[pfx_len] != '/') 12072 continue; 12073 12074 tp_name = prog->sec_name + pfx_len + 1; 12075 break; 12076 } 12077 12078 if (!tp_name) { 12079 pr_warn("prog '%s': invalid section name '%s'\n", 12080 prog->name, prog->sec_name); 12081 return -EINVAL; 12082 } 12083 12084 *link = bpf_program__attach_raw_tracepoint(prog, tp_name); 12085 return libbpf_get_error(*link); 12086 } 12087 12088 /* Common logic for all BPF program types that attach to a btf_id */ 12089 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog, 12090 const struct bpf_trace_opts *opts) 12091 { 12092 LIBBPF_OPTS(bpf_link_create_opts, link_opts); 12093 char errmsg[STRERR_BUFSIZE]; 12094 struct bpf_link *link; 12095 int prog_fd, pfd; 12096 12097 if (!OPTS_VALID(opts, bpf_trace_opts)) 12098 return libbpf_err_ptr(-EINVAL); 12099 12100 prog_fd = bpf_program__fd(prog); 12101 if (prog_fd < 0) { 12102 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12103 return libbpf_err_ptr(-EINVAL); 12104 } 12105 12106 link = calloc(1, sizeof(*link)); 12107 if (!link) 12108 return libbpf_err_ptr(-ENOMEM); 12109 link->detach = &bpf_link__detach_fd; 12110 12111 /* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */ 12112 link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0); 12113 pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts); 12114 if (pfd < 0) { 12115 pfd = -errno; 12116 free(link); 12117 pr_warn("prog '%s': failed to attach: %s\n", 12118 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 12119 return libbpf_err_ptr(pfd); 12120 } 12121 link->fd = pfd; 12122 return link; 12123 } 12124 12125 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) 12126 { 12127 return bpf_program__attach_btf_id(prog, NULL); 12128 } 12129 12130 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog, 12131 const struct bpf_trace_opts *opts) 12132 { 12133 return bpf_program__attach_btf_id(prog, opts); 12134 } 12135 12136 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog) 12137 { 12138 return bpf_program__attach_btf_id(prog, NULL); 12139 } 12140 12141 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12142 { 12143 *link = bpf_program__attach_trace(prog); 12144 return libbpf_get_error(*link); 12145 } 12146 12147 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12148 { 12149 *link = bpf_program__attach_lsm(prog); 12150 return libbpf_get_error(*link); 12151 } 12152 12153 static struct bpf_link * 12154 bpf_program_attach_fd(const struct bpf_program *prog, 12155 int target_fd, const char *target_name, 12156 const struct bpf_link_create_opts *opts) 12157 { 12158 enum bpf_attach_type attach_type; 12159 char errmsg[STRERR_BUFSIZE]; 12160 struct bpf_link *link; 12161 int prog_fd, link_fd; 12162 12163 prog_fd = bpf_program__fd(prog); 12164 if (prog_fd < 0) { 12165 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12166 return libbpf_err_ptr(-EINVAL); 12167 } 12168 12169 link = calloc(1, sizeof(*link)); 12170 if (!link) 12171 return libbpf_err_ptr(-ENOMEM); 12172 link->detach = &bpf_link__detach_fd; 12173 12174 attach_type = bpf_program__expected_attach_type(prog); 12175 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, opts); 12176 if (link_fd < 0) { 12177 link_fd = -errno; 12178 free(link); 12179 pr_warn("prog '%s': failed to attach to %s: %s\n", 12180 prog->name, target_name, 12181 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 12182 return libbpf_err_ptr(link_fd); 12183 } 12184 link->fd = link_fd; 12185 return link; 12186 } 12187 12188 struct bpf_link * 12189 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd) 12190 { 12191 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL); 12192 } 12193 12194 struct bpf_link * 12195 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd) 12196 { 12197 return bpf_program_attach_fd(prog, netns_fd, "netns", NULL); 12198 } 12199 12200 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex) 12201 { 12202 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 12203 return bpf_program_attach_fd(prog, ifindex, "xdp", NULL); 12204 } 12205 12206 struct bpf_link * 12207 bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex, 12208 const struct bpf_tcx_opts *opts) 12209 { 12210 LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 12211 __u32 relative_id; 12212 int relative_fd; 12213 12214 if (!OPTS_VALID(opts, bpf_tcx_opts)) 12215 return libbpf_err_ptr(-EINVAL); 12216 12217 relative_id = OPTS_GET(opts, relative_id, 0); 12218 relative_fd = OPTS_GET(opts, relative_fd, 0); 12219 12220 /* validate we don't have unexpected combinations of non-zero fields */ 12221 if (!ifindex) { 12222 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n", 12223 prog->name); 12224 return libbpf_err_ptr(-EINVAL); 12225 } 12226 if (relative_fd && relative_id) { 12227 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n", 12228 prog->name); 12229 return libbpf_err_ptr(-EINVAL); 12230 } 12231 12232 link_create_opts.tcx.expected_revision = OPTS_GET(opts, expected_revision, 0); 12233 link_create_opts.tcx.relative_fd = relative_fd; 12234 link_create_opts.tcx.relative_id = relative_id; 12235 link_create_opts.flags = OPTS_GET(opts, flags, 0); 12236 12237 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 12238 return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts); 12239 } 12240 12241 struct bpf_link * 12242 bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex, 12243 const struct bpf_netkit_opts *opts) 12244 { 12245 LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 12246 __u32 relative_id; 12247 int relative_fd; 12248 12249 if (!OPTS_VALID(opts, bpf_netkit_opts)) 12250 return libbpf_err_ptr(-EINVAL); 12251 12252 relative_id = OPTS_GET(opts, relative_id, 0); 12253 relative_fd = OPTS_GET(opts, relative_fd, 0); 12254 12255 /* validate we don't have unexpected combinations of non-zero fields */ 12256 if (!ifindex) { 12257 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n", 12258 prog->name); 12259 return libbpf_err_ptr(-EINVAL); 12260 } 12261 if (relative_fd && relative_id) { 12262 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n", 12263 prog->name); 12264 return libbpf_err_ptr(-EINVAL); 12265 } 12266 12267 link_create_opts.netkit.expected_revision = OPTS_GET(opts, expected_revision, 0); 12268 link_create_opts.netkit.relative_fd = relative_fd; 12269 link_create_opts.netkit.relative_id = relative_id; 12270 link_create_opts.flags = OPTS_GET(opts, flags, 0); 12271 12272 return bpf_program_attach_fd(prog, ifindex, "netkit", &link_create_opts); 12273 } 12274 12275 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, 12276 int target_fd, 12277 const char *attach_func_name) 12278 { 12279 int btf_id; 12280 12281 if (!!target_fd != !!attach_func_name) { 12282 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n", 12283 prog->name); 12284 return libbpf_err_ptr(-EINVAL); 12285 } 12286 12287 if (prog->type != BPF_PROG_TYPE_EXT) { 12288 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace", 12289 prog->name); 12290 return libbpf_err_ptr(-EINVAL); 12291 } 12292 12293 if (target_fd) { 12294 LIBBPF_OPTS(bpf_link_create_opts, target_opts); 12295 12296 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd); 12297 if (btf_id < 0) 12298 return libbpf_err_ptr(btf_id); 12299 12300 target_opts.target_btf_id = btf_id; 12301 12302 return bpf_program_attach_fd(prog, target_fd, "freplace", 12303 &target_opts); 12304 } else { 12305 /* no target, so use raw_tracepoint_open for compatibility 12306 * with old kernels 12307 */ 12308 return bpf_program__attach_trace(prog); 12309 } 12310 } 12311 12312 struct bpf_link * 12313 bpf_program__attach_iter(const struct bpf_program *prog, 12314 const struct bpf_iter_attach_opts *opts) 12315 { 12316 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 12317 char errmsg[STRERR_BUFSIZE]; 12318 struct bpf_link *link; 12319 int prog_fd, link_fd; 12320 __u32 target_fd = 0; 12321 12322 if (!OPTS_VALID(opts, bpf_iter_attach_opts)) 12323 return libbpf_err_ptr(-EINVAL); 12324 12325 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0); 12326 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0); 12327 12328 prog_fd = bpf_program__fd(prog); 12329 if (prog_fd < 0) { 12330 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12331 return libbpf_err_ptr(-EINVAL); 12332 } 12333 12334 link = calloc(1, sizeof(*link)); 12335 if (!link) 12336 return libbpf_err_ptr(-ENOMEM); 12337 link->detach = &bpf_link__detach_fd; 12338 12339 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER, 12340 &link_create_opts); 12341 if (link_fd < 0) { 12342 link_fd = -errno; 12343 free(link); 12344 pr_warn("prog '%s': failed to attach to iterator: %s\n", 12345 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 12346 return libbpf_err_ptr(link_fd); 12347 } 12348 link->fd = link_fd; 12349 return link; 12350 } 12351 12352 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12353 { 12354 *link = bpf_program__attach_iter(prog, NULL); 12355 return libbpf_get_error(*link); 12356 } 12357 12358 struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog, 12359 const struct bpf_netfilter_opts *opts) 12360 { 12361 LIBBPF_OPTS(bpf_link_create_opts, lopts); 12362 struct bpf_link *link; 12363 int prog_fd, link_fd; 12364 12365 if (!OPTS_VALID(opts, bpf_netfilter_opts)) 12366 return libbpf_err_ptr(-EINVAL); 12367 12368 prog_fd = bpf_program__fd(prog); 12369 if (prog_fd < 0) { 12370 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12371 return libbpf_err_ptr(-EINVAL); 12372 } 12373 12374 link = calloc(1, sizeof(*link)); 12375 if (!link) 12376 return libbpf_err_ptr(-ENOMEM); 12377 12378 link->detach = &bpf_link__detach_fd; 12379 12380 lopts.netfilter.pf = OPTS_GET(opts, pf, 0); 12381 lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0); 12382 lopts.netfilter.priority = OPTS_GET(opts, priority, 0); 12383 lopts.netfilter.flags = OPTS_GET(opts, flags, 0); 12384 12385 link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts); 12386 if (link_fd < 0) { 12387 char errmsg[STRERR_BUFSIZE]; 12388 12389 link_fd = -errno; 12390 free(link); 12391 pr_warn("prog '%s': failed to attach to netfilter: %s\n", 12392 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 12393 return libbpf_err_ptr(link_fd); 12394 } 12395 link->fd = link_fd; 12396 12397 return link; 12398 } 12399 12400 struct bpf_link *bpf_program__attach(const struct bpf_program *prog) 12401 { 12402 struct bpf_link *link = NULL; 12403 int err; 12404 12405 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) 12406 return libbpf_err_ptr(-EOPNOTSUPP); 12407 12408 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link); 12409 if (err) 12410 return libbpf_err_ptr(err); 12411 12412 /* When calling bpf_program__attach() explicitly, auto-attach support 12413 * is expected to work, so NULL returned link is considered an error. 12414 * This is different for skeleton's attach, see comment in 12415 * bpf_object__attach_skeleton(). 12416 */ 12417 if (!link) 12418 return libbpf_err_ptr(-EOPNOTSUPP); 12419 12420 return link; 12421 } 12422 12423 struct bpf_link_struct_ops { 12424 struct bpf_link link; 12425 int map_fd; 12426 }; 12427 12428 static int bpf_link__detach_struct_ops(struct bpf_link *link) 12429 { 12430 struct bpf_link_struct_ops *st_link; 12431 __u32 zero = 0; 12432 12433 st_link = container_of(link, struct bpf_link_struct_ops, link); 12434 12435 if (st_link->map_fd < 0) 12436 /* w/o a real link */ 12437 return bpf_map_delete_elem(link->fd, &zero); 12438 12439 return close(link->fd); 12440 } 12441 12442 struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map) 12443 { 12444 struct bpf_link_struct_ops *link; 12445 __u32 zero = 0; 12446 int err, fd; 12447 12448 if (!bpf_map__is_struct_ops(map) || map->fd == -1) 12449 return libbpf_err_ptr(-EINVAL); 12450 12451 link = calloc(1, sizeof(*link)); 12452 if (!link) 12453 return libbpf_err_ptr(-EINVAL); 12454 12455 /* kern_vdata should be prepared during the loading phase. */ 12456 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); 12457 /* It can be EBUSY if the map has been used to create or 12458 * update a link before. We don't allow updating the value of 12459 * a struct_ops once it is set. That ensures that the value 12460 * never changed. So, it is safe to skip EBUSY. 12461 */ 12462 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) { 12463 free(link); 12464 return libbpf_err_ptr(err); 12465 } 12466 12467 link->link.detach = bpf_link__detach_struct_ops; 12468 12469 if (!(map->def.map_flags & BPF_F_LINK)) { 12470 /* w/o a real link */ 12471 link->link.fd = map->fd; 12472 link->map_fd = -1; 12473 return &link->link; 12474 } 12475 12476 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL); 12477 if (fd < 0) { 12478 free(link); 12479 return libbpf_err_ptr(fd); 12480 } 12481 12482 link->link.fd = fd; 12483 link->map_fd = map->fd; 12484 12485 return &link->link; 12486 } 12487 12488 /* 12489 * Swap the back struct_ops of a link with a new struct_ops map. 12490 */ 12491 int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map) 12492 { 12493 struct bpf_link_struct_ops *st_ops_link; 12494 __u32 zero = 0; 12495 int err; 12496 12497 if (!bpf_map__is_struct_ops(map) || !map_is_created(map)) 12498 return -EINVAL; 12499 12500 st_ops_link = container_of(link, struct bpf_link_struct_ops, link); 12501 /* Ensure the type of a link is correct */ 12502 if (st_ops_link->map_fd < 0) 12503 return -EINVAL; 12504 12505 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); 12506 /* It can be EBUSY if the map has been used to create or 12507 * update a link before. We don't allow updating the value of 12508 * a struct_ops once it is set. That ensures that the value 12509 * never changed. So, it is safe to skip EBUSY. 12510 */ 12511 if (err && err != -EBUSY) 12512 return err; 12513 12514 err = bpf_link_update(link->fd, map->fd, NULL); 12515 if (err < 0) 12516 return err; 12517 12518 st_ops_link->map_fd = map->fd; 12519 12520 return 0; 12521 } 12522 12523 typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr, 12524 void *private_data); 12525 12526 static enum bpf_perf_event_ret 12527 perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 12528 void **copy_mem, size_t *copy_size, 12529 bpf_perf_event_print_t fn, void *private_data) 12530 { 12531 struct perf_event_mmap_page *header = mmap_mem; 12532 __u64 data_head = ring_buffer_read_head(header); 12533 __u64 data_tail = header->data_tail; 12534 void *base = ((__u8 *)header) + page_size; 12535 int ret = LIBBPF_PERF_EVENT_CONT; 12536 struct perf_event_header *ehdr; 12537 size_t ehdr_size; 12538 12539 while (data_head != data_tail) { 12540 ehdr = base + (data_tail & (mmap_size - 1)); 12541 ehdr_size = ehdr->size; 12542 12543 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 12544 void *copy_start = ehdr; 12545 size_t len_first = base + mmap_size - copy_start; 12546 size_t len_secnd = ehdr_size - len_first; 12547 12548 if (*copy_size < ehdr_size) { 12549 free(*copy_mem); 12550 *copy_mem = malloc(ehdr_size); 12551 if (!*copy_mem) { 12552 *copy_size = 0; 12553 ret = LIBBPF_PERF_EVENT_ERROR; 12554 break; 12555 } 12556 *copy_size = ehdr_size; 12557 } 12558 12559 memcpy(*copy_mem, copy_start, len_first); 12560 memcpy(*copy_mem + len_first, base, len_secnd); 12561 ehdr = *copy_mem; 12562 } 12563 12564 ret = fn(ehdr, private_data); 12565 data_tail += ehdr_size; 12566 if (ret != LIBBPF_PERF_EVENT_CONT) 12567 break; 12568 } 12569 12570 ring_buffer_write_tail(header, data_tail); 12571 return libbpf_err(ret); 12572 } 12573 12574 struct perf_buffer; 12575 12576 struct perf_buffer_params { 12577 struct perf_event_attr *attr; 12578 /* if event_cb is specified, it takes precendence */ 12579 perf_buffer_event_fn event_cb; 12580 /* sample_cb and lost_cb are higher-level common-case callbacks */ 12581 perf_buffer_sample_fn sample_cb; 12582 perf_buffer_lost_fn lost_cb; 12583 void *ctx; 12584 int cpu_cnt; 12585 int *cpus; 12586 int *map_keys; 12587 }; 12588 12589 struct perf_cpu_buf { 12590 struct perf_buffer *pb; 12591 void *base; /* mmap()'ed memory */ 12592 void *buf; /* for reconstructing segmented data */ 12593 size_t buf_size; 12594 int fd; 12595 int cpu; 12596 int map_key; 12597 }; 12598 12599 struct perf_buffer { 12600 perf_buffer_event_fn event_cb; 12601 perf_buffer_sample_fn sample_cb; 12602 perf_buffer_lost_fn lost_cb; 12603 void *ctx; /* passed into callbacks */ 12604 12605 size_t page_size; 12606 size_t mmap_size; 12607 struct perf_cpu_buf **cpu_bufs; 12608 struct epoll_event *events; 12609 int cpu_cnt; /* number of allocated CPU buffers */ 12610 int epoll_fd; /* perf event FD */ 12611 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 12612 }; 12613 12614 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 12615 struct perf_cpu_buf *cpu_buf) 12616 { 12617 if (!cpu_buf) 12618 return; 12619 if (cpu_buf->base && 12620 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 12621 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 12622 if (cpu_buf->fd >= 0) { 12623 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 12624 close(cpu_buf->fd); 12625 } 12626 free(cpu_buf->buf); 12627 free(cpu_buf); 12628 } 12629 12630 void perf_buffer__free(struct perf_buffer *pb) 12631 { 12632 int i; 12633 12634 if (IS_ERR_OR_NULL(pb)) 12635 return; 12636 if (pb->cpu_bufs) { 12637 for (i = 0; i < pb->cpu_cnt; i++) { 12638 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 12639 12640 if (!cpu_buf) 12641 continue; 12642 12643 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 12644 perf_buffer__free_cpu_buf(pb, cpu_buf); 12645 } 12646 free(pb->cpu_bufs); 12647 } 12648 if (pb->epoll_fd >= 0) 12649 close(pb->epoll_fd); 12650 free(pb->events); 12651 free(pb); 12652 } 12653 12654 static struct perf_cpu_buf * 12655 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 12656 int cpu, int map_key) 12657 { 12658 struct perf_cpu_buf *cpu_buf; 12659 char msg[STRERR_BUFSIZE]; 12660 int err; 12661 12662 cpu_buf = calloc(1, sizeof(*cpu_buf)); 12663 if (!cpu_buf) 12664 return ERR_PTR(-ENOMEM); 12665 12666 cpu_buf->pb = pb; 12667 cpu_buf->cpu = cpu; 12668 cpu_buf->map_key = map_key; 12669 12670 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 12671 -1, PERF_FLAG_FD_CLOEXEC); 12672 if (cpu_buf->fd < 0) { 12673 err = -errno; 12674 pr_warn("failed to open perf buffer event on cpu #%d: %s\n", 12675 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 12676 goto error; 12677 } 12678 12679 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 12680 PROT_READ | PROT_WRITE, MAP_SHARED, 12681 cpu_buf->fd, 0); 12682 if (cpu_buf->base == MAP_FAILED) { 12683 cpu_buf->base = NULL; 12684 err = -errno; 12685 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", 12686 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 12687 goto error; 12688 } 12689 12690 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 12691 err = -errno; 12692 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", 12693 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 12694 goto error; 12695 } 12696 12697 return cpu_buf; 12698 12699 error: 12700 perf_buffer__free_cpu_buf(pb, cpu_buf); 12701 return (struct perf_cpu_buf *)ERR_PTR(err); 12702 } 12703 12704 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 12705 struct perf_buffer_params *p); 12706 12707 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 12708 perf_buffer_sample_fn sample_cb, 12709 perf_buffer_lost_fn lost_cb, 12710 void *ctx, 12711 const struct perf_buffer_opts *opts) 12712 { 12713 const size_t attr_sz = sizeof(struct perf_event_attr); 12714 struct perf_buffer_params p = {}; 12715 struct perf_event_attr attr; 12716 __u32 sample_period; 12717 12718 if (!OPTS_VALID(opts, perf_buffer_opts)) 12719 return libbpf_err_ptr(-EINVAL); 12720 12721 sample_period = OPTS_GET(opts, sample_period, 1); 12722 if (!sample_period) 12723 sample_period = 1; 12724 12725 memset(&attr, 0, attr_sz); 12726 attr.size = attr_sz; 12727 attr.config = PERF_COUNT_SW_BPF_OUTPUT; 12728 attr.type = PERF_TYPE_SOFTWARE; 12729 attr.sample_type = PERF_SAMPLE_RAW; 12730 attr.sample_period = sample_period; 12731 attr.wakeup_events = sample_period; 12732 12733 p.attr = &attr; 12734 p.sample_cb = sample_cb; 12735 p.lost_cb = lost_cb; 12736 p.ctx = ctx; 12737 12738 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 12739 } 12740 12741 struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt, 12742 struct perf_event_attr *attr, 12743 perf_buffer_event_fn event_cb, void *ctx, 12744 const struct perf_buffer_raw_opts *opts) 12745 { 12746 struct perf_buffer_params p = {}; 12747 12748 if (!attr) 12749 return libbpf_err_ptr(-EINVAL); 12750 12751 if (!OPTS_VALID(opts, perf_buffer_raw_opts)) 12752 return libbpf_err_ptr(-EINVAL); 12753 12754 p.attr = attr; 12755 p.event_cb = event_cb; 12756 p.ctx = ctx; 12757 p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0); 12758 p.cpus = OPTS_GET(opts, cpus, NULL); 12759 p.map_keys = OPTS_GET(opts, map_keys, NULL); 12760 12761 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 12762 } 12763 12764 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 12765 struct perf_buffer_params *p) 12766 { 12767 const char *online_cpus_file = "/sys/devices/system/cpu/online"; 12768 struct bpf_map_info map; 12769 char msg[STRERR_BUFSIZE]; 12770 struct perf_buffer *pb; 12771 bool *online = NULL; 12772 __u32 map_info_len; 12773 int err, i, j, n; 12774 12775 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) { 12776 pr_warn("page count should be power of two, but is %zu\n", 12777 page_cnt); 12778 return ERR_PTR(-EINVAL); 12779 } 12780 12781 /* best-effort sanity checks */ 12782 memset(&map, 0, sizeof(map)); 12783 map_info_len = sizeof(map); 12784 err = bpf_map_get_info_by_fd(map_fd, &map, &map_info_len); 12785 if (err) { 12786 err = -errno; 12787 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return 12788 * -EBADFD, -EFAULT, or -E2BIG on real error 12789 */ 12790 if (err != -EINVAL) { 12791 pr_warn("failed to get map info for map FD %d: %s\n", 12792 map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); 12793 return ERR_PTR(err); 12794 } 12795 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n", 12796 map_fd); 12797 } else { 12798 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 12799 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 12800 map.name); 12801 return ERR_PTR(-EINVAL); 12802 } 12803 } 12804 12805 pb = calloc(1, sizeof(*pb)); 12806 if (!pb) 12807 return ERR_PTR(-ENOMEM); 12808 12809 pb->event_cb = p->event_cb; 12810 pb->sample_cb = p->sample_cb; 12811 pb->lost_cb = p->lost_cb; 12812 pb->ctx = p->ctx; 12813 12814 pb->page_size = getpagesize(); 12815 pb->mmap_size = pb->page_size * page_cnt; 12816 pb->map_fd = map_fd; 12817 12818 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 12819 if (pb->epoll_fd < 0) { 12820 err = -errno; 12821 pr_warn("failed to create epoll instance: %s\n", 12822 libbpf_strerror_r(err, msg, sizeof(msg))); 12823 goto error; 12824 } 12825 12826 if (p->cpu_cnt > 0) { 12827 pb->cpu_cnt = p->cpu_cnt; 12828 } else { 12829 pb->cpu_cnt = libbpf_num_possible_cpus(); 12830 if (pb->cpu_cnt < 0) { 12831 err = pb->cpu_cnt; 12832 goto error; 12833 } 12834 if (map.max_entries && map.max_entries < pb->cpu_cnt) 12835 pb->cpu_cnt = map.max_entries; 12836 } 12837 12838 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 12839 if (!pb->events) { 12840 err = -ENOMEM; 12841 pr_warn("failed to allocate events: out of memory\n"); 12842 goto error; 12843 } 12844 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 12845 if (!pb->cpu_bufs) { 12846 err = -ENOMEM; 12847 pr_warn("failed to allocate buffers: out of memory\n"); 12848 goto error; 12849 } 12850 12851 err = parse_cpu_mask_file(online_cpus_file, &online, &n); 12852 if (err) { 12853 pr_warn("failed to get online CPU mask: %d\n", err); 12854 goto error; 12855 } 12856 12857 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { 12858 struct perf_cpu_buf *cpu_buf; 12859 int cpu, map_key; 12860 12861 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 12862 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 12863 12864 /* in case user didn't explicitly requested particular CPUs to 12865 * be attached to, skip offline/not present CPUs 12866 */ 12867 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) 12868 continue; 12869 12870 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 12871 if (IS_ERR(cpu_buf)) { 12872 err = PTR_ERR(cpu_buf); 12873 goto error; 12874 } 12875 12876 pb->cpu_bufs[j] = cpu_buf; 12877 12878 err = bpf_map_update_elem(pb->map_fd, &map_key, 12879 &cpu_buf->fd, 0); 12880 if (err) { 12881 err = -errno; 12882 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 12883 cpu, map_key, cpu_buf->fd, 12884 libbpf_strerror_r(err, msg, sizeof(msg))); 12885 goto error; 12886 } 12887 12888 pb->events[j].events = EPOLLIN; 12889 pb->events[j].data.ptr = cpu_buf; 12890 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 12891 &pb->events[j]) < 0) { 12892 err = -errno; 12893 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 12894 cpu, cpu_buf->fd, 12895 libbpf_strerror_r(err, msg, sizeof(msg))); 12896 goto error; 12897 } 12898 j++; 12899 } 12900 pb->cpu_cnt = j; 12901 free(online); 12902 12903 return pb; 12904 12905 error: 12906 free(online); 12907 if (pb) 12908 perf_buffer__free(pb); 12909 return ERR_PTR(err); 12910 } 12911 12912 struct perf_sample_raw { 12913 struct perf_event_header header; 12914 uint32_t size; 12915 char data[]; 12916 }; 12917 12918 struct perf_sample_lost { 12919 struct perf_event_header header; 12920 uint64_t id; 12921 uint64_t lost; 12922 uint64_t sample_id; 12923 }; 12924 12925 static enum bpf_perf_event_ret 12926 perf_buffer__process_record(struct perf_event_header *e, void *ctx) 12927 { 12928 struct perf_cpu_buf *cpu_buf = ctx; 12929 struct perf_buffer *pb = cpu_buf->pb; 12930 void *data = e; 12931 12932 /* user wants full control over parsing perf event */ 12933 if (pb->event_cb) 12934 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 12935 12936 switch (e->type) { 12937 case PERF_RECORD_SAMPLE: { 12938 struct perf_sample_raw *s = data; 12939 12940 if (pb->sample_cb) 12941 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 12942 break; 12943 } 12944 case PERF_RECORD_LOST: { 12945 struct perf_sample_lost *s = data; 12946 12947 if (pb->lost_cb) 12948 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 12949 break; 12950 } 12951 default: 12952 pr_warn("unknown perf sample type %d\n", e->type); 12953 return LIBBPF_PERF_EVENT_ERROR; 12954 } 12955 return LIBBPF_PERF_EVENT_CONT; 12956 } 12957 12958 static int perf_buffer__process_records(struct perf_buffer *pb, 12959 struct perf_cpu_buf *cpu_buf) 12960 { 12961 enum bpf_perf_event_ret ret; 12962 12963 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size, 12964 pb->page_size, &cpu_buf->buf, 12965 &cpu_buf->buf_size, 12966 perf_buffer__process_record, cpu_buf); 12967 if (ret != LIBBPF_PERF_EVENT_CONT) 12968 return ret; 12969 return 0; 12970 } 12971 12972 int perf_buffer__epoll_fd(const struct perf_buffer *pb) 12973 { 12974 return pb->epoll_fd; 12975 } 12976 12977 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 12978 { 12979 int i, cnt, err; 12980 12981 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 12982 if (cnt < 0) 12983 return -errno; 12984 12985 for (i = 0; i < cnt; i++) { 12986 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 12987 12988 err = perf_buffer__process_records(pb, cpu_buf); 12989 if (err) { 12990 pr_warn("error while processing records: %d\n", err); 12991 return libbpf_err(err); 12992 } 12993 } 12994 return cnt; 12995 } 12996 12997 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer 12998 * manager. 12999 */ 13000 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb) 13001 { 13002 return pb->cpu_cnt; 13003 } 13004 13005 /* 13006 * Return perf_event FD of a ring buffer in *buf_idx* slot of 13007 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using 13008 * select()/poll()/epoll() Linux syscalls. 13009 */ 13010 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx) 13011 { 13012 struct perf_cpu_buf *cpu_buf; 13013 13014 if (buf_idx >= pb->cpu_cnt) 13015 return libbpf_err(-EINVAL); 13016 13017 cpu_buf = pb->cpu_bufs[buf_idx]; 13018 if (!cpu_buf) 13019 return libbpf_err(-ENOENT); 13020 13021 return cpu_buf->fd; 13022 } 13023 13024 int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size) 13025 { 13026 struct perf_cpu_buf *cpu_buf; 13027 13028 if (buf_idx >= pb->cpu_cnt) 13029 return libbpf_err(-EINVAL); 13030 13031 cpu_buf = pb->cpu_bufs[buf_idx]; 13032 if (!cpu_buf) 13033 return libbpf_err(-ENOENT); 13034 13035 *buf = cpu_buf->base; 13036 *buf_size = pb->mmap_size; 13037 return 0; 13038 } 13039 13040 /* 13041 * Consume data from perf ring buffer corresponding to slot *buf_idx* in 13042 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to 13043 * consume, do nothing and return success. 13044 * Returns: 13045 * - 0 on success; 13046 * - <0 on failure. 13047 */ 13048 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx) 13049 { 13050 struct perf_cpu_buf *cpu_buf; 13051 13052 if (buf_idx >= pb->cpu_cnt) 13053 return libbpf_err(-EINVAL); 13054 13055 cpu_buf = pb->cpu_bufs[buf_idx]; 13056 if (!cpu_buf) 13057 return libbpf_err(-ENOENT); 13058 13059 return perf_buffer__process_records(pb, cpu_buf); 13060 } 13061 13062 int perf_buffer__consume(struct perf_buffer *pb) 13063 { 13064 int i, err; 13065 13066 for (i = 0; i < pb->cpu_cnt; i++) { 13067 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 13068 13069 if (!cpu_buf) 13070 continue; 13071 13072 err = perf_buffer__process_records(pb, cpu_buf); 13073 if (err) { 13074 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err); 13075 return libbpf_err(err); 13076 } 13077 } 13078 return 0; 13079 } 13080 13081 int bpf_program__set_attach_target(struct bpf_program *prog, 13082 int attach_prog_fd, 13083 const char *attach_func_name) 13084 { 13085 int btf_obj_fd = 0, btf_id = 0, err; 13086 13087 if (!prog || attach_prog_fd < 0) 13088 return libbpf_err(-EINVAL); 13089 13090 if (prog->obj->loaded) 13091 return libbpf_err(-EINVAL); 13092 13093 if (attach_prog_fd && !attach_func_name) { 13094 /* remember attach_prog_fd and let bpf_program__load() find 13095 * BTF ID during the program load 13096 */ 13097 prog->attach_prog_fd = attach_prog_fd; 13098 return 0; 13099 } 13100 13101 if (attach_prog_fd) { 13102 btf_id = libbpf_find_prog_btf_id(attach_func_name, 13103 attach_prog_fd); 13104 if (btf_id < 0) 13105 return libbpf_err(btf_id); 13106 } else { 13107 if (!attach_func_name) 13108 return libbpf_err(-EINVAL); 13109 13110 /* load btf_vmlinux, if not yet */ 13111 err = bpf_object__load_vmlinux_btf(prog->obj, true); 13112 if (err) 13113 return libbpf_err(err); 13114 err = find_kernel_btf_id(prog->obj, attach_func_name, 13115 prog->expected_attach_type, 13116 &btf_obj_fd, &btf_id); 13117 if (err) 13118 return libbpf_err(err); 13119 } 13120 13121 prog->attach_btf_id = btf_id; 13122 prog->attach_btf_obj_fd = btf_obj_fd; 13123 prog->attach_prog_fd = attach_prog_fd; 13124 return 0; 13125 } 13126 13127 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) 13128 { 13129 int err = 0, n, len, start, end = -1; 13130 bool *tmp; 13131 13132 *mask = NULL; 13133 *mask_sz = 0; 13134 13135 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 13136 while (*s) { 13137 if (*s == ',' || *s == '\n') { 13138 s++; 13139 continue; 13140 } 13141 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); 13142 if (n <= 0 || n > 2) { 13143 pr_warn("Failed to get CPU range %s: %d\n", s, n); 13144 err = -EINVAL; 13145 goto cleanup; 13146 } else if (n == 1) { 13147 end = start; 13148 } 13149 if (start < 0 || start > end) { 13150 pr_warn("Invalid CPU range [%d,%d] in %s\n", 13151 start, end, s); 13152 err = -EINVAL; 13153 goto cleanup; 13154 } 13155 tmp = realloc(*mask, end + 1); 13156 if (!tmp) { 13157 err = -ENOMEM; 13158 goto cleanup; 13159 } 13160 *mask = tmp; 13161 memset(tmp + *mask_sz, 0, start - *mask_sz); 13162 memset(tmp + start, 1, end - start + 1); 13163 *mask_sz = end + 1; 13164 s += len; 13165 } 13166 if (!*mask_sz) { 13167 pr_warn("Empty CPU range\n"); 13168 return -EINVAL; 13169 } 13170 return 0; 13171 cleanup: 13172 free(*mask); 13173 *mask = NULL; 13174 return err; 13175 } 13176 13177 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) 13178 { 13179 int fd, err = 0, len; 13180 char buf[128]; 13181 13182 fd = open(fcpu, O_RDONLY | O_CLOEXEC); 13183 if (fd < 0) { 13184 err = -errno; 13185 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); 13186 return err; 13187 } 13188 len = read(fd, buf, sizeof(buf)); 13189 close(fd); 13190 if (len <= 0) { 13191 err = len ? -errno : -EINVAL; 13192 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); 13193 return err; 13194 } 13195 if (len >= sizeof(buf)) { 13196 pr_warn("CPU mask is too big in file %s\n", fcpu); 13197 return -E2BIG; 13198 } 13199 buf[len] = '\0'; 13200 13201 return parse_cpu_mask_str(buf, mask, mask_sz); 13202 } 13203 13204 int libbpf_num_possible_cpus(void) 13205 { 13206 static const char *fcpu = "/sys/devices/system/cpu/possible"; 13207 static int cpus; 13208 int err, n, i, tmp_cpus; 13209 bool *mask; 13210 13211 tmp_cpus = READ_ONCE(cpus); 13212 if (tmp_cpus > 0) 13213 return tmp_cpus; 13214 13215 err = parse_cpu_mask_file(fcpu, &mask, &n); 13216 if (err) 13217 return libbpf_err(err); 13218 13219 tmp_cpus = 0; 13220 for (i = 0; i < n; i++) { 13221 if (mask[i]) 13222 tmp_cpus++; 13223 } 13224 free(mask); 13225 13226 WRITE_ONCE(cpus, tmp_cpus); 13227 return tmp_cpus; 13228 } 13229 13230 static int populate_skeleton_maps(const struct bpf_object *obj, 13231 struct bpf_map_skeleton *maps, 13232 size_t map_cnt) 13233 { 13234 int i; 13235 13236 for (i = 0; i < map_cnt; i++) { 13237 struct bpf_map **map = maps[i].map; 13238 const char *name = maps[i].name; 13239 void **mmaped = maps[i].mmaped; 13240 13241 *map = bpf_object__find_map_by_name(obj, name); 13242 if (!*map) { 13243 pr_warn("failed to find skeleton map '%s'\n", name); 13244 return -ESRCH; 13245 } 13246 13247 /* externs shouldn't be pre-setup from user code */ 13248 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) 13249 *mmaped = (*map)->mmaped; 13250 } 13251 return 0; 13252 } 13253 13254 static int populate_skeleton_progs(const struct bpf_object *obj, 13255 struct bpf_prog_skeleton *progs, 13256 size_t prog_cnt) 13257 { 13258 int i; 13259 13260 for (i = 0; i < prog_cnt; i++) { 13261 struct bpf_program **prog = progs[i].prog; 13262 const char *name = progs[i].name; 13263 13264 *prog = bpf_object__find_program_by_name(obj, name); 13265 if (!*prog) { 13266 pr_warn("failed to find skeleton program '%s'\n", name); 13267 return -ESRCH; 13268 } 13269 } 13270 return 0; 13271 } 13272 13273 int bpf_object__open_skeleton(struct bpf_object_skeleton *s, 13274 const struct bpf_object_open_opts *opts) 13275 { 13276 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts, 13277 .object_name = s->name, 13278 ); 13279 struct bpf_object *obj; 13280 int err; 13281 13282 /* Attempt to preserve opts->object_name, unless overriden by user 13283 * explicitly. Overwriting object name for skeletons is discouraged, 13284 * as it breaks global data maps, because they contain object name 13285 * prefix as their own map name prefix. When skeleton is generated, 13286 * bpftool is making an assumption that this name will stay the same. 13287 */ 13288 if (opts) { 13289 memcpy(&skel_opts, opts, sizeof(*opts)); 13290 if (!opts->object_name) 13291 skel_opts.object_name = s->name; 13292 } 13293 13294 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); 13295 err = libbpf_get_error(obj); 13296 if (err) { 13297 pr_warn("failed to initialize skeleton BPF object '%s': %d\n", 13298 s->name, err); 13299 return libbpf_err(err); 13300 } 13301 13302 *s->obj = obj; 13303 err = populate_skeleton_maps(obj, s->maps, s->map_cnt); 13304 if (err) { 13305 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err); 13306 return libbpf_err(err); 13307 } 13308 13309 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt); 13310 if (err) { 13311 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err); 13312 return libbpf_err(err); 13313 } 13314 13315 return 0; 13316 } 13317 13318 int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s) 13319 { 13320 int err, len, var_idx, i; 13321 const char *var_name; 13322 const struct bpf_map *map; 13323 struct btf *btf; 13324 __u32 map_type_id; 13325 const struct btf_type *map_type, *var_type; 13326 const struct bpf_var_skeleton *var_skel; 13327 struct btf_var_secinfo *var; 13328 13329 if (!s->obj) 13330 return libbpf_err(-EINVAL); 13331 13332 btf = bpf_object__btf(s->obj); 13333 if (!btf) { 13334 pr_warn("subskeletons require BTF at runtime (object %s)\n", 13335 bpf_object__name(s->obj)); 13336 return libbpf_err(-errno); 13337 } 13338 13339 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt); 13340 if (err) { 13341 pr_warn("failed to populate subskeleton maps: %d\n", err); 13342 return libbpf_err(err); 13343 } 13344 13345 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt); 13346 if (err) { 13347 pr_warn("failed to populate subskeleton maps: %d\n", err); 13348 return libbpf_err(err); 13349 } 13350 13351 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) { 13352 var_skel = &s->vars[var_idx]; 13353 map = *var_skel->map; 13354 map_type_id = bpf_map__btf_value_type_id(map); 13355 map_type = btf__type_by_id(btf, map_type_id); 13356 13357 if (!btf_is_datasec(map_type)) { 13358 pr_warn("type for map '%1$s' is not a datasec: %2$s", 13359 bpf_map__name(map), 13360 __btf_kind_str(btf_kind(map_type))); 13361 return libbpf_err(-EINVAL); 13362 } 13363 13364 len = btf_vlen(map_type); 13365 var = btf_var_secinfos(map_type); 13366 for (i = 0; i < len; i++, var++) { 13367 var_type = btf__type_by_id(btf, var->type); 13368 var_name = btf__name_by_offset(btf, var_type->name_off); 13369 if (strcmp(var_name, var_skel->name) == 0) { 13370 *var_skel->addr = map->mmaped + var->offset; 13371 break; 13372 } 13373 } 13374 } 13375 return 0; 13376 } 13377 13378 void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s) 13379 { 13380 if (!s) 13381 return; 13382 free(s->maps); 13383 free(s->progs); 13384 free(s->vars); 13385 free(s); 13386 } 13387 13388 int bpf_object__load_skeleton(struct bpf_object_skeleton *s) 13389 { 13390 int i, err; 13391 13392 err = bpf_object__load(*s->obj); 13393 if (err) { 13394 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); 13395 return libbpf_err(err); 13396 } 13397 13398 for (i = 0; i < s->map_cnt; i++) { 13399 struct bpf_map *map = *s->maps[i].map; 13400 size_t mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 13401 int prot, map_fd = map->fd; 13402 void **mmaped = s->maps[i].mmaped; 13403 13404 if (!mmaped) 13405 continue; 13406 13407 if (!(map->def.map_flags & BPF_F_MMAPABLE)) { 13408 *mmaped = NULL; 13409 continue; 13410 } 13411 13412 if (map->def.map_flags & BPF_F_RDONLY_PROG) 13413 prot = PROT_READ; 13414 else 13415 prot = PROT_READ | PROT_WRITE; 13416 13417 /* Remap anonymous mmap()-ed "map initialization image" as 13418 * a BPF map-backed mmap()-ed memory, but preserving the same 13419 * memory address. This will cause kernel to change process' 13420 * page table to point to a different piece of kernel memory, 13421 * but from userspace point of view memory address (and its 13422 * contents, being identical at this point) will stay the 13423 * same. This mapping will be released by bpf_object__close() 13424 * as per normal clean up procedure, so we don't need to worry 13425 * about it from skeleton's clean up perspective. 13426 */ 13427 *mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0); 13428 if (*mmaped == MAP_FAILED) { 13429 err = -errno; 13430 *mmaped = NULL; 13431 pr_warn("failed to re-mmap() map '%s': %d\n", 13432 bpf_map__name(map), err); 13433 return libbpf_err(err); 13434 } 13435 } 13436 13437 return 0; 13438 } 13439 13440 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) 13441 { 13442 int i, err; 13443 13444 for (i = 0; i < s->prog_cnt; i++) { 13445 struct bpf_program *prog = *s->progs[i].prog; 13446 struct bpf_link **link = s->progs[i].link; 13447 13448 if (!prog->autoload || !prog->autoattach) 13449 continue; 13450 13451 /* auto-attaching not supported for this program */ 13452 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) 13453 continue; 13454 13455 /* if user already set the link manually, don't attempt auto-attach */ 13456 if (*link) 13457 continue; 13458 13459 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link); 13460 if (err) { 13461 pr_warn("prog '%s': failed to auto-attach: %d\n", 13462 bpf_program__name(prog), err); 13463 return libbpf_err(err); 13464 } 13465 13466 /* It's possible that for some SEC() definitions auto-attach 13467 * is supported in some cases (e.g., if definition completely 13468 * specifies target information), but is not in other cases. 13469 * SEC("uprobe") is one such case. If user specified target 13470 * binary and function name, such BPF program can be 13471 * auto-attached. But if not, it shouldn't trigger skeleton's 13472 * attach to fail. It should just be skipped. 13473 * attach_fn signals such case with returning 0 (no error) and 13474 * setting link to NULL. 13475 */ 13476 } 13477 13478 return 0; 13479 } 13480 13481 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) 13482 { 13483 int i; 13484 13485 for (i = 0; i < s->prog_cnt; i++) { 13486 struct bpf_link **link = s->progs[i].link; 13487 13488 bpf_link__destroy(*link); 13489 *link = NULL; 13490 } 13491 } 13492 13493 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) 13494 { 13495 if (!s) 13496 return; 13497 13498 if (s->progs) 13499 bpf_object__detach_skeleton(s); 13500 if (s->obj) 13501 bpf_object__close(*s->obj); 13502 free(s->maps); 13503 free(s->progs); 13504 free(s); 13505 } 13506