1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 /* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13 #ifndef _GNU_SOURCE 14 #define _GNU_SOURCE 15 #endif 16 #include <stdlib.h> 17 #include <stdio.h> 18 #include <stdarg.h> 19 #include <libgen.h> 20 #include <inttypes.h> 21 #include <limits.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <endian.h> 25 #include <fcntl.h> 26 #include <errno.h> 27 #include <ctype.h> 28 #include <asm/unistd.h> 29 #include <linux/err.h> 30 #include <linux/kernel.h> 31 #include <linux/bpf.h> 32 #include <linux/btf.h> 33 #include <linux/filter.h> 34 #include <linux/limits.h> 35 #include <linux/perf_event.h> 36 #include <linux/bpf_perf_event.h> 37 #include <linux/ring_buffer.h> 38 #include <sys/epoll.h> 39 #include <sys/ioctl.h> 40 #include <sys/mman.h> 41 #include <sys/stat.h> 42 #include <sys/types.h> 43 #include <sys/vfs.h> 44 #include <sys/utsname.h> 45 #include <sys/resource.h> 46 #include <libelf.h> 47 #include <gelf.h> 48 #include <zlib.h> 49 50 #include "libbpf.h" 51 #include "bpf.h" 52 #include "btf.h" 53 #include "libbpf_internal.h" 54 #include "hashmap.h" 55 #include "bpf_gen_internal.h" 56 #include "zip.h" 57 58 #ifndef BPF_FS_MAGIC 59 #define BPF_FS_MAGIC 0xcafe4a11 60 #endif 61 62 #define MAX_EVENT_NAME_LEN 64 63 64 #define BPF_FS_DEFAULT_PATH "/sys/fs/bpf" 65 66 #define BPF_INSN_SZ (sizeof(struct bpf_insn)) 67 68 /* vsprintf() in __base_pr() uses nonliteral format string. It may break 69 * compilation if user enables corresponding warning. Disable it explicitly. 70 */ 71 #pragma GCC diagnostic ignored "-Wformat-nonliteral" 72 73 #define __printf(a, b) __attribute__((format(printf, a, b))) 74 75 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); 76 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog); 77 static int map_set_def_max_entries(struct bpf_map *map); 78 79 static const char * const attach_type_name[] = { 80 [BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress", 81 [BPF_CGROUP_INET_EGRESS] = "cgroup_inet_egress", 82 [BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create", 83 [BPF_CGROUP_INET_SOCK_RELEASE] = "cgroup_inet_sock_release", 84 [BPF_CGROUP_SOCK_OPS] = "cgroup_sock_ops", 85 [BPF_CGROUP_DEVICE] = "cgroup_device", 86 [BPF_CGROUP_INET4_BIND] = "cgroup_inet4_bind", 87 [BPF_CGROUP_INET6_BIND] = "cgroup_inet6_bind", 88 [BPF_CGROUP_INET4_CONNECT] = "cgroup_inet4_connect", 89 [BPF_CGROUP_INET6_CONNECT] = "cgroup_inet6_connect", 90 [BPF_CGROUP_UNIX_CONNECT] = "cgroup_unix_connect", 91 [BPF_CGROUP_INET4_POST_BIND] = "cgroup_inet4_post_bind", 92 [BPF_CGROUP_INET6_POST_BIND] = "cgroup_inet6_post_bind", 93 [BPF_CGROUP_INET4_GETPEERNAME] = "cgroup_inet4_getpeername", 94 [BPF_CGROUP_INET6_GETPEERNAME] = "cgroup_inet6_getpeername", 95 [BPF_CGROUP_UNIX_GETPEERNAME] = "cgroup_unix_getpeername", 96 [BPF_CGROUP_INET4_GETSOCKNAME] = "cgroup_inet4_getsockname", 97 [BPF_CGROUP_INET6_GETSOCKNAME] = "cgroup_inet6_getsockname", 98 [BPF_CGROUP_UNIX_GETSOCKNAME] = "cgroup_unix_getsockname", 99 [BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg", 100 [BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg", 101 [BPF_CGROUP_UNIX_SENDMSG] = "cgroup_unix_sendmsg", 102 [BPF_CGROUP_SYSCTL] = "cgroup_sysctl", 103 [BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg", 104 [BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg", 105 [BPF_CGROUP_UNIX_RECVMSG] = "cgroup_unix_recvmsg", 106 [BPF_CGROUP_GETSOCKOPT] = "cgroup_getsockopt", 107 [BPF_CGROUP_SETSOCKOPT] = "cgroup_setsockopt", 108 [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser", 109 [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict", 110 [BPF_SK_SKB_VERDICT] = "sk_skb_verdict", 111 [BPF_SK_MSG_VERDICT] = "sk_msg_verdict", 112 [BPF_LIRC_MODE2] = "lirc_mode2", 113 [BPF_FLOW_DISSECTOR] = "flow_dissector", 114 [BPF_TRACE_RAW_TP] = "trace_raw_tp", 115 [BPF_TRACE_FENTRY] = "trace_fentry", 116 [BPF_TRACE_FEXIT] = "trace_fexit", 117 [BPF_MODIFY_RETURN] = "modify_return", 118 [BPF_TRACE_FSESSION] = "trace_fsession", 119 [BPF_LSM_MAC] = "lsm_mac", 120 [BPF_LSM_CGROUP] = "lsm_cgroup", 121 [BPF_SK_LOOKUP] = "sk_lookup", 122 [BPF_TRACE_ITER] = "trace_iter", 123 [BPF_XDP_DEVMAP] = "xdp_devmap", 124 [BPF_XDP_CPUMAP] = "xdp_cpumap", 125 [BPF_XDP] = "xdp", 126 [BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select", 127 [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_reuseport_select_or_migrate", 128 [BPF_PERF_EVENT] = "perf_event", 129 [BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi", 130 [BPF_STRUCT_OPS] = "struct_ops", 131 [BPF_NETFILTER] = "netfilter", 132 [BPF_TCX_INGRESS] = "tcx_ingress", 133 [BPF_TCX_EGRESS] = "tcx_egress", 134 [BPF_TRACE_UPROBE_MULTI] = "trace_uprobe_multi", 135 [BPF_NETKIT_PRIMARY] = "netkit_primary", 136 [BPF_NETKIT_PEER] = "netkit_peer", 137 [BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session", 138 [BPF_TRACE_UPROBE_SESSION] = "trace_uprobe_session", 139 [BPF_TRACE_FENTRY_MULTI] = "trace_fentry_multi", 140 [BPF_TRACE_FEXIT_MULTI] = "trace_fexit_multi", 141 [BPF_TRACE_FSESSION_MULTI] = "trace_fsession_multi", 142 }; 143 144 static const char * const link_type_name[] = { 145 [BPF_LINK_TYPE_UNSPEC] = "unspec", 146 [BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", 147 [BPF_LINK_TYPE_TRACING] = "tracing", 148 [BPF_LINK_TYPE_CGROUP] = "cgroup", 149 [BPF_LINK_TYPE_ITER] = "iter", 150 [BPF_LINK_TYPE_NETNS] = "netns", 151 [BPF_LINK_TYPE_XDP] = "xdp", 152 [BPF_LINK_TYPE_PERF_EVENT] = "perf_event", 153 [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi", 154 [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops", 155 [BPF_LINK_TYPE_NETFILTER] = "netfilter", 156 [BPF_LINK_TYPE_TCX] = "tcx", 157 [BPF_LINK_TYPE_UPROBE_MULTI] = "uprobe_multi", 158 [BPF_LINK_TYPE_NETKIT] = "netkit", 159 [BPF_LINK_TYPE_SOCKMAP] = "sockmap", 160 [BPF_LINK_TYPE_TRACING_MULTI] = "tracing_multi", 161 }; 162 163 static const char * const map_type_name[] = { 164 [BPF_MAP_TYPE_UNSPEC] = "unspec", 165 [BPF_MAP_TYPE_HASH] = "hash", 166 [BPF_MAP_TYPE_ARRAY] = "array", 167 [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array", 168 [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array", 169 [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash", 170 [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array", 171 [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace", 172 [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array", 173 [BPF_MAP_TYPE_LRU_HASH] = "lru_hash", 174 [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash", 175 [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie", 176 [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps", 177 [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps", 178 [BPF_MAP_TYPE_DEVMAP] = "devmap", 179 [BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash", 180 [BPF_MAP_TYPE_SOCKMAP] = "sockmap", 181 [BPF_MAP_TYPE_CPUMAP] = "cpumap", 182 [BPF_MAP_TYPE_XSKMAP] = "xskmap", 183 [BPF_MAP_TYPE_SOCKHASH] = "sockhash", 184 [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage", 185 [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray", 186 [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage", 187 [BPF_MAP_TYPE_QUEUE] = "queue", 188 [BPF_MAP_TYPE_STACK] = "stack", 189 [BPF_MAP_TYPE_SK_STORAGE] = "sk_storage", 190 [BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops", 191 [BPF_MAP_TYPE_RINGBUF] = "ringbuf", 192 [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage", 193 [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage", 194 [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter", 195 [BPF_MAP_TYPE_USER_RINGBUF] = "user_ringbuf", 196 [BPF_MAP_TYPE_CGRP_STORAGE] = "cgrp_storage", 197 [BPF_MAP_TYPE_ARENA] = "arena", 198 [BPF_MAP_TYPE_INSN_ARRAY] = "insn_array", 199 [BPF_MAP_TYPE_RHASH] = "rhash", 200 }; 201 202 static const char * const prog_type_name[] = { 203 [BPF_PROG_TYPE_UNSPEC] = "unspec", 204 [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", 205 [BPF_PROG_TYPE_KPROBE] = "kprobe", 206 [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls", 207 [BPF_PROG_TYPE_SCHED_ACT] = "sched_act", 208 [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint", 209 [BPF_PROG_TYPE_XDP] = "xdp", 210 [BPF_PROG_TYPE_PERF_EVENT] = "perf_event", 211 [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb", 212 [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock", 213 [BPF_PROG_TYPE_LWT_IN] = "lwt_in", 214 [BPF_PROG_TYPE_LWT_OUT] = "lwt_out", 215 [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", 216 [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", 217 [BPF_PROG_TYPE_SK_SKB] = "sk_skb", 218 [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device", 219 [BPF_PROG_TYPE_SK_MSG] = "sk_msg", 220 [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", 221 [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr", 222 [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local", 223 [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2", 224 [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport", 225 [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector", 226 [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl", 227 [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable", 228 [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt", 229 [BPF_PROG_TYPE_TRACING] = "tracing", 230 [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops", 231 [BPF_PROG_TYPE_EXT] = "ext", 232 [BPF_PROG_TYPE_LSM] = "lsm", 233 [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup", 234 [BPF_PROG_TYPE_SYSCALL] = "syscall", 235 [BPF_PROG_TYPE_NETFILTER] = "netfilter", 236 }; 237 238 static int __base_pr(enum libbpf_print_level level, const char *format, 239 va_list args) 240 { 241 const char *env_var = "LIBBPF_LOG_LEVEL"; 242 static enum libbpf_print_level min_level = LIBBPF_INFO; 243 static bool initialized; 244 245 if (!initialized) { 246 char *verbosity; 247 248 initialized = true; 249 verbosity = getenv(env_var); 250 if (verbosity) { 251 if (strcasecmp(verbosity, "warn") == 0) 252 min_level = LIBBPF_WARN; 253 else if (strcasecmp(verbosity, "debug") == 0) 254 min_level = LIBBPF_DEBUG; 255 else if (strcasecmp(verbosity, "info") == 0) 256 min_level = LIBBPF_INFO; 257 else 258 fprintf(stderr, "libbpf: unrecognized '%s' envvar value: '%s', should be one of 'warn', 'debug', or 'info'.\n", 259 env_var, verbosity); 260 } 261 } 262 263 /* if too verbose, skip logging */ 264 if (level > min_level) 265 return 0; 266 267 return vfprintf(stderr, format, args); 268 } 269 270 static libbpf_print_fn_t __libbpf_pr = __base_pr; 271 272 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) 273 { 274 libbpf_print_fn_t old_print_fn; 275 276 old_print_fn = __atomic_exchange_n(&__libbpf_pr, fn, __ATOMIC_RELAXED); 277 278 return old_print_fn; 279 } 280 281 __printf(2, 3) 282 void libbpf_print(enum libbpf_print_level level, const char *format, ...) 283 { 284 va_list args; 285 int old_errno; 286 libbpf_print_fn_t print_fn; 287 288 print_fn = __atomic_load_n(&__libbpf_pr, __ATOMIC_RELAXED); 289 if (!print_fn) 290 return; 291 292 old_errno = errno; 293 294 va_start(args, format); 295 print_fn(level, format, args); 296 va_end(args); 297 298 errno = old_errno; 299 } 300 301 static void pr_perm_msg(int err) 302 { 303 struct rlimit limit; 304 char buf[100]; 305 306 if (err != -EPERM || geteuid() != 0) 307 return; 308 309 err = getrlimit(RLIMIT_MEMLOCK, &limit); 310 if (err) 311 return; 312 313 if (limit.rlim_cur == RLIM_INFINITY) 314 return; 315 316 if (limit.rlim_cur < 1024) 317 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); 318 else if (limit.rlim_cur < 1024*1024) 319 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); 320 else 321 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); 322 323 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", 324 buf); 325 } 326 327 /* Copied from tools/perf/util/util.h */ 328 #ifndef zfree 329 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 330 #endif 331 332 #ifndef zclose 333 # define zclose(fd) ({ \ 334 int ___err = 0; \ 335 if ((fd) >= 0) \ 336 ___err = close((fd)); \ 337 fd = -1; \ 338 ___err; }) 339 #endif 340 341 static inline __u64 ptr_to_u64(const void *ptr) 342 { 343 return (__u64) (unsigned long) ptr; 344 } 345 346 int libbpf_set_strict_mode(enum libbpf_strict_mode mode) 347 { 348 /* as of v1.0 libbpf_set_strict_mode() is a no-op */ 349 return 0; 350 } 351 352 __u32 libbpf_major_version(void) 353 { 354 return LIBBPF_MAJOR_VERSION; 355 } 356 357 __u32 libbpf_minor_version(void) 358 { 359 return LIBBPF_MINOR_VERSION; 360 } 361 362 const char *libbpf_version_string(void) 363 { 364 #define __S(X) #X 365 #define _S(X) __S(X) 366 return "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION); 367 #undef _S 368 #undef __S 369 } 370 371 enum reloc_type { 372 RELO_LD64, 373 RELO_CALL, 374 RELO_DATA, 375 RELO_EXTERN_LD64, 376 RELO_EXTERN_CALL, 377 RELO_SUBPROG_ADDR, 378 RELO_CORE, 379 RELO_INSN_ARRAY, 380 }; 381 382 struct reloc_desc { 383 enum reloc_type type; 384 int insn_idx; 385 union { 386 const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */ 387 struct { 388 int map_idx; 389 unsigned int sym_off; 390 /* 391 * The following two fields can be unionized, as the 392 * ext_idx field is used for extern symbols, and the 393 * sym_size is used for jump tables, which are never 394 * extern 395 */ 396 union { 397 int ext_idx; 398 int sym_size; 399 }; 400 }; 401 }; 402 }; 403 404 /* stored as sec_def->cookie for all libbpf-supported SEC()s */ 405 enum sec_def_flags { 406 SEC_NONE = 0, 407 /* expected_attach_type is optional, if kernel doesn't support that */ 408 SEC_EXP_ATTACH_OPT = 1, 409 /* legacy, only used by libbpf_get_type_names() and 410 * libbpf_attach_type_by_name(), not used by libbpf itself at all. 411 * This used to be associated with cgroup (and few other) BPF programs 412 * that were attachable through BPF_PROG_ATTACH command. Pretty 413 * meaningless nowadays, though. 414 */ 415 SEC_ATTACHABLE = 2, 416 SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT, 417 /* attachment target is specified through BTF ID in either kernel or 418 * other BPF program's BTF object 419 */ 420 SEC_ATTACH_BTF = 4, 421 /* BPF program type allows sleeping/blocking in kernel */ 422 SEC_SLEEPABLE = 8, 423 /* BPF program support non-linear XDP buffer */ 424 SEC_XDP_FRAGS = 16, 425 /* Setup proper attach type for usdt probes. */ 426 SEC_USDT = 32, 427 }; 428 429 struct bpf_sec_def { 430 char *sec; 431 enum bpf_prog_type prog_type; 432 enum bpf_attach_type expected_attach_type; 433 long cookie; 434 int handler_id; 435 436 libbpf_prog_setup_fn_t prog_setup_fn; 437 libbpf_prog_prepare_load_fn_t prog_prepare_load_fn; 438 libbpf_prog_attach_fn_t prog_attach_fn; 439 }; 440 441 struct bpf_light_subprog { 442 __u32 sec_insn_off; 443 __u32 sub_insn_off; 444 }; 445 446 /* 447 * bpf_prog should be a better name but it has been used in 448 * linux/filter.h. 449 */ 450 struct bpf_program { 451 char *name; 452 char *sec_name; 453 size_t sec_idx; 454 const struct bpf_sec_def *sec_def; 455 /* this program's instruction offset (in number of instructions) 456 * within its containing ELF section 457 */ 458 size_t sec_insn_off; 459 /* number of original instructions in ELF section belonging to this 460 * program, not taking into account subprogram instructions possible 461 * appended later during relocation 462 */ 463 size_t sec_insn_cnt; 464 /* Offset (in number of instructions) of the start of instruction 465 * belonging to this BPF program within its containing main BPF 466 * program. For the entry-point (main) BPF program, this is always 467 * zero. For a sub-program, this gets reset before each of main BPF 468 * programs are processed and relocated and is used to determined 469 * whether sub-program was already appended to the main program, and 470 * if yes, at which instruction offset. 471 */ 472 size_t sub_insn_off; 473 474 /* instructions that belong to BPF program; insns[0] is located at 475 * sec_insn_off instruction within its ELF section in ELF file, so 476 * when mapping ELF file instruction index to the local instruction, 477 * one needs to subtract sec_insn_off; and vice versa. 478 */ 479 struct bpf_insn *insns; 480 /* actual number of instruction in this BPF program's image; for 481 * entry-point BPF programs this includes the size of main program 482 * itself plus all the used sub-programs, appended at the end 483 */ 484 size_t insns_cnt; 485 486 struct reloc_desc *reloc_desc; 487 int nr_reloc; 488 489 /* BPF verifier log settings */ 490 char *log_buf; 491 size_t log_size; 492 __u32 log_level; 493 494 struct bpf_object *obj; 495 496 int fd; 497 bool autoload; 498 bool autoattach; 499 bool sym_global; 500 bool mark_btf_static; 501 enum bpf_prog_type type; 502 enum bpf_attach_type expected_attach_type; 503 int exception_cb_idx; 504 505 int prog_ifindex; 506 __u32 attach_btf_obj_fd; 507 __u32 attach_btf_id; 508 __u32 attach_prog_fd; 509 510 void *func_info; 511 __u32 func_info_rec_size; 512 __u32 func_info_cnt; 513 514 void *line_info; 515 __u32 line_info_rec_size; 516 __u32 line_info_cnt; 517 __u32 prog_flags; 518 __u8 hash[SHA256_DIGEST_LENGTH]; 519 520 struct bpf_light_subprog *subprogs; 521 __u32 subprog_cnt; 522 }; 523 524 struct bpf_struct_ops { 525 struct bpf_program **progs; 526 __u32 *kern_func_off; 527 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ 528 void *data; 529 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in 530 * btf_vmlinux's format. 531 * struct bpf_struct_ops_tcp_congestion_ops { 532 * [... some other kernel fields ...] 533 * struct tcp_congestion_ops data; 534 * } 535 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) 536 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" 537 * from "data". 538 */ 539 void *kern_vdata; 540 __u32 type_id; 541 }; 542 543 #define DATA_SEC ".data" 544 #define PERCPU_SEC ".percpu" 545 #define BSS_SEC ".bss" 546 #define RODATA_SEC ".rodata" 547 #define KCONFIG_SEC ".kconfig" 548 #define KSYMS_SEC ".ksyms" 549 #define STRUCT_OPS_SEC ".struct_ops" 550 #define STRUCT_OPS_LINK_SEC ".struct_ops.link" 551 #define ARENA_SEC ".addr_space.1" 552 553 enum libbpf_map_type { 554 LIBBPF_MAP_UNSPEC, 555 LIBBPF_MAP_DATA, 556 LIBBPF_MAP_BSS, 557 LIBBPF_MAP_RODATA, 558 LIBBPF_MAP_KCONFIG, 559 LIBBPF_MAP_PERCPU, 560 }; 561 562 struct bpf_map_def { 563 unsigned int type; 564 unsigned int key_size; 565 unsigned int value_size; 566 unsigned int max_entries; 567 unsigned int map_flags; 568 }; 569 570 struct bpf_map { 571 struct bpf_object *obj; 572 char *name; 573 /* real_name is defined for special internal maps (.rodata*, 574 * .data*, .bss, .kconfig) and preserves their original ELF section 575 * name. This is important to be able to find corresponding BTF 576 * DATASEC information. 577 */ 578 char *real_name; 579 int fd; 580 int sec_idx; 581 size_t sec_offset; 582 int map_ifindex; 583 int inner_map_fd; 584 struct bpf_map_def def; 585 __u32 numa_node; 586 __u32 btf_var_idx; 587 int mod_btf_fd; 588 __u32 btf_key_type_id; 589 __u32 btf_value_type_id; 590 __u32 btf_vmlinux_value_type_id; 591 enum libbpf_map_type libbpf_type; 592 void *mmaped; 593 struct bpf_struct_ops *st_ops; 594 struct bpf_map *inner_map; 595 void **init_slots; 596 int init_slots_sz; 597 char *pin_path; 598 bool pinned; 599 bool reused; 600 bool autocreate; 601 bool autoattach; 602 __u64 map_extra; 603 struct bpf_program *excl_prog; 604 }; 605 606 enum extern_type { 607 EXT_UNKNOWN, 608 EXT_KCFG, 609 EXT_KSYM, 610 }; 611 612 enum kcfg_type { 613 KCFG_UNKNOWN, 614 KCFG_CHAR, 615 KCFG_BOOL, 616 KCFG_INT, 617 KCFG_TRISTATE, 618 KCFG_CHAR_ARR, 619 }; 620 621 struct extern_desc { 622 enum extern_type type; 623 int sym_idx; 624 int btf_id; 625 int sec_btf_id; 626 char *name; 627 char *essent_name; 628 bool is_set; 629 bool is_weak; 630 union { 631 struct { 632 enum kcfg_type type; 633 int sz; 634 int align; 635 int data_off; 636 bool is_signed; 637 } kcfg; 638 struct { 639 unsigned long long addr; 640 641 /* target btf_id of the corresponding kernel var. */ 642 int kernel_btf_obj_fd; 643 int kernel_btf_id; 644 645 /* local btf_id of the ksym extern's type. */ 646 __u32 type_id; 647 /* BTF fd index to be patched in for insn->off, this is 648 * 0 for vmlinux BTF, index in obj->fd_array for module 649 * BTF 650 */ 651 __s16 btf_fd_idx; 652 } ksym; 653 }; 654 }; 655 656 struct module_btf { 657 struct btf *btf; 658 char *name; 659 __u32 id; 660 int fd; 661 int fd_array_idx; 662 }; 663 664 enum sec_type { 665 SEC_UNUSED = 0, 666 SEC_RELO, 667 SEC_BSS, 668 SEC_DATA, 669 SEC_RODATA, 670 SEC_ST_OPS, 671 SEC_PERCPU, 672 }; 673 674 struct elf_sec_desc { 675 enum sec_type sec_type; 676 Elf64_Shdr *shdr; 677 Elf_Data *data; 678 }; 679 680 struct elf_state { 681 int fd; 682 const void *obj_buf; 683 size_t obj_buf_sz; 684 Elf *elf; 685 Elf64_Ehdr *ehdr; 686 Elf_Data *symbols; 687 Elf_Data *arena_data; 688 size_t shstrndx; /* section index for section name strings */ 689 size_t strtabidx; 690 struct elf_sec_desc *secs; 691 size_t sec_cnt; 692 int btf_maps_shndx; 693 __u32 btf_maps_sec_btf_id; 694 int text_shndx; 695 int symbols_shndx; 696 bool has_st_ops; 697 int arena_data_shndx; 698 int jumptables_data_shndx; 699 }; 700 701 struct usdt_manager; 702 703 enum bpf_object_state { 704 OBJ_OPEN, 705 OBJ_PREPARED, 706 OBJ_LOADED, 707 }; 708 709 struct bpf_object { 710 char name[BPF_OBJ_NAME_LEN]; 711 char license[64]; 712 __u32 kern_version; 713 714 enum bpf_object_state state; 715 struct bpf_program *programs; 716 size_t nr_programs; 717 struct bpf_map *maps; 718 size_t nr_maps; 719 size_t maps_cap; 720 721 char *kconfig; 722 struct extern_desc *externs; 723 int nr_extern; 724 int kconfig_map_idx; 725 726 bool has_subcalls; 727 bool has_rodata; 728 729 struct bpf_gen *gen_loader; 730 731 /* Information when doing ELF related work. Only valid if efile.elf is not NULL */ 732 struct elf_state efile; 733 734 unsigned char byteorder; 735 736 struct btf *btf; 737 struct btf_ext *btf_ext; 738 739 /* Parse and load BTF vmlinux if any of the programs in the object need 740 * it at load time. 741 */ 742 struct btf *btf_vmlinux; 743 /* Path to the custom BTF to be used for BPF CO-RE relocations as an 744 * override for vmlinux BTF. 745 */ 746 char *btf_custom_path; 747 /* vmlinux BTF override for CO-RE relocations */ 748 struct btf *btf_vmlinux_override; 749 /* Lazily initialized kernel module BTFs */ 750 struct module_btf *btf_modules; 751 bool btf_modules_loaded; 752 size_t btf_module_cnt; 753 size_t btf_module_cap; 754 755 /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */ 756 char *log_buf; 757 size_t log_size; 758 __u32 log_level; 759 760 int *fd_array; 761 size_t fd_array_cap; 762 size_t fd_array_cnt; 763 764 struct usdt_manager *usdt_man; 765 766 int arena_map_idx; 767 void *arena_data; 768 size_t arena_data_sz; 769 size_t arena_data_off; 770 771 void *jumptables_data; 772 size_t jumptables_data_sz; 773 774 struct { 775 struct bpf_program *prog; 776 unsigned int sym_off; 777 int fd; 778 } *jumptable_maps; 779 size_t jumptable_map_cnt; 780 781 struct kern_feature_cache *feat_cache; 782 char *token_path; 783 int token_fd; 784 785 char path[]; 786 }; 787 788 static const char *elf_sym_str(const struct bpf_object *obj, size_t off); 789 static const char *elf_sec_str(const struct bpf_object *obj, size_t off); 790 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx); 791 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name); 792 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn); 793 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn); 794 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn); 795 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx); 796 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx); 797 798 void bpf_program__unload(struct bpf_program *prog) 799 { 800 if (!prog) 801 return; 802 803 zclose(prog->fd); 804 805 zfree(&prog->func_info); 806 zfree(&prog->line_info); 807 zfree(&prog->subprogs); 808 } 809 810 static void bpf_program__exit(struct bpf_program *prog) 811 { 812 if (!prog) 813 return; 814 815 bpf_program__unload(prog); 816 zfree(&prog->name); 817 zfree(&prog->sec_name); 818 zfree(&prog->insns); 819 zfree(&prog->reloc_desc); 820 821 prog->nr_reloc = 0; 822 prog->insns_cnt = 0; 823 prog->sec_idx = -1; 824 } 825 826 static bool insn_is_subprog_call(const struct bpf_insn *insn) 827 { 828 return BPF_CLASS(insn->code) == BPF_JMP && 829 BPF_OP(insn->code) == BPF_CALL && 830 BPF_SRC(insn->code) == BPF_K && 831 insn->src_reg == BPF_PSEUDO_CALL && 832 insn->dst_reg == 0 && 833 insn->off == 0; 834 } 835 836 static bool is_call_insn(const struct bpf_insn *insn) 837 { 838 return insn->code == (BPF_JMP | BPF_CALL); 839 } 840 841 static bool insn_is_pseudo_func(struct bpf_insn *insn) 842 { 843 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; 844 } 845 846 static int 847 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, 848 const char *name, size_t sec_idx, const char *sec_name, 849 size_t sec_off, void *insn_data, size_t insn_data_sz) 850 { 851 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) { 852 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n", 853 sec_name, name, sec_off, insn_data_sz); 854 return -EINVAL; 855 } 856 857 memset(prog, 0, sizeof(*prog)); 858 prog->obj = obj; 859 860 prog->sec_idx = sec_idx; 861 prog->sec_insn_off = sec_off / BPF_INSN_SZ; 862 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; 863 /* insns_cnt can later be increased by appending used subprograms */ 864 prog->insns_cnt = prog->sec_insn_cnt; 865 866 prog->type = BPF_PROG_TYPE_UNSPEC; 867 prog->fd = -1; 868 prog->exception_cb_idx = -1; 869 870 /* libbpf's convention for SEC("?abc...") is that it's just like 871 * SEC("abc...") but the corresponding bpf_program starts out with 872 * autoload set to false. 873 */ 874 if (sec_name[0] == '?') { 875 prog->autoload = false; 876 /* from now on forget there was ? in section name */ 877 sec_name++; 878 } else { 879 prog->autoload = true; 880 } 881 882 prog->autoattach = true; 883 884 /* inherit object's log_level */ 885 prog->log_level = obj->log_level; 886 887 prog->sec_name = strdup(sec_name); 888 if (!prog->sec_name) 889 goto errout; 890 891 prog->name = strdup(name); 892 if (!prog->name) 893 goto errout; 894 895 prog->insns = malloc(insn_data_sz); 896 if (!prog->insns) 897 goto errout; 898 memcpy(prog->insns, insn_data, insn_data_sz); 899 900 return 0; 901 errout: 902 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name); 903 bpf_program__exit(prog); 904 return -ENOMEM; 905 } 906 907 static int 908 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, 909 const char *sec_name, int sec_idx) 910 { 911 Elf_Data *symbols = obj->efile.symbols; 912 struct bpf_program *prog, *progs; 913 void *data = sec_data->d_buf; 914 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; 915 int nr_progs, err, i; 916 const char *name; 917 Elf64_Sym *sym; 918 919 progs = obj->programs; 920 nr_progs = obj->nr_programs; 921 nr_syms = symbols->d_size / sizeof(Elf64_Sym); 922 923 for (i = 0; i < nr_syms; i++) { 924 sym = elf_sym_by_idx(obj, i); 925 926 if (sym->st_shndx != sec_idx) 927 continue; 928 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) 929 continue; 930 931 prog_sz = sym->st_size; 932 sec_off = sym->st_value; 933 934 name = elf_sym_str(obj, sym->st_name); 935 if (!name) { 936 pr_warn("sec '%s': failed to get symbol name for offset %zu\n", 937 sec_name, sec_off); 938 return -LIBBPF_ERRNO__FORMAT; 939 } 940 941 if (sec_off + prog_sz > sec_sz || sec_off + prog_sz < sec_off) { 942 pr_warn("sec '%s': program at offset %zu crosses section boundary\n", 943 sec_name, sec_off); 944 return -LIBBPF_ERRNO__FORMAT; 945 } 946 947 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { 948 pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name); 949 return -ENOTSUP; 950 } 951 952 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n", 953 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz); 954 955 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs)); 956 if (!progs) { 957 /* 958 * In this case the original obj->programs 959 * is still valid, so don't need special treat for 960 * bpf_close_object(). 961 */ 962 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n", 963 sec_name, name); 964 return -ENOMEM; 965 } 966 obj->programs = progs; 967 968 prog = &progs[nr_progs]; 969 970 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, 971 sec_off, data + sec_off, prog_sz); 972 if (err) 973 return err; 974 975 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL) 976 prog->sym_global = true; 977 978 /* if function is a global/weak symbol, but has restricted 979 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC 980 * as static to enable more permissive BPF verification mode 981 * with more outside context available to BPF verifier 982 */ 983 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN 984 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) 985 prog->mark_btf_static = true; 986 987 nr_progs++; 988 obj->nr_programs = nr_progs; 989 } 990 991 return 0; 992 } 993 994 static void bpf_object_bswap_progs(struct bpf_object *obj) 995 { 996 struct bpf_program *prog = obj->programs; 997 struct bpf_insn *insn; 998 int p, i; 999 1000 for (p = 0; p < obj->nr_programs; p++, prog++) { 1001 insn = prog->insns; 1002 for (i = 0; i < prog->insns_cnt; i++, insn++) 1003 bpf_insn_bswap(insn); 1004 } 1005 pr_debug("converted %zu BPF programs to native byte order\n", obj->nr_programs); 1006 } 1007 1008 static const struct btf_member * 1009 find_member_by_offset(const struct btf_type *t, __u32 bit_offset) 1010 { 1011 struct btf_member *m; 1012 int i; 1013 1014 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 1015 if (btf_member_bit_offset(t, i) == bit_offset) 1016 return m; 1017 } 1018 1019 return NULL; 1020 } 1021 1022 static const struct btf_member * 1023 find_member_by_name(const struct btf *btf, const struct btf_type *t, 1024 const char *name) 1025 { 1026 struct btf_member *m; 1027 int i; 1028 1029 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 1030 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) 1031 return m; 1032 } 1033 1034 return NULL; 1035 } 1036 1037 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, 1038 __u16 kind, struct btf **res_btf, 1039 struct module_btf **res_mod_btf); 1040 1041 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" 1042 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 1043 const char *name, __u32 kind); 1044 1045 static int 1046 find_struct_ops_kern_types(struct bpf_object *obj, const char *tname_raw, 1047 struct module_btf **mod_btf, 1048 const struct btf_type **type, __u32 *type_id, 1049 const struct btf_type **vtype, __u32 *vtype_id, 1050 const struct btf_member **data_member) 1051 { 1052 const struct btf_type *kern_type, *kern_vtype; 1053 const struct btf_member *kern_data_member; 1054 struct btf *btf = NULL; 1055 __s32 kern_vtype_id, kern_type_id; 1056 char tname[192], stname[256]; 1057 __u32 i; 1058 1059 snprintf(tname, sizeof(tname), "%.*s", 1060 (int)bpf_core_essential_name_len(tname_raw), tname_raw); 1061 1062 snprintf(stname, sizeof(stname), "%s%s", STRUCT_OPS_VALUE_PREFIX, tname); 1063 1064 /* Look for the corresponding "map_value" type that will be used 1065 * in map_update(BPF_MAP_TYPE_STRUCT_OPS) first, figure out the btf 1066 * and the mod_btf. 1067 * For example, find "struct bpf_struct_ops_tcp_congestion_ops". 1068 */ 1069 kern_vtype_id = find_ksym_btf_id(obj, stname, BTF_KIND_STRUCT, &btf, mod_btf); 1070 if (kern_vtype_id < 0) { 1071 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", stname); 1072 return kern_vtype_id; 1073 } 1074 kern_vtype = btf__type_by_id(btf, kern_vtype_id); 1075 1076 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT); 1077 if (kern_type_id < 0) { 1078 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", tname); 1079 return kern_type_id; 1080 } 1081 kern_type = btf__type_by_id(btf, kern_type_id); 1082 1083 /* Find "struct tcp_congestion_ops" from 1084 * struct bpf_struct_ops_tcp_congestion_ops { 1085 * [ ... ] 1086 * struct tcp_congestion_ops data; 1087 * } 1088 */ 1089 kern_data_member = btf_members(kern_vtype); 1090 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { 1091 if (kern_data_member->type == kern_type_id) 1092 break; 1093 } 1094 if (i == btf_vlen(kern_vtype)) { 1095 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s\n", 1096 tname, stname); 1097 return -EINVAL; 1098 } 1099 1100 *type = kern_type; 1101 *type_id = kern_type_id; 1102 *vtype = kern_vtype; 1103 *vtype_id = kern_vtype_id; 1104 *data_member = kern_data_member; 1105 1106 return 0; 1107 } 1108 1109 static bool bpf_map__is_struct_ops(const struct bpf_map *map) 1110 { 1111 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; 1112 } 1113 1114 static bool is_valid_st_ops_program(struct bpf_object *obj, 1115 const struct bpf_program *prog) 1116 { 1117 int i; 1118 1119 for (i = 0; i < obj->nr_programs; i++) { 1120 if (&obj->programs[i] == prog) 1121 return prog->type == BPF_PROG_TYPE_STRUCT_OPS; 1122 } 1123 1124 return false; 1125 } 1126 1127 /* For each struct_ops program P, referenced from some struct_ops map M, 1128 * enable P.autoload if there are Ms for which M.autocreate is true, 1129 * disable P.autoload if for all Ms M.autocreate is false. 1130 * Don't change P.autoload for programs that are not referenced from any maps. 1131 */ 1132 static int bpf_object_adjust_struct_ops_autoload(struct bpf_object *obj) 1133 { 1134 struct bpf_program *prog, *slot_prog; 1135 struct bpf_map *map; 1136 int i, j, k, vlen; 1137 1138 for (i = 0; i < obj->nr_programs; ++i) { 1139 int should_load = false; 1140 int use_cnt = 0; 1141 1142 prog = &obj->programs[i]; 1143 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) 1144 continue; 1145 1146 for (j = 0; j < obj->nr_maps; ++j) { 1147 const struct btf_type *type; 1148 1149 map = &obj->maps[j]; 1150 if (!bpf_map__is_struct_ops(map)) 1151 continue; 1152 1153 type = btf__type_by_id(obj->btf, map->st_ops->type_id); 1154 vlen = btf_vlen(type); 1155 for (k = 0; k < vlen; ++k) { 1156 slot_prog = map->st_ops->progs[k]; 1157 if (prog != slot_prog) 1158 continue; 1159 1160 use_cnt++; 1161 if (map->autocreate) 1162 should_load = true; 1163 } 1164 } 1165 if (use_cnt) 1166 prog->autoload = should_load; 1167 } 1168 1169 return 0; 1170 } 1171 1172 /* Init the map's fields that depend on kern_btf */ 1173 static int bpf_map__init_kern_struct_ops(struct bpf_map *map) 1174 { 1175 const struct btf_member *member, *kern_member, *kern_data_member; 1176 const struct btf_type *type, *kern_type, *kern_vtype; 1177 __u32 i, kern_type_id, kern_vtype_id, kern_data_off; 1178 struct bpf_object *obj = map->obj; 1179 const struct btf *btf = obj->btf; 1180 struct bpf_struct_ops *st_ops; 1181 const struct btf *kern_btf; 1182 struct module_btf *mod_btf = NULL; 1183 void *data, *kern_data; 1184 const char *tname; 1185 int err; 1186 1187 st_ops = map->st_ops; 1188 type = btf__type_by_id(btf, st_ops->type_id); 1189 tname = btf__name_by_offset(btf, type->name_off); 1190 err = find_struct_ops_kern_types(obj, tname, &mod_btf, 1191 &kern_type, &kern_type_id, 1192 &kern_vtype, &kern_vtype_id, 1193 &kern_data_member); 1194 if (err) 1195 return err; 1196 1197 kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux; 1198 1199 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", 1200 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); 1201 1202 map->mod_btf_fd = mod_btf ? mod_btf->fd : -1; 1203 map->def.value_size = kern_vtype->size; 1204 map->btf_vmlinux_value_type_id = kern_vtype_id; 1205 1206 st_ops->kern_vdata = calloc(1, kern_vtype->size); 1207 if (!st_ops->kern_vdata) 1208 return -ENOMEM; 1209 1210 data = st_ops->data; 1211 kern_data_off = kern_data_member->offset / 8; 1212 kern_data = st_ops->kern_vdata + kern_data_off; 1213 1214 member = btf_members(type); 1215 for (i = 0; i < btf_vlen(type); i++, member++) { 1216 const struct btf_type *mtype, *kern_mtype; 1217 __u32 mtype_id, kern_mtype_id; 1218 void *mdata, *kern_mdata; 1219 struct bpf_program *prog; 1220 __s64 msize, kern_msize; 1221 __u32 moff, kern_moff; 1222 __u32 kern_member_idx; 1223 const char *mname; 1224 1225 mname = btf__name_by_offset(btf, member->name_off); 1226 moff = member->offset / 8; 1227 mdata = data + moff; 1228 msize = btf__resolve_size(btf, member->type); 1229 if (msize < 0) { 1230 pr_warn("struct_ops init_kern %s: failed to resolve the size of member %s\n", 1231 map->name, mname); 1232 return msize; 1233 } 1234 1235 kern_member = find_member_by_name(kern_btf, kern_type, mname); 1236 if (!kern_member) { 1237 if (!libbpf_is_mem_zeroed(mdata, msize)) { 1238 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", 1239 map->name, mname); 1240 return -ENOTSUP; 1241 } 1242 1243 if (st_ops->progs[i]) { 1244 /* If we had declaratively set struct_ops callback, we need to 1245 * force its autoload to false, because it doesn't have 1246 * a chance of succeeding from POV of the current struct_ops map. 1247 * If this program is still referenced somewhere else, though, 1248 * then bpf_object_adjust_struct_ops_autoload() will update its 1249 * autoload accordingly. 1250 */ 1251 st_ops->progs[i]->autoload = false; 1252 st_ops->progs[i] = NULL; 1253 } 1254 1255 /* Skip all-zero/NULL fields if they are not present in the kernel BTF */ 1256 pr_info("struct_ops %s: member %s not found in kernel, skipping it as it's set to zero\n", 1257 map->name, mname); 1258 continue; 1259 } 1260 1261 kern_member_idx = kern_member - btf_members(kern_type); 1262 if (btf_member_bitfield_size(type, i) || 1263 btf_member_bitfield_size(kern_type, kern_member_idx)) { 1264 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", 1265 map->name, mname); 1266 return -ENOTSUP; 1267 } 1268 1269 kern_moff = kern_member->offset / 8; 1270 kern_mdata = kern_data + kern_moff; 1271 1272 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); 1273 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, 1274 &kern_mtype_id); 1275 if (BTF_INFO_KIND(mtype->info) != 1276 BTF_INFO_KIND(kern_mtype->info)) { 1277 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", 1278 map->name, mname, BTF_INFO_KIND(mtype->info), 1279 BTF_INFO_KIND(kern_mtype->info)); 1280 return -ENOTSUP; 1281 } 1282 1283 if (btf_is_ptr(mtype)) { 1284 prog = *(void **)mdata; 1285 /* just like for !kern_member case above, reset declaratively 1286 * set (at compile time) program's autload to false, 1287 * if user replaced it with another program or NULL 1288 */ 1289 if (st_ops->progs[i] && st_ops->progs[i] != prog) 1290 st_ops->progs[i]->autoload = false; 1291 1292 /* Update the value from the shadow type */ 1293 st_ops->progs[i] = prog; 1294 if (!prog) 1295 continue; 1296 1297 if (!is_valid_st_ops_program(obj, prog)) { 1298 pr_warn("struct_ops init_kern %s: member %s is not a struct_ops program\n", 1299 map->name, mname); 1300 return -ENOTSUP; 1301 } 1302 1303 kern_mtype = skip_mods_and_typedefs(kern_btf, 1304 kern_mtype->type, 1305 &kern_mtype_id); 1306 1307 /* mtype->type must be a func_proto which was 1308 * guaranteed in bpf_object__collect_st_ops_relos(), 1309 * so only check kern_mtype for func_proto here. 1310 */ 1311 if (!btf_is_func_proto(kern_mtype)) { 1312 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n", 1313 map->name, mname); 1314 return -ENOTSUP; 1315 } 1316 1317 if (mod_btf) 1318 prog->attach_btf_obj_fd = mod_btf->fd; 1319 1320 /* if we haven't yet processed this BPF program, record proper 1321 * attach_btf_id and member_idx 1322 */ 1323 if (!prog->attach_btf_id) { 1324 prog->attach_btf_id = kern_type_id; 1325 prog->expected_attach_type = kern_member_idx; 1326 } 1327 1328 /* struct_ops BPF prog can be re-used between multiple 1329 * .struct_ops & .struct_ops.link as long as it's the 1330 * same struct_ops struct definition and the same 1331 * function pointer field 1332 */ 1333 if (prog->attach_btf_id != kern_type_id) { 1334 pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: attach_btf_id %u != kern_type_id %u\n", 1335 map->name, mname, prog->name, prog->sec_name, prog->type, 1336 prog->attach_btf_id, kern_type_id); 1337 return -EINVAL; 1338 } 1339 if (prog->expected_attach_type != kern_member_idx) { 1340 pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: expected_attach_type %u != kern_member_idx %u\n", 1341 map->name, mname, prog->name, prog->sec_name, prog->type, 1342 prog->expected_attach_type, kern_member_idx); 1343 return -EINVAL; 1344 } 1345 1346 st_ops->kern_func_off[i] = kern_data_off + kern_moff; 1347 1348 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", 1349 map->name, mname, prog->name, moff, 1350 kern_moff); 1351 1352 continue; 1353 } 1354 1355 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); 1356 if (kern_msize < 0 || msize != kern_msize) { 1357 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", 1358 map->name, mname, (ssize_t)msize, 1359 (ssize_t)kern_msize); 1360 return -ENOTSUP; 1361 } 1362 1363 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", 1364 map->name, mname, (unsigned int)msize, 1365 moff, kern_moff); 1366 memcpy(kern_mdata, mdata, msize); 1367 } 1368 1369 return 0; 1370 } 1371 1372 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) 1373 { 1374 struct bpf_map *map; 1375 size_t i; 1376 int err; 1377 1378 for (i = 0; i < obj->nr_maps; i++) { 1379 map = &obj->maps[i]; 1380 1381 if (!bpf_map__is_struct_ops(map)) 1382 continue; 1383 1384 if (!map->autocreate) 1385 continue; 1386 1387 err = bpf_map__init_kern_struct_ops(map); 1388 if (err) 1389 return err; 1390 } 1391 1392 return 0; 1393 } 1394 1395 static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name, 1396 int shndx, Elf_Data *data) 1397 { 1398 const struct btf_type *type, *datasec; 1399 const struct btf_var_secinfo *vsi; 1400 struct bpf_struct_ops *st_ops; 1401 const char *tname, *var_name; 1402 __s32 type_id, datasec_id; 1403 const struct btf *btf; 1404 struct bpf_map *map; 1405 __u32 i; 1406 1407 if (shndx == -1) 1408 return 0; 1409 1410 btf = obj->btf; 1411 datasec_id = btf__find_by_name_kind(btf, sec_name, 1412 BTF_KIND_DATASEC); 1413 if (datasec_id < 0) { 1414 pr_warn("struct_ops init: DATASEC %s not found\n", 1415 sec_name); 1416 return -EINVAL; 1417 } 1418 1419 datasec = btf__type_by_id(btf, datasec_id); 1420 vsi = btf_var_secinfos(datasec); 1421 for (i = 0; i < btf_vlen(datasec); i++, vsi++) { 1422 type = btf__type_by_id(obj->btf, vsi->type); 1423 var_name = btf__name_by_offset(obj->btf, type->name_off); 1424 1425 type_id = btf__resolve_type(obj->btf, vsi->type); 1426 if (type_id < 0) { 1427 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", 1428 vsi->type, sec_name); 1429 return -EINVAL; 1430 } 1431 1432 type = btf__type_by_id(obj->btf, type_id); 1433 tname = btf__name_by_offset(obj->btf, type->name_off); 1434 if (!tname[0]) { 1435 pr_warn("struct_ops init: anonymous type is not supported\n"); 1436 return -ENOTSUP; 1437 } 1438 if (!btf_is_struct(type)) { 1439 pr_warn("struct_ops init: %s is not a struct\n", tname); 1440 return -EINVAL; 1441 } 1442 1443 map = bpf_object__add_map(obj); 1444 if (IS_ERR(map)) 1445 return PTR_ERR(map); 1446 1447 map->sec_idx = shndx; 1448 map->sec_offset = vsi->offset; 1449 map->name = strdup(var_name); 1450 if (!map->name) 1451 return -ENOMEM; 1452 map->btf_value_type_id = type_id; 1453 1454 /* Follow same convention as for programs autoload: 1455 * SEC("?.struct_ops") means map is not created by default. 1456 */ 1457 if (sec_name[0] == '?') { 1458 map->autocreate = false; 1459 /* from now on forget there was ? in section name */ 1460 sec_name++; 1461 } 1462 1463 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; 1464 map->def.key_size = sizeof(int); 1465 map->def.value_size = type->size; 1466 map->def.max_entries = 1; 1467 map->def.map_flags = strcmp(sec_name, STRUCT_OPS_LINK_SEC) == 0 ? BPF_F_LINK : 0; 1468 map->autoattach = true; 1469 1470 map->st_ops = calloc(1, sizeof(*map->st_ops)); 1471 if (!map->st_ops) 1472 return -ENOMEM; 1473 st_ops = map->st_ops; 1474 st_ops->data = malloc(type->size); 1475 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); 1476 st_ops->kern_func_off = malloc(btf_vlen(type) * 1477 sizeof(*st_ops->kern_func_off)); 1478 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) 1479 return -ENOMEM; 1480 1481 if (vsi->offset + type->size > data->d_size) { 1482 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", 1483 var_name, sec_name); 1484 return -EINVAL; 1485 } 1486 1487 memcpy(st_ops->data, 1488 data->d_buf + vsi->offset, 1489 type->size); 1490 st_ops->type_id = type_id; 1491 1492 pr_debug("struct_ops init: struct %s(type_id=%d) %s found at offset %u\n", 1493 tname, type_id, var_name, vsi->offset); 1494 } 1495 1496 return 0; 1497 } 1498 1499 static int bpf_object_init_struct_ops(struct bpf_object *obj) 1500 { 1501 const char *sec_name; 1502 int sec_idx, err; 1503 1504 for (sec_idx = 0; sec_idx < obj->efile.sec_cnt; ++sec_idx) { 1505 struct elf_sec_desc *desc = &obj->efile.secs[sec_idx]; 1506 1507 if (desc->sec_type != SEC_ST_OPS) 1508 continue; 1509 1510 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 1511 if (!sec_name) 1512 return -LIBBPF_ERRNO__FORMAT; 1513 1514 err = init_struct_ops_maps(obj, sec_name, sec_idx, desc->data); 1515 if (err) 1516 return err; 1517 } 1518 1519 return 0; 1520 } 1521 1522 static struct bpf_object *bpf_object__new(const char *path, 1523 const void *obj_buf, 1524 size_t obj_buf_sz, 1525 const char *obj_name) 1526 { 1527 struct bpf_object *obj; 1528 char *end; 1529 1530 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 1531 if (!obj) { 1532 pr_warn("alloc memory failed for %s\n", path); 1533 return ERR_PTR(-ENOMEM); 1534 } 1535 1536 strcpy(obj->path, path); 1537 if (obj_name) { 1538 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); 1539 } else { 1540 /* Using basename() GNU version which doesn't modify arg. */ 1541 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); 1542 end = strchr(obj->name, '.'); 1543 if (end) 1544 *end = 0; 1545 } 1546 1547 obj->efile.fd = -1; 1548 /* 1549 * Caller of this function should also call 1550 * bpf_object__elf_finish() after data collection to return 1551 * obj_buf to user. If not, we should duplicate the buffer to 1552 * avoid user freeing them before elf finish. 1553 */ 1554 obj->efile.obj_buf = obj_buf; 1555 obj->efile.obj_buf_sz = obj_buf_sz; 1556 obj->efile.btf_maps_shndx = -1; 1557 obj->kconfig_map_idx = -1; 1558 obj->arena_map_idx = -1; 1559 1560 obj->kern_version = get_kernel_version(); 1561 obj->state = OBJ_OPEN; 1562 1563 return obj; 1564 } 1565 1566 static void bpf_object__elf_finish(struct bpf_object *obj) 1567 { 1568 if (!obj->efile.elf) 1569 return; 1570 1571 elf_end(obj->efile.elf); 1572 obj->efile.elf = NULL; 1573 obj->efile.ehdr = NULL; 1574 obj->efile.symbols = NULL; 1575 obj->efile.arena_data = NULL; 1576 1577 zfree(&obj->efile.secs); 1578 obj->efile.sec_cnt = 0; 1579 zclose(obj->efile.fd); 1580 obj->efile.obj_buf = NULL; 1581 obj->efile.obj_buf_sz = 0; 1582 } 1583 1584 static int bpf_object__elf_init(struct bpf_object *obj) 1585 { 1586 Elf64_Ehdr *ehdr; 1587 int err = 0; 1588 Elf *elf; 1589 1590 if (obj->efile.elf) { 1591 pr_warn("elf: init internal error\n"); 1592 return -LIBBPF_ERRNO__LIBELF; 1593 } 1594 1595 if (obj->efile.obj_buf_sz > 0) { 1596 /* obj_buf should have been validated by bpf_object__open_mem(). */ 1597 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz); 1598 } else { 1599 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC); 1600 if (obj->efile.fd < 0) { 1601 err = -errno; 1602 pr_warn("elf: failed to open %s: %s\n", obj->path, errstr(err)); 1603 return err; 1604 } 1605 1606 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); 1607 } 1608 1609 if (!elf) { 1610 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); 1611 err = -LIBBPF_ERRNO__LIBELF; 1612 goto errout; 1613 } 1614 1615 obj->efile.elf = elf; 1616 1617 if (elf_kind(elf) != ELF_K_ELF) { 1618 err = -LIBBPF_ERRNO__FORMAT; 1619 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path); 1620 goto errout; 1621 } 1622 1623 if (gelf_getclass(elf) != ELFCLASS64) { 1624 err = -LIBBPF_ERRNO__FORMAT; 1625 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path); 1626 goto errout; 1627 } 1628 1629 obj->efile.ehdr = ehdr = elf64_getehdr(elf); 1630 if (!obj->efile.ehdr) { 1631 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); 1632 err = -LIBBPF_ERRNO__FORMAT; 1633 goto errout; 1634 } 1635 1636 /* Validate ELF object endianness... */ 1637 if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB && 1638 ehdr->e_ident[EI_DATA] != ELFDATA2MSB) { 1639 err = -LIBBPF_ERRNO__ENDIAN; 1640 pr_warn("elf: '%s' has unknown byte order\n", obj->path); 1641 goto errout; 1642 } 1643 /* and save after bpf_object_open() frees ELF data */ 1644 obj->byteorder = ehdr->e_ident[EI_DATA]; 1645 1646 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) { 1647 pr_warn("elf: failed to get section names section index for %s: %s\n", 1648 obj->path, elf_errmsg(-1)); 1649 err = -LIBBPF_ERRNO__FORMAT; 1650 goto errout; 1651 } 1652 1653 /* ELF is corrupted/truncated, avoid calling elf_strptr. */ 1654 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) { 1655 pr_warn("elf: failed to get section names strings from %s: %s\n", 1656 obj->path, elf_errmsg(-1)); 1657 err = -LIBBPF_ERRNO__FORMAT; 1658 goto errout; 1659 } 1660 1661 /* Old LLVM set e_machine to EM_NONE */ 1662 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) { 1663 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); 1664 err = -LIBBPF_ERRNO__FORMAT; 1665 goto errout; 1666 } 1667 1668 return 0; 1669 errout: 1670 bpf_object__elf_finish(obj); 1671 return err; 1672 } 1673 1674 static bool is_native_endianness(struct bpf_object *obj) 1675 { 1676 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1677 return obj->byteorder == ELFDATA2LSB; 1678 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 1679 return obj->byteorder == ELFDATA2MSB; 1680 #else 1681 # error "Unrecognized __BYTE_ORDER__" 1682 #endif 1683 } 1684 1685 static int 1686 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 1687 { 1688 if (!data) { 1689 pr_warn("invalid license section in %s\n", obj->path); 1690 return -LIBBPF_ERRNO__FORMAT; 1691 } 1692 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't 1693 * go over allowed ELF data section buffer 1694 */ 1695 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license))); 1696 pr_debug("license of %s is %s\n", obj->path, obj->license); 1697 return 0; 1698 } 1699 1700 static int 1701 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 1702 { 1703 __u32 kver; 1704 1705 if (!data || size != sizeof(kver)) { 1706 pr_warn("invalid kver section in %s\n", obj->path); 1707 return -LIBBPF_ERRNO__FORMAT; 1708 } 1709 memcpy(&kver, data, sizeof(kver)); 1710 obj->kern_version = kver; 1711 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 1712 return 0; 1713 } 1714 1715 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 1716 { 1717 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 1718 type == BPF_MAP_TYPE_HASH_OF_MAPS) 1719 return true; 1720 return false; 1721 } 1722 1723 static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size) 1724 { 1725 Elf_Data *data; 1726 Elf_Scn *scn; 1727 1728 if (!name) 1729 return -EINVAL; 1730 1731 scn = elf_sec_by_name(obj, name); 1732 data = elf_sec_data(obj, scn); 1733 if (data) { 1734 *size = data->d_size; 1735 return 0; /* found it */ 1736 } 1737 1738 return -ENOENT; 1739 } 1740 1741 static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name) 1742 { 1743 Elf_Data *symbols = obj->efile.symbols; 1744 const char *sname; 1745 size_t si; 1746 1747 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) { 1748 Elf64_Sym *sym = elf_sym_by_idx(obj, si); 1749 1750 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT) 1751 continue; 1752 1753 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL && 1754 ELF64_ST_BIND(sym->st_info) != STB_WEAK) 1755 continue; 1756 1757 sname = elf_sym_str(obj, sym->st_name); 1758 if (!sname) { 1759 pr_warn("failed to get sym name string for var %s\n", name); 1760 return ERR_PTR(-EIO); 1761 } 1762 if (strcmp(name, sname) == 0) 1763 return sym; 1764 } 1765 1766 return ERR_PTR(-ENOENT); 1767 } 1768 1769 #ifndef MFD_CLOEXEC 1770 #define MFD_CLOEXEC 0x0001U 1771 #endif 1772 #ifndef MFD_NOEXEC_SEAL 1773 #define MFD_NOEXEC_SEAL 0x0008U 1774 #endif 1775 1776 static int create_placeholder_fd(void) 1777 { 1778 unsigned int flags = MFD_CLOEXEC | MFD_NOEXEC_SEAL; 1779 const char *name = "libbpf-placeholder-fd"; 1780 int fd; 1781 1782 fd = ensure_good_fd(sys_memfd_create(name, flags)); 1783 if (fd >= 0) 1784 return fd; 1785 else if (errno != EINVAL) 1786 return -errno; 1787 1788 /* Possibly running on kernel without MFD_NOEXEC_SEAL */ 1789 fd = ensure_good_fd(sys_memfd_create(name, flags & ~MFD_NOEXEC_SEAL)); 1790 if (fd < 0) 1791 return -errno; 1792 return fd; 1793 } 1794 1795 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 1796 { 1797 struct bpf_map *map; 1798 int err; 1799 1800 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap, 1801 sizeof(*obj->maps), obj->nr_maps + 1); 1802 if (err) 1803 return ERR_PTR(err); 1804 1805 map = &obj->maps[obj->nr_maps++]; 1806 map->obj = obj; 1807 /* Preallocate map FD without actually creating BPF map just yet. 1808 * These map FD "placeholders" will be reused later without changing 1809 * FD value when map is actually created in the kernel. 1810 * 1811 * This is useful to be able to perform BPF program relocations 1812 * without having to create BPF maps before that step. This allows us 1813 * to finalize and load BTF very late in BPF object's loading phase, 1814 * right before BPF maps have to be created and BPF programs have to 1815 * be loaded. By having these map FD placeholders we can perform all 1816 * the sanitizations, relocations, and any other adjustments before we 1817 * start creating actual BPF kernel objects (BTF, maps, progs). 1818 */ 1819 map->fd = create_placeholder_fd(); 1820 if (map->fd < 0) 1821 return ERR_PTR(map->fd); 1822 map->inner_map_fd = -1; 1823 map->autocreate = true; 1824 1825 return map; 1826 } 1827 1828 static size_t array_map_mmap_sz(unsigned int value_sz, unsigned int max_entries) 1829 { 1830 const long page_sz = sysconf(_SC_PAGE_SIZE); 1831 size_t map_sz; 1832 1833 map_sz = (size_t)roundup(value_sz, 8) * max_entries; 1834 map_sz = roundup(map_sz, page_sz); 1835 return map_sz; 1836 } 1837 1838 static size_t bpf_map_mmap_sz(const struct bpf_map *map) 1839 { 1840 const long page_sz = sysconf(_SC_PAGE_SIZE); 1841 1842 switch (map->def.type) { 1843 case BPF_MAP_TYPE_ARRAY: 1844 case BPF_MAP_TYPE_PERCPU_ARRAY: 1845 return array_map_mmap_sz(map->def.value_size, map->def.max_entries); 1846 case BPF_MAP_TYPE_ARENA: 1847 return page_sz * map->def.max_entries; 1848 default: 1849 return 0; /* not supported */ 1850 } 1851 } 1852 1853 static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz) 1854 { 1855 void *mmaped; 1856 1857 if (!map->mmaped) 1858 return -EINVAL; 1859 1860 if (old_sz == new_sz) 1861 return 0; 1862 1863 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1864 if (mmaped == MAP_FAILED) 1865 return -errno; 1866 1867 memcpy(mmaped, map->mmaped, min(old_sz, new_sz)); 1868 munmap(map->mmaped, old_sz); 1869 map->mmaped = mmaped; 1870 return 0; 1871 } 1872 1873 static char *internal_map_name(struct bpf_object *obj, const char *real_name, 1874 enum libbpf_map_type type) 1875 { 1876 char map_name[BPF_OBJ_NAME_LEN], *p; 1877 int pfx_len, sfx_len = max((size_t)7, strlen(real_name)); 1878 1879 /* This is one of the more confusing parts of libbpf for various 1880 * reasons, some of which are historical. The original idea for naming 1881 * internal names was to include as much of BPF object name prefix as 1882 * possible, so that it can be distinguished from similar internal 1883 * maps of a different BPF object. 1884 * As an example, let's say we have bpf_object named 'my_object_name' 1885 * and internal map corresponding to '.rodata' ELF section. The final 1886 * map name advertised to user and to the kernel will be 1887 * 'my_objec.rodata', taking first 8 characters of object name and 1888 * entire 7 characters of '.rodata'. 1889 * Somewhat confusingly, if internal map ELF section name is shorter 1890 * than 7 characters, e.g., '.bss', we still reserve 7 characters 1891 * for the suffix, even though we only have 4 actual characters, and 1892 * resulting map will be called 'my_objec.bss', not even using all 15 1893 * characters allowed by the kernel. Oh well, at least the truncated 1894 * object name is somewhat consistent in this case. But if the map 1895 * name is '.kconfig', we'll still have entirety of '.kconfig' added 1896 * (8 chars) and thus will be left with only first 7 characters of the 1897 * object name ('my_obje'). Happy guessing, user, that the final map 1898 * name will be "my_obje.kconfig". 1899 * Now, with libbpf starting to support arbitrarily named .rodata.* 1900 * and .data.* data sections, it's possible that ELF section name is 1901 * longer than allowed 15 chars, so we now need to be careful to take 1902 * only up to 15 first characters of ELF name, taking no BPF object 1903 * name characters at all. So '.rodata.abracadabra' will result in 1904 * '.rodata.abracad' kernel and user-visible name. 1905 * We need to keep this convoluted logic intact for .data, .bss and 1906 * .rodata maps, but for new custom .data.custom and .rodata.custom 1907 * maps we use their ELF names as is, not prepending bpf_object name 1908 * in front. We still need to truncate them to 15 characters for the 1909 * kernel. Full name can be recovered for such maps by using DATASEC 1910 * BTF type associated with such map's value type, though. 1911 */ 1912 if (sfx_len >= BPF_OBJ_NAME_LEN) 1913 sfx_len = BPF_OBJ_NAME_LEN - 1; 1914 1915 /* 1916 * Don't prefix the bpf_object name if this is a custom dot map 1917 * (containing two or more dots) or a percpu data map. 1918 */ 1919 if (strchr(real_name + 1, '.') != NULL || type == LIBBPF_MAP_PERCPU) 1920 pfx_len = 0; 1921 else 1922 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name)); 1923 1924 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, 1925 sfx_len, real_name); 1926 1927 /* sanities map name to characters allowed by kernel */ 1928 for (p = map_name; *p && p < map_name + sizeof(map_name); p++) 1929 if (!isalnum(*p) && *p != '_' && *p != '.') 1930 *p = '_'; 1931 1932 return strdup(map_name); 1933 } 1934 1935 static int 1936 map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map); 1937 1938 /* Internal BPF map is mmap()'able only if at least one of corresponding 1939 * DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL 1940 * variable and it's not marked as __hidden (which turns it into, effectively, 1941 * a STATIC variable). 1942 */ 1943 static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map) 1944 { 1945 const struct btf_type *t, *vt; 1946 struct btf_var_secinfo *vsi; 1947 int i, n; 1948 1949 if (!map->btf_value_type_id) 1950 return false; 1951 1952 /* 1953 * The internal PERCPU maps are not mmapable because the underlying 1954 * percpu_array maps do not have mmap support. 1955 */ 1956 if (map->libbpf_type == LIBBPF_MAP_PERCPU) 1957 return false; 1958 1959 t = btf__type_by_id(obj->btf, map->btf_value_type_id); 1960 if (!btf_is_datasec(t)) 1961 return false; 1962 1963 vsi = btf_var_secinfos(t); 1964 for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) { 1965 vt = btf__type_by_id(obj->btf, vsi->type); 1966 if (!btf_is_var(vt)) 1967 continue; 1968 1969 if (btf_var(vt)->linkage != BTF_VAR_STATIC) 1970 return true; 1971 } 1972 1973 return false; 1974 } 1975 1976 static int 1977 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 1978 const char *real_name, int sec_idx, void *data, size_t data_sz) 1979 { 1980 bool is_percpu = type == LIBBPF_MAP_PERCPU; 1981 struct bpf_map_def *def; 1982 struct bpf_map *map; 1983 size_t mmap_sz; 1984 int err; 1985 1986 map = bpf_object__add_map(obj); 1987 if (IS_ERR(map)) 1988 return PTR_ERR(map); 1989 1990 map->libbpf_type = type; 1991 map->sec_idx = sec_idx; 1992 map->sec_offset = 0; 1993 map->real_name = strdup(real_name); 1994 map->name = internal_map_name(obj, real_name, type); 1995 if (!map->real_name || !map->name) { 1996 zfree(&map->real_name); 1997 zfree(&map->name); 1998 return -ENOMEM; 1999 } 2000 2001 def = &map->def; 2002 def->type = is_percpu ? BPF_MAP_TYPE_PERCPU_ARRAY : BPF_MAP_TYPE_ARRAY; 2003 def->key_size = sizeof(int); 2004 def->value_size = data_sz; 2005 def->max_entries = 1; 2006 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG 2007 ? BPF_F_RDONLY_PROG : 0; 2008 2009 /* failures are fine because of maps like .rodata.str1.1 */ 2010 (void) map_fill_btf_type_info(obj, map); 2011 2012 if (map_is_mmapable(obj, map)) 2013 def->map_flags |= BPF_F_MMAPABLE; 2014 2015 pr_debug("map '%s' (global %sdata): at sec_idx %d, offset %zu, flags %x.\n", 2016 map->name, is_percpu ? "percpu " : "", map->sec_idx, 2017 map->sec_offset, def->map_flags); 2018 2019 mmap_sz = bpf_map_mmap_sz(map); 2020 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 2021 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 2022 if (map->mmaped == MAP_FAILED) { 2023 err = -errno; 2024 map->mmaped = NULL; 2025 pr_warn("failed to alloc map '%s' content buffer: %s\n", map->name, errstr(err)); 2026 zfree(&map->real_name); 2027 zfree(&map->name); 2028 return err; 2029 } 2030 2031 if (data) 2032 memcpy(map->mmaped, data, data_sz); 2033 2034 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 2035 return 0; 2036 } 2037 2038 static int bpf_object__init_global_data_maps(struct bpf_object *obj) 2039 { 2040 struct elf_sec_desc *sec_desc; 2041 const char *sec_name; 2042 int err = 0, sec_idx; 2043 2044 /* 2045 * Populate obj->maps with libbpf internal maps. 2046 */ 2047 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) { 2048 sec_desc = &obj->efile.secs[sec_idx]; 2049 2050 /* Skip recognized sections with size 0. */ 2051 if (!sec_desc->data || sec_desc->data->d_size == 0) 2052 continue; 2053 2054 switch (sec_desc->sec_type) { 2055 case SEC_DATA: 2056 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 2057 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 2058 sec_name, sec_idx, 2059 sec_desc->data->d_buf, 2060 sec_desc->data->d_size); 2061 break; 2062 case SEC_RODATA: 2063 obj->has_rodata = true; 2064 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 2065 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 2066 sec_name, sec_idx, 2067 sec_desc->data->d_buf, 2068 sec_desc->data->d_size); 2069 break; 2070 case SEC_BSS: 2071 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 2072 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 2073 sec_name, sec_idx, 2074 NULL, 2075 sec_desc->data->d_size); 2076 break; 2077 case SEC_PERCPU: 2078 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 2079 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_PERCPU, 2080 sec_name, sec_idx, 2081 sec_desc->data->d_buf, 2082 sec_desc->data->d_size); 2083 break; 2084 default: 2085 /* skip */ 2086 break; 2087 } 2088 if (err) 2089 return err; 2090 } 2091 return 0; 2092 } 2093 2094 2095 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, 2096 const void *name) 2097 { 2098 int i; 2099 2100 for (i = 0; i < obj->nr_extern; i++) { 2101 if (strcmp(obj->externs[i].name, name) == 0) 2102 return &obj->externs[i]; 2103 } 2104 return NULL; 2105 } 2106 2107 static struct extern_desc *find_extern_by_name_with_len(const struct bpf_object *obj, 2108 const void *name, int len) 2109 { 2110 const char *ext_name; 2111 int i; 2112 2113 for (i = 0; i < obj->nr_extern; i++) { 2114 ext_name = obj->externs[i].name; 2115 if (strlen(ext_name) == len && strncmp(ext_name, name, len) == 0) 2116 return &obj->externs[i]; 2117 } 2118 return NULL; 2119 } 2120 2121 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, 2122 char value) 2123 { 2124 switch (ext->kcfg.type) { 2125 case KCFG_BOOL: 2126 if (value == 'm') { 2127 pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n", 2128 ext->name, value); 2129 return -EINVAL; 2130 } 2131 *(bool *)ext_val = value == 'y' ? true : false; 2132 break; 2133 case KCFG_TRISTATE: 2134 if (value == 'y') 2135 *(enum libbpf_tristate *)ext_val = TRI_YES; 2136 else if (value == 'm') 2137 *(enum libbpf_tristate *)ext_val = TRI_MODULE; 2138 else /* value == 'n' */ 2139 *(enum libbpf_tristate *)ext_val = TRI_NO; 2140 break; 2141 case KCFG_CHAR: 2142 *(char *)ext_val = value; 2143 break; 2144 case KCFG_UNKNOWN: 2145 case KCFG_INT: 2146 case KCFG_CHAR_ARR: 2147 default: 2148 pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n", 2149 ext->name, value); 2150 return -EINVAL; 2151 } 2152 ext->is_set = true; 2153 return 0; 2154 } 2155 2156 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, 2157 const char *value) 2158 { 2159 size_t len; 2160 2161 if (ext->kcfg.type != KCFG_CHAR_ARR) { 2162 pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n", 2163 ext->name, value); 2164 return -EINVAL; 2165 } 2166 2167 len = strlen(value); 2168 if (len < 2 || value[len - 1] != '"') { 2169 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n", 2170 ext->name, value); 2171 return -EINVAL; 2172 } 2173 2174 /* strip quotes */ 2175 len -= 2; 2176 if (len >= ext->kcfg.sz) { 2177 pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n", 2178 ext->name, value, len, ext->kcfg.sz - 1); 2179 len = ext->kcfg.sz - 1; 2180 } 2181 memcpy(ext_val, value + 1, len); 2182 ext_val[len] = '\0'; 2183 ext->is_set = true; 2184 return 0; 2185 } 2186 2187 static int parse_u64(const char *value, __u64 *res) 2188 { 2189 char *value_end; 2190 int err; 2191 2192 errno = 0; 2193 *res = strtoull(value, &value_end, 0); 2194 if (errno) { 2195 err = -errno; 2196 pr_warn("failed to parse '%s': %s\n", value, errstr(err)); 2197 return err; 2198 } 2199 if (*value_end) { 2200 pr_warn("failed to parse '%s' as integer completely\n", value); 2201 return -EINVAL; 2202 } 2203 return 0; 2204 } 2205 2206 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) 2207 { 2208 int bit_sz = ext->kcfg.sz * 8; 2209 2210 if (ext->kcfg.sz == 8) 2211 return true; 2212 2213 /* Validate that value stored in u64 fits in integer of `ext->sz` 2214 * bytes size without any loss of information. If the target integer 2215 * is signed, we rely on the following limits of integer type of 2216 * Y bits and subsequent transformation: 2217 * 2218 * -2^(Y-1) <= X <= 2^(Y-1) - 1 2219 * 0 <= X + 2^(Y-1) <= 2^Y - 1 2220 * 0 <= X + 2^(Y-1) < 2^Y 2221 * 2222 * For unsigned target integer, check that all the (64 - Y) bits are 2223 * zero. 2224 */ 2225 if (ext->kcfg.is_signed) 2226 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); 2227 else 2228 return (v >> bit_sz) == 0; 2229 } 2230 2231 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, 2232 __u64 value) 2233 { 2234 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR && 2235 ext->kcfg.type != KCFG_BOOL) { 2236 pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n", 2237 ext->name, (unsigned long long)value); 2238 return -EINVAL; 2239 } 2240 if (ext->kcfg.type == KCFG_BOOL && value > 1) { 2241 pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n", 2242 ext->name, (unsigned long long)value); 2243 return -EINVAL; 2244 2245 } 2246 if (!is_kcfg_value_in_range(ext, value)) { 2247 pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n", 2248 ext->name, (unsigned long long)value, ext->kcfg.sz); 2249 return -ERANGE; 2250 } 2251 switch (ext->kcfg.sz) { 2252 case 1: 2253 *(__u8 *)ext_val = value; 2254 break; 2255 case 2: 2256 *(__u16 *)ext_val = value; 2257 break; 2258 case 4: 2259 *(__u32 *)ext_val = value; 2260 break; 2261 case 8: 2262 *(__u64 *)ext_val = value; 2263 break; 2264 default: 2265 return -EINVAL; 2266 } 2267 ext->is_set = true; 2268 return 0; 2269 } 2270 2271 static int bpf_object__process_kconfig_line(struct bpf_object *obj, 2272 char *buf, void *data) 2273 { 2274 struct extern_desc *ext; 2275 char *sep, *value; 2276 int len, err = 0; 2277 void *ext_val; 2278 __u64 num; 2279 2280 if (!str_has_pfx(buf, "CONFIG_")) 2281 return 0; 2282 2283 sep = strchr(buf, '='); 2284 if (!sep) { 2285 pr_warn("failed to parse '%s': no separator\n", buf); 2286 return -EINVAL; 2287 } 2288 2289 /* Trim ending '\n' */ 2290 len = strlen(buf); 2291 if (buf[len - 1] == '\n') 2292 buf[len - 1] = '\0'; 2293 /* Split on '=' and ensure that a value is present. */ 2294 *sep = '\0'; 2295 if (!sep[1]) { 2296 *sep = '='; 2297 pr_warn("failed to parse '%s': no value\n", buf); 2298 return -EINVAL; 2299 } 2300 2301 ext = find_extern_by_name(obj, buf); 2302 if (!ext || ext->is_set) 2303 return 0; 2304 2305 ext_val = data + ext->kcfg.data_off; 2306 value = sep + 1; 2307 2308 switch (*value) { 2309 case 'y': case 'n': case 'm': 2310 err = set_kcfg_value_tri(ext, ext_val, *value); 2311 break; 2312 case '"': 2313 err = set_kcfg_value_str(ext, ext_val, value); 2314 break; 2315 default: 2316 /* assume integer */ 2317 err = parse_u64(value, &num); 2318 if (err) { 2319 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value); 2320 return err; 2321 } 2322 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { 2323 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value); 2324 return -EINVAL; 2325 } 2326 err = set_kcfg_value_num(ext, ext_val, num); 2327 break; 2328 } 2329 if (err) 2330 return err; 2331 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value); 2332 return 0; 2333 } 2334 2335 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) 2336 { 2337 char buf[PATH_MAX]; 2338 struct utsname uts; 2339 int len, err = 0; 2340 gzFile file; 2341 2342 uname(&uts); 2343 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); 2344 if (len < 0) 2345 return -EINVAL; 2346 else if (len >= PATH_MAX) 2347 return -ENAMETOOLONG; 2348 2349 /* gzopen also accepts uncompressed files. */ 2350 file = gzopen(buf, "re"); 2351 if (!file) 2352 file = gzopen("/proc/config.gz", "re"); 2353 2354 if (!file) { 2355 pr_warn("failed to open system Kconfig\n"); 2356 return -ENOENT; 2357 } 2358 2359 while (gzgets(file, buf, sizeof(buf))) { 2360 err = bpf_object__process_kconfig_line(obj, buf, data); 2361 if (err) { 2362 pr_warn("error parsing system Kconfig line '%s': %s\n", 2363 buf, errstr(err)); 2364 goto out; 2365 } 2366 } 2367 2368 out: 2369 gzclose(file); 2370 return err; 2371 } 2372 2373 static int bpf_object__read_kconfig_mem(struct bpf_object *obj, 2374 const char *config, void *data) 2375 { 2376 char buf[PATH_MAX]; 2377 int err = 0; 2378 FILE *file; 2379 2380 file = fmemopen((void *)config, strlen(config), "r"); 2381 if (!file) { 2382 err = -errno; 2383 pr_warn("failed to open in-memory Kconfig: %s\n", errstr(err)); 2384 return err; 2385 } 2386 2387 while (fgets(buf, sizeof(buf), file)) { 2388 err = bpf_object__process_kconfig_line(obj, buf, data); 2389 if (err) { 2390 pr_warn("error parsing in-memory Kconfig line '%s': %s\n", 2391 buf, errstr(err)); 2392 break; 2393 } 2394 } 2395 2396 fclose(file); 2397 return err; 2398 } 2399 2400 static int bpf_object__init_kconfig_map(struct bpf_object *obj) 2401 { 2402 struct extern_desc *last_ext = NULL, *ext; 2403 size_t map_sz; 2404 int i, err; 2405 2406 for (i = 0; i < obj->nr_extern; i++) { 2407 ext = &obj->externs[i]; 2408 if (ext->type == EXT_KCFG) 2409 last_ext = ext; 2410 } 2411 2412 if (!last_ext) 2413 return 0; 2414 2415 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; 2416 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, 2417 ".kconfig", obj->efile.symbols_shndx, 2418 NULL, map_sz); 2419 if (err) 2420 return err; 2421 2422 obj->kconfig_map_idx = obj->nr_maps - 1; 2423 2424 return 0; 2425 } 2426 2427 const struct btf_type * 2428 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) 2429 { 2430 const struct btf_type *t = btf__type_by_id(btf, id); 2431 2432 if (res_id) 2433 *res_id = id; 2434 2435 while (btf_is_mod(t) || btf_is_typedef(t)) { 2436 if (res_id) 2437 *res_id = t->type; 2438 t = btf__type_by_id(btf, t->type); 2439 } 2440 2441 return t; 2442 } 2443 2444 static const struct btf_type * 2445 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) 2446 { 2447 const struct btf_type *t; 2448 2449 t = skip_mods_and_typedefs(btf, id, NULL); 2450 if (!btf_is_ptr(t)) 2451 return NULL; 2452 2453 t = skip_mods_and_typedefs(btf, t->type, res_id); 2454 2455 return btf_is_func_proto(t) ? t : NULL; 2456 } 2457 2458 static const char *__btf_kind_str(__u16 kind) 2459 { 2460 switch (kind) { 2461 case BTF_KIND_UNKN: return "void"; 2462 case BTF_KIND_INT: return "int"; 2463 case BTF_KIND_PTR: return "ptr"; 2464 case BTF_KIND_ARRAY: return "array"; 2465 case BTF_KIND_STRUCT: return "struct"; 2466 case BTF_KIND_UNION: return "union"; 2467 case BTF_KIND_ENUM: return "enum"; 2468 case BTF_KIND_FWD: return "fwd"; 2469 case BTF_KIND_TYPEDEF: return "typedef"; 2470 case BTF_KIND_VOLATILE: return "volatile"; 2471 case BTF_KIND_CONST: return "const"; 2472 case BTF_KIND_RESTRICT: return "restrict"; 2473 case BTF_KIND_FUNC: return "func"; 2474 case BTF_KIND_FUNC_PROTO: return "func_proto"; 2475 case BTF_KIND_VAR: return "var"; 2476 case BTF_KIND_DATASEC: return "datasec"; 2477 case BTF_KIND_FLOAT: return "float"; 2478 case BTF_KIND_DECL_TAG: return "decl_tag"; 2479 case BTF_KIND_TYPE_TAG: return "type_tag"; 2480 case BTF_KIND_ENUM64: return "enum64"; 2481 default: return "unknown"; 2482 } 2483 } 2484 2485 const char *btf_kind_str(const struct btf_type *t) 2486 { 2487 return __btf_kind_str(btf_kind(t)); 2488 } 2489 2490 /* 2491 * Fetch integer attribute of BTF map definition. Such attributes are 2492 * represented using a pointer to an array, in which dimensionality of array 2493 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 2494 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 2495 * type definition, while using only sizeof(void *) space in ELF data section. 2496 */ 2497 static bool get_map_field_int(const char *map_name, const struct btf *btf, 2498 const struct btf_member *m, __u32 *res) 2499 { 2500 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); 2501 const char *name = btf__name_by_offset(btf, m->name_off); 2502 const struct btf_array *arr_info; 2503 const struct btf_type *arr_t; 2504 2505 if (!btf_is_ptr(t)) { 2506 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n", 2507 map_name, name, btf_kind_str(t)); 2508 return false; 2509 } 2510 2511 arr_t = btf__type_by_id(btf, t->type); 2512 if (!arr_t) { 2513 pr_warn("map '%s': attr '%s': type [%u] not found.\n", 2514 map_name, name, t->type); 2515 return false; 2516 } 2517 if (!btf_is_array(arr_t)) { 2518 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n", 2519 map_name, name, btf_kind_str(arr_t)); 2520 return false; 2521 } 2522 arr_info = btf_array(arr_t); 2523 *res = arr_info->nelems; 2524 return true; 2525 } 2526 2527 static bool get_map_field_long(const char *map_name, const struct btf *btf, 2528 const struct btf_member *m, __u64 *res) 2529 { 2530 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); 2531 const char *name = btf__name_by_offset(btf, m->name_off); 2532 2533 if (btf_is_ptr(t)) { 2534 __u32 res32; 2535 bool ret; 2536 2537 ret = get_map_field_int(map_name, btf, m, &res32); 2538 if (ret) 2539 *res = (__u64)res32; 2540 return ret; 2541 } 2542 2543 if (!btf_is_enum(t) && !btf_is_enum64(t)) { 2544 pr_warn("map '%s': attr '%s': expected ENUM or ENUM64, got %s.\n", 2545 map_name, name, btf_kind_str(t)); 2546 return false; 2547 } 2548 2549 if (btf_vlen(t) != 1) { 2550 pr_warn("map '%s': attr '%s': invalid __ulong\n", 2551 map_name, name); 2552 return false; 2553 } 2554 2555 if (btf_is_enum(t)) { 2556 const struct btf_enum *e = btf_enum(t); 2557 2558 *res = e->val; 2559 } else { 2560 const struct btf_enum64 *e = btf_enum64(t); 2561 2562 *res = btf_enum64_value(e); 2563 } 2564 return true; 2565 } 2566 2567 static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name) 2568 { 2569 int len; 2570 2571 len = snprintf(buf, buf_sz, "%s/%s", path, name); 2572 if (len < 0) 2573 return -EINVAL; 2574 if (len >= buf_sz) 2575 return -ENAMETOOLONG; 2576 2577 return 0; 2578 } 2579 2580 static int build_map_pin_path(struct bpf_map *map, const char *path) 2581 { 2582 char buf[PATH_MAX]; 2583 int err; 2584 2585 if (!path) 2586 path = BPF_FS_DEFAULT_PATH; 2587 2588 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 2589 if (err) 2590 return err; 2591 2592 return bpf_map__set_pin_path(map, buf); 2593 } 2594 2595 /* should match definition in bpf_helpers.h */ 2596 enum libbpf_pin_type { 2597 LIBBPF_PIN_NONE, 2598 /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ 2599 LIBBPF_PIN_BY_NAME, 2600 }; 2601 2602 int parse_btf_map_def(const char *map_name, struct btf *btf, 2603 const struct btf_type *def_t, bool strict, 2604 struct btf_map_def *map_def, struct btf_map_def *inner_def) 2605 { 2606 const struct btf_type *t; 2607 const struct btf_member *m; 2608 bool is_inner = inner_def == NULL; 2609 int vlen, i; 2610 2611 vlen = btf_vlen(def_t); 2612 m = btf_members(def_t); 2613 for (i = 0; i < vlen; i++, m++) { 2614 const char *name = btf__name_by_offset(btf, m->name_off); 2615 2616 if (!name) { 2617 pr_warn("map '%s': invalid field #%d.\n", map_name, i); 2618 return -EINVAL; 2619 } 2620 if (strcmp(name, "type") == 0) { 2621 if (!get_map_field_int(map_name, btf, m, &map_def->map_type)) 2622 return -EINVAL; 2623 map_def->parts |= MAP_DEF_MAP_TYPE; 2624 } else if (strcmp(name, "max_entries") == 0) { 2625 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries)) 2626 return -EINVAL; 2627 map_def->parts |= MAP_DEF_MAX_ENTRIES; 2628 } else if (strcmp(name, "map_flags") == 0) { 2629 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags)) 2630 return -EINVAL; 2631 map_def->parts |= MAP_DEF_MAP_FLAGS; 2632 } else if (strcmp(name, "numa_node") == 0) { 2633 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node)) 2634 return -EINVAL; 2635 map_def->parts |= MAP_DEF_NUMA_NODE; 2636 } else if (strcmp(name, "key_size") == 0) { 2637 __u32 sz; 2638 2639 if (!get_map_field_int(map_name, btf, m, &sz)) 2640 return -EINVAL; 2641 if (map_def->key_size && map_def->key_size != sz) { 2642 pr_warn("map '%s': conflicting key size %u != %u.\n", 2643 map_name, map_def->key_size, sz); 2644 return -EINVAL; 2645 } 2646 map_def->key_size = sz; 2647 map_def->parts |= MAP_DEF_KEY_SIZE; 2648 } else if (strcmp(name, "key") == 0) { 2649 __s64 sz; 2650 2651 t = btf__type_by_id(btf, m->type); 2652 if (!t) { 2653 pr_warn("map '%s': key type [%u] not found.\n", 2654 map_name, m->type); 2655 return -EINVAL; 2656 } 2657 if (!btf_is_ptr(t)) { 2658 pr_warn("map '%s': key spec is not PTR: %s.\n", 2659 map_name, btf_kind_str(t)); 2660 return -EINVAL; 2661 } 2662 sz = btf__resolve_size(btf, t->type); 2663 if (sz < 0) { 2664 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", 2665 map_name, t->type, (ssize_t)sz); 2666 return sz; 2667 } 2668 if (map_def->key_size && map_def->key_size != sz) { 2669 pr_warn("map '%s': conflicting key size %u != %zd.\n", 2670 map_name, map_def->key_size, (ssize_t)sz); 2671 return -EINVAL; 2672 } 2673 map_def->key_size = sz; 2674 map_def->key_type_id = t->type; 2675 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE; 2676 } else if (strcmp(name, "value_size") == 0) { 2677 __u32 sz; 2678 2679 if (!get_map_field_int(map_name, btf, m, &sz)) 2680 return -EINVAL; 2681 if (map_def->value_size && map_def->value_size != sz) { 2682 pr_warn("map '%s': conflicting value size %u != %u.\n", 2683 map_name, map_def->value_size, sz); 2684 return -EINVAL; 2685 } 2686 map_def->value_size = sz; 2687 map_def->parts |= MAP_DEF_VALUE_SIZE; 2688 } else if (strcmp(name, "value") == 0) { 2689 __s64 sz; 2690 2691 t = btf__type_by_id(btf, m->type); 2692 if (!t) { 2693 pr_warn("map '%s': value type [%u] not found.\n", 2694 map_name, m->type); 2695 return -EINVAL; 2696 } 2697 if (!btf_is_ptr(t)) { 2698 pr_warn("map '%s': value spec is not PTR: %s.\n", 2699 map_name, btf_kind_str(t)); 2700 return -EINVAL; 2701 } 2702 sz = btf__resolve_size(btf, t->type); 2703 if (sz < 0) { 2704 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", 2705 map_name, t->type, (ssize_t)sz); 2706 return sz; 2707 } 2708 if (map_def->value_size && map_def->value_size != sz) { 2709 pr_warn("map '%s': conflicting value size %u != %zd.\n", 2710 map_name, map_def->value_size, (ssize_t)sz); 2711 return -EINVAL; 2712 } 2713 map_def->value_size = sz; 2714 map_def->value_type_id = t->type; 2715 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE; 2716 } 2717 else if (strcmp(name, "values") == 0) { 2718 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type); 2719 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY; 2720 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value"; 2721 char inner_map_name[128]; 2722 int err; 2723 2724 if (is_inner) { 2725 pr_warn("map '%s': multi-level inner maps not supported.\n", 2726 map_name); 2727 return -ENOTSUP; 2728 } 2729 if (i != vlen - 1) { 2730 pr_warn("map '%s': '%s' member should be last.\n", 2731 map_name, name); 2732 return -EINVAL; 2733 } 2734 if (!is_map_in_map && !is_prog_array) { 2735 pr_warn("map '%s': should be map-in-map or prog-array.\n", 2736 map_name); 2737 return -ENOTSUP; 2738 } 2739 if (map_def->value_size && map_def->value_size != 4) { 2740 pr_warn("map '%s': conflicting value size %u != 4.\n", 2741 map_name, map_def->value_size); 2742 return -EINVAL; 2743 } 2744 map_def->value_size = 4; 2745 t = btf__type_by_id(btf, m->type); 2746 if (!t) { 2747 pr_warn("map '%s': %s type [%u] not found.\n", 2748 map_name, desc, m->type); 2749 return -EINVAL; 2750 } 2751 if (!btf_is_array(t) || btf_array(t)->nelems) { 2752 pr_warn("map '%s': %s spec is not a zero-sized array.\n", 2753 map_name, desc); 2754 return -EINVAL; 2755 } 2756 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL); 2757 if (!btf_is_ptr(t)) { 2758 pr_warn("map '%s': %s def is of unexpected kind %s.\n", 2759 map_name, desc, btf_kind_str(t)); 2760 return -EINVAL; 2761 } 2762 t = skip_mods_and_typedefs(btf, t->type, NULL); 2763 if (is_prog_array) { 2764 if (!btf_is_func_proto(t)) { 2765 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n", 2766 map_name, btf_kind_str(t)); 2767 return -EINVAL; 2768 } 2769 continue; 2770 } 2771 if (!btf_is_struct(t)) { 2772 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2773 map_name, btf_kind_str(t)); 2774 return -EINVAL; 2775 } 2776 2777 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name); 2778 err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL); 2779 if (err) 2780 return err; 2781 2782 map_def->parts |= MAP_DEF_INNER_MAP; 2783 } else if (strcmp(name, "pinning") == 0) { 2784 __u32 val; 2785 2786 if (is_inner) { 2787 pr_warn("map '%s': inner def can't be pinned.\n", map_name); 2788 return -EINVAL; 2789 } 2790 if (!get_map_field_int(map_name, btf, m, &val)) 2791 return -EINVAL; 2792 if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) { 2793 pr_warn("map '%s': invalid pinning value %u.\n", 2794 map_name, val); 2795 return -EINVAL; 2796 } 2797 map_def->pinning = val; 2798 map_def->parts |= MAP_DEF_PINNING; 2799 } else if (strcmp(name, "map_extra") == 0) { 2800 __u64 map_extra; 2801 2802 if (!get_map_field_long(map_name, btf, m, &map_extra)) 2803 return -EINVAL; 2804 map_def->map_extra = map_extra; 2805 map_def->parts |= MAP_DEF_MAP_EXTRA; 2806 } else { 2807 if (strict) { 2808 pr_warn("map '%s': unknown field '%s'.\n", map_name, name); 2809 return -ENOTSUP; 2810 } 2811 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name); 2812 } 2813 } 2814 2815 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) { 2816 pr_warn("map '%s': map type isn't specified.\n", map_name); 2817 return -EINVAL; 2818 } 2819 2820 return 0; 2821 } 2822 2823 static size_t adjust_ringbuf_sz(size_t sz) 2824 { 2825 __u32 page_sz = sysconf(_SC_PAGE_SIZE); 2826 __u32 mul; 2827 2828 /* if user forgot to set any size, make sure they see error */ 2829 if (sz == 0) 2830 return 0; 2831 /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be 2832 * a power-of-2 multiple of kernel's page size. If user diligently 2833 * satisfied these conditions, pass the size through. 2834 */ 2835 if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz)) 2836 return sz; 2837 2838 /* Otherwise find closest (page_sz * power_of_2) product bigger than 2839 * user-set size to satisfy both user size request and kernel 2840 * requirements and substitute correct max_entries for map creation. 2841 */ 2842 for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) { 2843 if (mul * page_sz > sz) 2844 return mul * page_sz; 2845 } 2846 2847 /* if it's impossible to satisfy the conditions (i.e., user size is 2848 * very close to UINT_MAX but is not a power-of-2 multiple of 2849 * page_size) then just return original size and let kernel reject it 2850 */ 2851 return sz; 2852 } 2853 2854 static bool map_is_ringbuf(const struct bpf_map *map) 2855 { 2856 return map->def.type == BPF_MAP_TYPE_RINGBUF || 2857 map->def.type == BPF_MAP_TYPE_USER_RINGBUF; 2858 } 2859 2860 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) 2861 { 2862 map->def.type = def->map_type; 2863 map->def.key_size = def->key_size; 2864 map->def.value_size = def->value_size; 2865 map->def.max_entries = def->max_entries; 2866 map->def.map_flags = def->map_flags; 2867 map->map_extra = def->map_extra; 2868 2869 map->numa_node = def->numa_node; 2870 map->btf_key_type_id = def->key_type_id; 2871 map->btf_value_type_id = def->value_type_id; 2872 2873 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ 2874 if (map_is_ringbuf(map)) 2875 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); 2876 2877 if (def->parts & MAP_DEF_MAP_TYPE) 2878 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); 2879 2880 if (def->parts & MAP_DEF_KEY_TYPE) 2881 pr_debug("map '%s': found key [%u], sz = %u.\n", 2882 map->name, def->key_type_id, def->key_size); 2883 else if (def->parts & MAP_DEF_KEY_SIZE) 2884 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size); 2885 2886 if (def->parts & MAP_DEF_VALUE_TYPE) 2887 pr_debug("map '%s': found value [%u], sz = %u.\n", 2888 map->name, def->value_type_id, def->value_size); 2889 else if (def->parts & MAP_DEF_VALUE_SIZE) 2890 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size); 2891 2892 if (def->parts & MAP_DEF_MAX_ENTRIES) 2893 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries); 2894 if (def->parts & MAP_DEF_MAP_FLAGS) 2895 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags); 2896 if (def->parts & MAP_DEF_MAP_EXTRA) 2897 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name, 2898 (unsigned long long)def->map_extra); 2899 if (def->parts & MAP_DEF_PINNING) 2900 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning); 2901 if (def->parts & MAP_DEF_NUMA_NODE) 2902 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node); 2903 2904 if (def->parts & MAP_DEF_INNER_MAP) 2905 pr_debug("map '%s': found inner map definition.\n", map->name); 2906 } 2907 2908 static const char *btf_var_linkage_str(__u32 linkage) 2909 { 2910 switch (linkage) { 2911 case BTF_VAR_STATIC: return "static"; 2912 case BTF_VAR_GLOBAL_ALLOCATED: return "global"; 2913 case BTF_VAR_GLOBAL_EXTERN: return "extern"; 2914 default: return "unknown"; 2915 } 2916 } 2917 2918 static int bpf_object__init_user_btf_map(struct bpf_object *obj, 2919 const struct btf_type *sec, 2920 int var_idx, int sec_idx, 2921 const Elf_Data *data, bool strict, 2922 const char *pin_root_path) 2923 { 2924 struct btf_map_def map_def = {}, inner_def = {}; 2925 const struct btf_type *var, *def; 2926 const struct btf_var_secinfo *vi; 2927 const struct btf_var *var_extra; 2928 const char *map_name; 2929 struct bpf_map *map; 2930 int err; 2931 2932 vi = btf_var_secinfos(sec) + var_idx; 2933 var = btf__type_by_id(obj->btf, vi->type); 2934 var_extra = btf_var(var); 2935 map_name = btf__name_by_offset(obj->btf, var->name_off); 2936 2937 if (str_is_empty(map_name)) { 2938 pr_warn("map #%d: empty name.\n", var_idx); 2939 return -EINVAL; 2940 } 2941 if ((__u64)vi->offset + vi->size > data->d_size) { 2942 pr_warn("map '%s' BTF data is corrupted.\n", map_name); 2943 return -EINVAL; 2944 } 2945 if (!btf_is_var(var)) { 2946 pr_warn("map '%s': unexpected var kind %s.\n", 2947 map_name, btf_kind_str(var)); 2948 return -EINVAL; 2949 } 2950 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) { 2951 pr_warn("map '%s': unsupported map linkage %s.\n", 2952 map_name, btf_var_linkage_str(var_extra->linkage)); 2953 return -EOPNOTSUPP; 2954 } 2955 2956 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 2957 if (!btf_is_struct(def)) { 2958 pr_warn("map '%s': unexpected def kind %s.\n", 2959 map_name, btf_kind_str(var)); 2960 return -EINVAL; 2961 } 2962 if (def->size > vi->size) { 2963 pr_warn("map '%s': invalid def size.\n", map_name); 2964 return -EINVAL; 2965 } 2966 2967 map = bpf_object__add_map(obj); 2968 if (IS_ERR(map)) 2969 return PTR_ERR(map); 2970 map->name = strdup(map_name); 2971 if (!map->name) { 2972 pr_warn("map '%s': failed to alloc map name.\n", map_name); 2973 return -ENOMEM; 2974 } 2975 map->libbpf_type = LIBBPF_MAP_UNSPEC; 2976 map->def.type = BPF_MAP_TYPE_UNSPEC; 2977 map->sec_idx = sec_idx; 2978 map->sec_offset = vi->offset; 2979 map->btf_var_idx = var_idx; 2980 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 2981 map_name, map->sec_idx, map->sec_offset); 2982 2983 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); 2984 if (err) 2985 return err; 2986 2987 fill_map_from_def(map, &map_def); 2988 2989 if (map_def.pinning == LIBBPF_PIN_BY_NAME) { 2990 err = build_map_pin_path(map, pin_root_path); 2991 if (err) { 2992 pr_warn("map '%s': couldn't build pin path.\n", map->name); 2993 return err; 2994 } 2995 } 2996 2997 if (map_def.parts & MAP_DEF_INNER_MAP) { 2998 map->inner_map = calloc(1, sizeof(*map->inner_map)); 2999 if (!map->inner_map) 3000 return -ENOMEM; 3001 map->inner_map->fd = create_placeholder_fd(); 3002 if (map->inner_map->fd < 0) 3003 return map->inner_map->fd; 3004 map->inner_map->sec_idx = sec_idx; 3005 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1); 3006 if (!map->inner_map->name) 3007 return -ENOMEM; 3008 sprintf(map->inner_map->name, "%s.inner", map_name); 3009 3010 fill_map_from_def(map->inner_map, &inner_def); 3011 } 3012 3013 err = map_fill_btf_type_info(obj, map); 3014 if (err) 3015 return err; 3016 3017 return 0; 3018 } 3019 3020 static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map, 3021 const char *sec_name, int sec_idx, 3022 void *data, size_t data_sz) 3023 { 3024 const long page_sz = sysconf(_SC_PAGE_SIZE); 3025 const size_t data_alloc_sz = roundup(data_sz, page_sz); 3026 size_t mmap_sz; 3027 3028 mmap_sz = bpf_map_mmap_sz(map); 3029 if (data_alloc_sz > mmap_sz) { 3030 pr_warn("elf: sec '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n", 3031 sec_name, mmap_sz, data_sz); 3032 return -E2BIG; 3033 } 3034 3035 obj->arena_data = malloc(data_sz); 3036 if (!obj->arena_data) 3037 return -ENOMEM; 3038 memcpy(obj->arena_data, data, data_sz); 3039 obj->arena_data_sz = data_sz; 3040 3041 /* make bpf_map__init_value() work for ARENA maps */ 3042 map->mmaped = obj->arena_data; 3043 3044 return 0; 3045 } 3046 3047 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, 3048 const char *pin_root_path) 3049 { 3050 const struct btf_type *sec = NULL; 3051 int nr_types, i, vlen, err; 3052 const struct btf_type *t; 3053 const char *name; 3054 Elf_Data *data; 3055 Elf_Scn *scn; 3056 3057 if (obj->efile.btf_maps_shndx < 0) 3058 return 0; 3059 3060 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); 3061 data = elf_sec_data(obj, scn); 3062 if (!data) { 3063 pr_warn("elf: failed to get %s map definitions for %s\n", 3064 MAPS_ELF_SEC, obj->path); 3065 return -EINVAL; 3066 } 3067 3068 nr_types = btf__type_cnt(obj->btf); 3069 for (i = 1; i < nr_types; i++) { 3070 t = btf__type_by_id(obj->btf, i); 3071 if (!btf_is_datasec(t)) 3072 continue; 3073 name = btf__name_by_offset(obj->btf, t->name_off); 3074 if (strcmp(name, MAPS_ELF_SEC) == 0) { 3075 sec = t; 3076 obj->efile.btf_maps_sec_btf_id = i; 3077 break; 3078 } 3079 } 3080 3081 if (!sec) { 3082 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 3083 return -ENOENT; 3084 } 3085 3086 vlen = btf_vlen(sec); 3087 for (i = 0; i < vlen; i++) { 3088 err = bpf_object__init_user_btf_map(obj, sec, i, 3089 obj->efile.btf_maps_shndx, 3090 data, strict, 3091 pin_root_path); 3092 if (err) 3093 return err; 3094 } 3095 3096 for (i = 0; i < obj->nr_maps; i++) { 3097 struct bpf_map *map = &obj->maps[i]; 3098 3099 if (map->def.type != BPF_MAP_TYPE_ARENA) 3100 continue; 3101 3102 if (obj->arena_map_idx >= 0) { 3103 pr_warn("map '%s': only single ARENA map is supported (map '%s' is also ARENA)\n", 3104 map->name, obj->maps[obj->arena_map_idx].name); 3105 return -EINVAL; 3106 } 3107 obj->arena_map_idx = i; 3108 3109 if (obj->efile.arena_data) { 3110 err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx, 3111 obj->efile.arena_data->d_buf, 3112 obj->efile.arena_data->d_size); 3113 if (err) 3114 return err; 3115 } 3116 } 3117 if (obj->efile.arena_data && obj->arena_map_idx < 0) { 3118 pr_warn("elf: sec '%s': to use global __arena variables the ARENA map should be explicitly declared in SEC(\".maps\")\n", 3119 ARENA_SEC); 3120 return -ENOENT; 3121 } 3122 3123 return 0; 3124 } 3125 3126 static int bpf_object__init_maps(struct bpf_object *obj, 3127 const struct bpf_object_open_opts *opts) 3128 { 3129 const char *pin_root_path; 3130 bool strict; 3131 int err = 0; 3132 3133 strict = !OPTS_GET(opts, relaxed_maps, false); 3134 pin_root_path = OPTS_GET(opts, pin_root_path, NULL); 3135 3136 err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path); 3137 err = err ?: bpf_object__init_global_data_maps(obj); 3138 err = err ?: bpf_object__init_kconfig_map(obj); 3139 err = err ?: bpf_object_init_struct_ops(obj); 3140 3141 return err; 3142 } 3143 3144 static bool section_have_execinstr(struct bpf_object *obj, int idx) 3145 { 3146 Elf64_Shdr *sh; 3147 3148 sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx)); 3149 if (!sh) 3150 return false; 3151 3152 return sh->sh_flags & SHF_EXECINSTR; 3153 } 3154 3155 static bool starts_with_qmark(const char *s) 3156 { 3157 return s && s[0] == '?'; 3158 } 3159 3160 static bool btf_needs_sanitization(struct bpf_object *obj) 3161 { 3162 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 3163 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 3164 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 3165 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 3166 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); 3167 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); 3168 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); 3169 bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC); 3170 bool has_layout = kernel_supports(obj, FEAT_BTF_LAYOUT); 3171 3172 return !has_func || !has_datasec || !has_func_global || !has_float || 3173 !has_decl_tag || !has_type_tag || !has_enum64 || !has_qmark_datasec || 3174 !has_layout; 3175 } 3176 3177 struct btf *bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *orig_btf) 3178 { 3179 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 3180 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 3181 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 3182 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 3183 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); 3184 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); 3185 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); 3186 bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC); 3187 bool has_layout = kernel_supports(obj, FEAT_BTF_LAYOUT); 3188 int enum64_placeholder_id = 0; 3189 const struct btf_header *hdr; 3190 struct btf *btf = NULL; 3191 const void *raw_data; 3192 struct btf_type *t; 3193 int i, j, vlen; 3194 __u32 sz; 3195 int err; 3196 3197 /* clone BTF to sanitize a copy and leave the original intact */ 3198 raw_data = btf__raw_data(orig_btf, &sz); 3199 if (!raw_data) 3200 return ERR_PTR(-ENOMEM); 3201 /* btf_header() gives us endian-safe header info */ 3202 hdr = btf_header(orig_btf); 3203 3204 if (!has_layout && hdr->hdr_len >= sizeof(struct btf_header) && 3205 (hdr->layout_len != 0 || hdr->layout_off != 0)) { 3206 const struct btf_header *old_hdr = raw_data; 3207 struct btf_header *new_hdr; 3208 void *new_raw_data; 3209 __u32 new_str_off; 3210 3211 /* 3212 * Need to rewrite BTF to exclude layout information and 3213 * move string section to immediately after types. 3214 */ 3215 new_raw_data = malloc(sz); 3216 if (!new_raw_data) 3217 return ERR_PTR(-ENOMEM); 3218 3219 memcpy(new_raw_data, raw_data, sz); 3220 new_hdr = new_raw_data; 3221 new_hdr->layout_off = 0; 3222 new_hdr->layout_len = 0; 3223 new_str_off = hdr->type_off + hdr->type_len; 3224 /* Handle swapped endian case */ 3225 if (old_hdr->magic != hdr->magic) 3226 new_hdr->str_off = bswap_32(new_str_off); 3227 else 3228 new_hdr->str_off = new_str_off; 3229 3230 memmove(new_raw_data + hdr->hdr_len + new_str_off, 3231 new_raw_data + hdr->hdr_len + hdr->str_off, 3232 hdr->str_len); 3233 sz = hdr->hdr_len + hdr->type_off + hdr->type_len + hdr->str_len; 3234 btf = btf__new(new_raw_data, sz); 3235 free(new_raw_data); 3236 } else { 3237 btf = btf__new(raw_data, sz); 3238 } 3239 err = libbpf_get_error(btf); 3240 if (err) 3241 return ERR_PTR(err); 3242 3243 /* enforce 8-byte pointers for BPF-targeted BTFs */ 3244 btf__set_pointer_size(btf, 8); 3245 3246 for (i = 1; i < btf__type_cnt(btf); i++) { 3247 t = (struct btf_type *)btf__type_by_id(btf, i); 3248 3249 if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) { 3250 /* replace VAR/DECL_TAG with INT */ 3251 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 3252 /* 3253 * using size = 1 is the safest choice, 4 will be too 3254 * big and cause kernel BTF validation failure if 3255 * original variable took less than 4 bytes 3256 */ 3257 t->size = 1; 3258 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); 3259 } else if (!has_datasec && btf_is_datasec(t)) { 3260 /* replace DATASEC with STRUCT */ 3261 const struct btf_var_secinfo *v = btf_var_secinfos(t); 3262 struct btf_member *m = btf_members(t); 3263 struct btf_type *vt; 3264 char *name; 3265 3266 name = (char *)btf__name_by_offset(btf, t->name_off); 3267 while (*name) { 3268 if (*name == '.' || *name == '?') 3269 *name = '_'; 3270 name++; 3271 } 3272 3273 vlen = btf_vlen(t); 3274 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 3275 for (j = 0; j < vlen; j++, v++, m++) { 3276 /* order of field assignments is important */ 3277 m->offset = v->offset * 8; 3278 m->type = v->type; 3279 /* preserve variable name as member name */ 3280 vt = (void *)btf__type_by_id(btf, v->type); 3281 m->name_off = vt->name_off; 3282 } 3283 } else if (!has_qmark_datasec && btf_is_datasec(t) && 3284 starts_with_qmark(btf__name_by_offset(btf, t->name_off))) { 3285 /* replace '?' prefix with '_' for DATASEC names */ 3286 char *name; 3287 3288 name = (char *)btf__name_by_offset(btf, t->name_off); 3289 if (name[0] == '?') 3290 name[0] = '_'; 3291 } else if (!has_func && btf_is_func_proto(t)) { 3292 /* replace FUNC_PROTO with ENUM */ 3293 vlen = btf_vlen(t); 3294 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 3295 t->size = sizeof(__u32); /* kernel enforced */ 3296 } else if (!has_func && btf_is_func(t)) { 3297 /* replace FUNC with TYPEDEF */ 3298 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 3299 } else if (!has_func_global && btf_is_func(t)) { 3300 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ 3301 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); 3302 } else if (!has_float && btf_is_float(t)) { 3303 /* replace FLOAT with an equally-sized empty STRUCT; 3304 * since C compilers do not accept e.g. "float" as a 3305 * valid struct name, make it anonymous 3306 */ 3307 t->name_off = 0; 3308 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); 3309 } else if (!has_type_tag && btf_is_type_tag(t)) { 3310 /* replace TYPE_TAG with a CONST */ 3311 t->name_off = 0; 3312 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0); 3313 } else if (!has_enum64 && btf_is_enum(t)) { 3314 /* clear the kflag */ 3315 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false); 3316 } else if (!has_enum64 && btf_is_enum64(t)) { 3317 /* replace ENUM64 with a union */ 3318 struct btf_member *m; 3319 3320 if (enum64_placeholder_id == 0) { 3321 enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0); 3322 if (enum64_placeholder_id < 0) { 3323 btf__free(btf); 3324 return ERR_PTR(enum64_placeholder_id); 3325 } 3326 t = (struct btf_type *)btf__type_by_id(btf, i); 3327 } 3328 3329 m = btf_members(t); 3330 vlen = btf_vlen(t); 3331 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen); 3332 for (j = 0; j < vlen; j++, m++) { 3333 m->type = enum64_placeholder_id; 3334 m->offset = 0; 3335 } 3336 } 3337 } 3338 3339 return btf; 3340 } 3341 3342 static bool libbpf_needs_btf(const struct bpf_object *obj) 3343 { 3344 return obj->efile.btf_maps_shndx >= 0 || 3345 obj->efile.has_st_ops || 3346 obj->nr_extern > 0; 3347 } 3348 3349 static bool kernel_needs_btf(const struct bpf_object *obj) 3350 { 3351 return obj->efile.has_st_ops; 3352 } 3353 3354 static int bpf_object__init_btf(struct bpf_object *obj, 3355 Elf_Data *btf_data, 3356 Elf_Data *btf_ext_data) 3357 { 3358 int err = -ENOENT; 3359 3360 if (btf_data) { 3361 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 3362 err = libbpf_get_error(obj->btf); 3363 if (err) { 3364 obj->btf = NULL; 3365 pr_warn("Error loading ELF section %s: %s.\n", BTF_ELF_SEC, errstr(err)); 3366 goto out; 3367 } 3368 /* enforce 8-byte pointers for BPF-targeted BTFs */ 3369 btf__set_pointer_size(obj->btf, 8); 3370 } 3371 if (btf_ext_data) { 3372 struct btf_ext_info *ext_segs[3]; 3373 int seg_num, sec_num; 3374 3375 if (!obj->btf) { 3376 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 3377 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 3378 goto out; 3379 } 3380 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); 3381 err = libbpf_get_error(obj->btf_ext); 3382 if (err) { 3383 pr_warn("Error loading ELF section %s: %s. Ignored and continue.\n", 3384 BTF_EXT_ELF_SEC, errstr(err)); 3385 obj->btf_ext = NULL; 3386 goto out; 3387 } 3388 3389 /* setup .BTF.ext to ELF section mapping */ 3390 ext_segs[0] = &obj->btf_ext->func_info; 3391 ext_segs[1] = &obj->btf_ext->line_info; 3392 ext_segs[2] = &obj->btf_ext->core_relo_info; 3393 for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) { 3394 struct btf_ext_info *seg = ext_segs[seg_num]; 3395 const struct btf_ext_info_sec *sec; 3396 const char *sec_name; 3397 Elf_Scn *scn; 3398 3399 if (seg->sec_cnt == 0) 3400 continue; 3401 3402 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs)); 3403 if (!seg->sec_idxs) { 3404 err = -ENOMEM; 3405 goto out; 3406 } 3407 3408 sec_num = 0; 3409 for_each_btf_ext_sec(seg, sec) { 3410 /* preventively increment index to avoid doing 3411 * this before every continue below 3412 */ 3413 sec_num++; 3414 3415 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 3416 if (str_is_empty(sec_name)) 3417 continue; 3418 scn = elf_sec_by_name(obj, sec_name); 3419 if (!scn) 3420 continue; 3421 3422 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn); 3423 } 3424 } 3425 } 3426 out: 3427 if (err && libbpf_needs_btf(obj)) { 3428 pr_warn("BTF is required, but is missing or corrupted.\n"); 3429 return err; 3430 } 3431 return 0; 3432 } 3433 3434 static int compare_vsi_off(const void *_a, const void *_b) 3435 { 3436 const struct btf_var_secinfo *a = _a; 3437 const struct btf_var_secinfo *b = _b; 3438 3439 return a->offset - b->offset; 3440 } 3441 3442 static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, 3443 struct btf_type *t) 3444 { 3445 __u32 size = 0, i, vars = btf_vlen(t); 3446 const char *sec_name = btf__name_by_offset(btf, t->name_off); 3447 struct btf_var_secinfo *vsi; 3448 bool fixup_offsets = false; 3449 int err; 3450 3451 if (!sec_name) { 3452 pr_debug("No name found in string section for DATASEC kind.\n"); 3453 return -ENOENT; 3454 } 3455 3456 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and 3457 * variable offsets set at the previous step. Further, not every 3458 * extern BTF VAR has corresponding ELF symbol preserved, so we skip 3459 * all fixups altogether for such sections and go straight to sorting 3460 * VARs within their DATASEC. 3461 */ 3462 if (strcmp(sec_name, KCONFIG_SEC) == 0 || strcmp(sec_name, KSYMS_SEC) == 0) 3463 goto sort_vars; 3464 3465 /* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to 3466 * fix this up. But BPF static linker already fixes this up and fills 3467 * all the sizes and offsets during static linking. So this step has 3468 * to be optional. But the STV_HIDDEN handling is non-optional for any 3469 * non-extern DATASEC, so the variable fixup loop below handles both 3470 * functions at the same time, paying the cost of BTF VAR <-> ELF 3471 * symbol matching just once. 3472 */ 3473 if (t->size == 0) { 3474 err = find_elf_sec_sz(obj, sec_name, &size); 3475 if (err || !size) { 3476 pr_debug("sec '%s': failed to determine size from ELF: size %u, err %s\n", 3477 sec_name, size, errstr(err)); 3478 return -ENOENT; 3479 } 3480 3481 t->size = size; 3482 fixup_offsets = true; 3483 } 3484 3485 for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) { 3486 const struct btf_type *t_var; 3487 struct btf_var *var; 3488 const char *var_name; 3489 Elf64_Sym *sym; 3490 3491 t_var = btf__type_by_id(btf, vsi->type); 3492 if (!t_var || !btf_is_var(t_var)) { 3493 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name); 3494 return -EINVAL; 3495 } 3496 3497 var = btf_var(t_var); 3498 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN) 3499 continue; 3500 3501 var_name = btf__name_by_offset(btf, t_var->name_off); 3502 if (!var_name) { 3503 pr_debug("sec '%s': failed to find name of DATASEC's member #%u\n", 3504 sec_name, i); 3505 return -ENOENT; 3506 } 3507 3508 sym = find_elf_var_sym(obj, var_name); 3509 if (IS_ERR(sym)) { 3510 pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n", 3511 sec_name, var_name); 3512 return -ENOENT; 3513 } 3514 3515 if (fixup_offsets) 3516 vsi->offset = sym->st_value; 3517 3518 /* if variable is a global/weak symbol, but has restricted 3519 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR 3520 * as static. This follows similar logic for functions (BPF 3521 * subprogs) and influences libbpf's further decisions about 3522 * whether to make global data BPF array maps as 3523 * BPF_F_MMAPABLE. 3524 */ 3525 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN 3526 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL) 3527 var->linkage = BTF_VAR_STATIC; 3528 } 3529 3530 sort_vars: 3531 qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off); 3532 return 0; 3533 } 3534 3535 static int bpf_object_fixup_btf(struct bpf_object *obj) 3536 { 3537 int i, n, err = 0; 3538 3539 if (!obj->btf) 3540 return 0; 3541 3542 n = btf__type_cnt(obj->btf); 3543 for (i = 1; i < n; i++) { 3544 struct btf_type *t = btf_type_by_id(obj->btf, i); 3545 3546 /* Loader needs to fix up some of the things compiler 3547 * couldn't get its hands on while emitting BTF. This 3548 * is section size and global variable offset. We use 3549 * the info from the ELF itself for this purpose. 3550 */ 3551 if (btf_is_datasec(t)) { 3552 err = btf_fixup_datasec(obj, obj->btf, t); 3553 if (err) 3554 return err; 3555 } 3556 } 3557 3558 return 0; 3559 } 3560 3561 static bool prog_needs_vmlinux_btf(struct bpf_program *prog) 3562 { 3563 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 3564 prog->type == BPF_PROG_TYPE_LSM) 3565 return true; 3566 3567 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs 3568 * also need vmlinux BTF 3569 */ 3570 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) 3571 return true; 3572 3573 return false; 3574 } 3575 3576 static bool map_needs_vmlinux_btf(struct bpf_map *map) 3577 { 3578 return bpf_map__is_struct_ops(map); 3579 } 3580 3581 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) 3582 { 3583 struct bpf_program *prog; 3584 struct bpf_map *map; 3585 int i; 3586 3587 /* CO-RE relocations need kernel BTF, only when btf_custom_path 3588 * is not specified 3589 */ 3590 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path) 3591 return true; 3592 3593 /* Support for typed ksyms needs kernel BTF */ 3594 for (i = 0; i < obj->nr_extern; i++) { 3595 const struct extern_desc *ext; 3596 3597 ext = &obj->externs[i]; 3598 if (ext->type == EXT_KSYM && ext->ksym.type_id) 3599 return true; 3600 } 3601 3602 bpf_object__for_each_program(prog, obj) { 3603 if (!prog->autoload) 3604 continue; 3605 if (prog_needs_vmlinux_btf(prog)) 3606 return true; 3607 } 3608 3609 bpf_object__for_each_map(map, obj) { 3610 if (map_needs_vmlinux_btf(map)) 3611 return true; 3612 } 3613 3614 return false; 3615 } 3616 3617 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) 3618 { 3619 int err; 3620 3621 /* btf_vmlinux could be loaded earlier */ 3622 if (obj->btf_vmlinux || obj->gen_loader) 3623 return 0; 3624 3625 if (!force && !obj_needs_vmlinux_btf(obj)) 3626 return 0; 3627 3628 obj->btf_vmlinux = btf__load_vmlinux_btf(); 3629 err = libbpf_get_error(obj->btf_vmlinux); 3630 if (err) { 3631 pr_warn("Error loading vmlinux BTF: %s\n", errstr(err)); 3632 obj->btf_vmlinux = NULL; 3633 return err; 3634 } 3635 return 0; 3636 } 3637 3638 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 3639 { 3640 struct btf *kern_btf = obj->btf; 3641 bool btf_mandatory, sanitize; 3642 int i, err = 0; 3643 3644 if (!obj->btf) 3645 return 0; 3646 3647 if (!kernel_supports(obj, FEAT_BTF)) { 3648 if (kernel_needs_btf(obj)) { 3649 err = -EOPNOTSUPP; 3650 goto report; 3651 } 3652 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n"); 3653 return 0; 3654 } 3655 3656 /* Even though some subprogs are global/weak, user might prefer more 3657 * permissive BPF verification process that BPF verifier performs for 3658 * static functions, taking into account more context from the caller 3659 * functions. In such case, they need to mark such subprogs with 3660 * __attribute__((visibility("hidden"))) and libbpf will adjust 3661 * corresponding FUNC BTF type to be marked as static and trigger more 3662 * involved BPF verification process. 3663 */ 3664 for (i = 0; i < obj->nr_programs; i++) { 3665 struct bpf_program *prog = &obj->programs[i]; 3666 struct btf_type *t; 3667 const char *name; 3668 int j, n; 3669 3670 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) 3671 continue; 3672 3673 n = btf__type_cnt(obj->btf); 3674 for (j = 1; j < n; j++) { 3675 t = btf_type_by_id(obj->btf, j); 3676 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) 3677 continue; 3678 3679 name = btf__str_by_offset(obj->btf, t->name_off); 3680 if (strcmp(name, prog->name) != 0) 3681 continue; 3682 3683 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0); 3684 break; 3685 } 3686 } 3687 3688 sanitize = btf_needs_sanitization(obj); 3689 if (sanitize) { 3690 kern_btf = bpf_object__sanitize_btf(obj, obj->btf); 3691 if (IS_ERR(kern_btf)) 3692 return PTR_ERR(kern_btf); 3693 } 3694 3695 if (obj->gen_loader) { 3696 __u32 raw_size = 0; 3697 const void *raw_data = btf__raw_data(kern_btf, &raw_size); 3698 3699 if (!raw_data) 3700 return -ENOMEM; 3701 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size); 3702 /* Pretend to have valid FD to pass various fd >= 0 checks. 3703 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 3704 */ 3705 btf__set_fd(kern_btf, 0); 3706 } else { 3707 /* currently BPF_BTF_LOAD only supports log_level 1 */ 3708 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size, 3709 obj->log_level ? 1 : 0, obj->token_fd); 3710 } 3711 if (sanitize) { 3712 if (!err) { 3713 /* move fd to libbpf's BTF */ 3714 btf__set_fd(obj->btf, btf__fd(kern_btf)); 3715 btf__set_fd(kern_btf, -1); 3716 } 3717 btf__free(kern_btf); 3718 } 3719 report: 3720 if (err) { 3721 btf_mandatory = kernel_needs_btf(obj); 3722 if (btf_mandatory) { 3723 pr_warn("Error loading .BTF into kernel: %s. BTF is mandatory, can't proceed.\n", 3724 errstr(err)); 3725 } else { 3726 pr_info("Error loading .BTF into kernel: %s. BTF is optional, ignoring.\n", 3727 errstr(err)); 3728 err = 0; 3729 } 3730 } 3731 return err; 3732 } 3733 3734 static const char *elf_sym_str(const struct bpf_object *obj, size_t off) 3735 { 3736 const char *name; 3737 3738 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); 3739 if (!name) { 3740 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 3741 off, obj->path, elf_errmsg(-1)); 3742 return NULL; 3743 } 3744 3745 return name; 3746 } 3747 3748 static const char *elf_sec_str(const struct bpf_object *obj, size_t off) 3749 { 3750 const char *name; 3751 3752 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); 3753 if (!name) { 3754 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 3755 off, obj->path, elf_errmsg(-1)); 3756 return NULL; 3757 } 3758 3759 return name; 3760 } 3761 3762 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx) 3763 { 3764 Elf_Scn *scn; 3765 3766 scn = elf_getscn(obj->efile.elf, idx); 3767 if (!scn) { 3768 pr_warn("elf: failed to get section(%zu) from %s: %s\n", 3769 idx, obj->path, elf_errmsg(-1)); 3770 return NULL; 3771 } 3772 return scn; 3773 } 3774 3775 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name) 3776 { 3777 Elf_Scn *scn = NULL; 3778 Elf *elf = obj->efile.elf; 3779 const char *sec_name; 3780 3781 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3782 sec_name = elf_sec_name(obj, scn); 3783 if (!sec_name) 3784 return NULL; 3785 3786 if (strcmp(sec_name, name) != 0) 3787 continue; 3788 3789 return scn; 3790 } 3791 return NULL; 3792 } 3793 3794 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn) 3795 { 3796 Elf64_Shdr *shdr; 3797 3798 if (!scn) 3799 return NULL; 3800 3801 shdr = elf64_getshdr(scn); 3802 if (!shdr) { 3803 pr_warn("elf: failed to get section(%zu) header from %s: %s\n", 3804 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 3805 return NULL; 3806 } 3807 3808 return shdr; 3809 } 3810 3811 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn) 3812 { 3813 const char *name; 3814 Elf64_Shdr *sh; 3815 3816 if (!scn) 3817 return NULL; 3818 3819 sh = elf_sec_hdr(obj, scn); 3820 if (!sh) 3821 return NULL; 3822 3823 name = elf_sec_str(obj, sh->sh_name); 3824 if (!name) { 3825 pr_warn("elf: failed to get section(%zu) name from %s: %s\n", 3826 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 3827 return NULL; 3828 } 3829 3830 return name; 3831 } 3832 3833 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn) 3834 { 3835 Elf_Data *data; 3836 3837 if (!scn) 3838 return NULL; 3839 3840 data = elf_getdata(scn, 0); 3841 if (!data) { 3842 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n", 3843 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>", 3844 obj->path, elf_errmsg(-1)); 3845 return NULL; 3846 } 3847 3848 return data; 3849 } 3850 3851 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx) 3852 { 3853 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym)) 3854 return NULL; 3855 3856 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx; 3857 } 3858 3859 static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx) 3860 { 3861 if (idx >= data->d_size / sizeof(Elf64_Rel)) 3862 return NULL; 3863 3864 return (Elf64_Rel *)data->d_buf + idx; 3865 } 3866 3867 static bool is_sec_name_dwarf(const char *name) 3868 { 3869 /* approximation, but the actual list is too long */ 3870 return str_has_pfx(name, ".debug_"); 3871 } 3872 3873 static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name) 3874 { 3875 /* no special handling of .strtab */ 3876 if (hdr->sh_type == SHT_STRTAB) 3877 return true; 3878 3879 /* ignore .llvm_addrsig section as well */ 3880 if (hdr->sh_type == SHT_LLVM_ADDRSIG) 3881 return true; 3882 3883 /* no subprograms will lead to an empty .text section, ignore it */ 3884 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 && 3885 strcmp(name, ".text") == 0) 3886 return true; 3887 3888 /* DWARF sections */ 3889 if (is_sec_name_dwarf(name)) 3890 return true; 3891 3892 if (str_has_pfx(name, ".rel")) { 3893 name += sizeof(".rel") - 1; 3894 /* DWARF section relocations */ 3895 if (is_sec_name_dwarf(name)) 3896 return true; 3897 3898 /* .BTF and .BTF.ext don't need relocations */ 3899 if (strcmp(name, BTF_ELF_SEC) == 0 || 3900 strcmp(name, BTF_EXT_ELF_SEC) == 0) 3901 return true; 3902 } 3903 3904 return false; 3905 } 3906 3907 static int cmp_progs(const void *_a, const void *_b) 3908 { 3909 const struct bpf_program *a = _a; 3910 const struct bpf_program *b = _b; 3911 3912 if (a->sec_idx != b->sec_idx) 3913 return a->sec_idx < b->sec_idx ? -1 : 1; 3914 3915 /* sec_insn_off can't be the same within the section */ 3916 return a->sec_insn_off < b->sec_insn_off ? -1 : 1; 3917 } 3918 3919 static int bpf_object__elf_collect(struct bpf_object *obj) 3920 { 3921 struct elf_sec_desc *sec_desc; 3922 Elf *elf = obj->efile.elf; 3923 Elf_Data *btf_ext_data = NULL; 3924 Elf_Data *btf_data = NULL; 3925 int idx = 0, err = 0; 3926 const char *name; 3927 Elf_Data *data; 3928 Elf_Scn *scn; 3929 Elf64_Shdr *sh; 3930 3931 /* ELF section indices are 0-based, but sec #0 is special "invalid" 3932 * section. Since section count retrieved by elf_getshdrnum() does 3933 * include sec #0, it is already the necessary size of an array to keep 3934 * all the sections. 3935 */ 3936 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) { 3937 pr_warn("elf: failed to get the number of sections for %s: %s\n", 3938 obj->path, elf_errmsg(-1)); 3939 return -LIBBPF_ERRNO__FORMAT; 3940 } 3941 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs)); 3942 if (!obj->efile.secs) 3943 return -ENOMEM; 3944 3945 /* a bunch of ELF parsing functionality depends on processing symbols, 3946 * so do the first pass and find the symbol table 3947 */ 3948 scn = NULL; 3949 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3950 sh = elf_sec_hdr(obj, scn); 3951 if (!sh) 3952 return -LIBBPF_ERRNO__FORMAT; 3953 3954 if (sh->sh_type == SHT_SYMTAB) { 3955 if (obj->efile.symbols) { 3956 pr_warn("elf: multiple symbol tables in %s\n", obj->path); 3957 return -LIBBPF_ERRNO__FORMAT; 3958 } 3959 3960 data = elf_sec_data(obj, scn); 3961 if (!data) 3962 return -LIBBPF_ERRNO__FORMAT; 3963 3964 idx = elf_ndxscn(scn); 3965 3966 obj->efile.symbols = data; 3967 obj->efile.symbols_shndx = idx; 3968 obj->efile.strtabidx = sh->sh_link; 3969 } 3970 } 3971 3972 if (!obj->efile.symbols) { 3973 pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n", 3974 obj->path); 3975 return -ENOENT; 3976 } 3977 3978 scn = NULL; 3979 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3980 idx = elf_ndxscn(scn); 3981 sec_desc = &obj->efile.secs[idx]; 3982 3983 sh = elf_sec_hdr(obj, scn); 3984 if (!sh) 3985 return -LIBBPF_ERRNO__FORMAT; 3986 3987 name = elf_sec_str(obj, sh->sh_name); 3988 if (!name) 3989 return -LIBBPF_ERRNO__FORMAT; 3990 3991 if (ignore_elf_section(sh, name)) 3992 continue; 3993 3994 data = elf_sec_data(obj, scn); 3995 if (!data) 3996 return -LIBBPF_ERRNO__FORMAT; 3997 3998 pr_debug("elf: section(%d) %s, size %lu, link %d, flags %lx, type=%d\n", 3999 idx, name, (unsigned long)data->d_size, 4000 (int)sh->sh_link, (unsigned long)sh->sh_flags, 4001 (int)sh->sh_type); 4002 4003 if (strcmp(name, "license") == 0) { 4004 err = bpf_object__init_license(obj, data->d_buf, data->d_size); 4005 if (err) 4006 return err; 4007 } else if (strcmp(name, "version") == 0) { 4008 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); 4009 if (err) 4010 return err; 4011 } else if (strcmp(name, "maps") == 0) { 4012 pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n"); 4013 return -ENOTSUP; 4014 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 4015 obj->efile.btf_maps_shndx = idx; 4016 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 4017 if (sh->sh_type != SHT_PROGBITS) 4018 return -LIBBPF_ERRNO__FORMAT; 4019 btf_data = data; 4020 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 4021 if (sh->sh_type != SHT_PROGBITS) 4022 return -LIBBPF_ERRNO__FORMAT; 4023 btf_ext_data = data; 4024 } else if (sh->sh_type == SHT_SYMTAB) { 4025 /* already processed during the first pass above */ 4026 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) { 4027 if (sh->sh_flags & SHF_EXECINSTR) { 4028 if (strcmp(name, ".text") == 0) 4029 obj->efile.text_shndx = idx; 4030 err = bpf_object__add_programs(obj, data, name, idx); 4031 if (err) 4032 return err; 4033 } else if (strcmp(name, DATA_SEC) == 0 || 4034 str_has_pfx(name, DATA_SEC ".")) { 4035 sec_desc->sec_type = SEC_DATA; 4036 sec_desc->shdr = sh; 4037 sec_desc->data = data; 4038 } else if (strcmp(name, RODATA_SEC) == 0 || 4039 str_has_pfx(name, RODATA_SEC ".")) { 4040 sec_desc->sec_type = SEC_RODATA; 4041 sec_desc->shdr = sh; 4042 sec_desc->data = data; 4043 } else if (strcmp(name, PERCPU_SEC) == 0 || 4044 str_has_pfx(name, PERCPU_SEC ".")) { 4045 sec_desc->sec_type = SEC_PERCPU; 4046 sec_desc->shdr = sh; 4047 sec_desc->data = data; 4048 } else if (strcmp(name, STRUCT_OPS_SEC) == 0 || 4049 strcmp(name, STRUCT_OPS_LINK_SEC) == 0 || 4050 strcmp(name, "?" STRUCT_OPS_SEC) == 0 || 4051 strcmp(name, "?" STRUCT_OPS_LINK_SEC) == 0) { 4052 sec_desc->sec_type = SEC_ST_OPS; 4053 sec_desc->shdr = sh; 4054 sec_desc->data = data; 4055 obj->efile.has_st_ops = true; 4056 } else if (strcmp(name, ARENA_SEC) == 0) { 4057 obj->efile.arena_data = data; 4058 obj->efile.arena_data_shndx = idx; 4059 } else if (strcmp(name, JUMPTABLES_SEC) == 0) { 4060 obj->jumptables_data = malloc(data->d_size); 4061 if (!obj->jumptables_data) 4062 return -ENOMEM; 4063 memcpy(obj->jumptables_data, data->d_buf, data->d_size); 4064 obj->jumptables_data_sz = data->d_size; 4065 obj->efile.jumptables_data_shndx = idx; 4066 } else { 4067 pr_info("elf: skipping unrecognized data section(%d) %s\n", 4068 idx, name); 4069 } 4070 } else if (sh->sh_type == SHT_REL) { 4071 int targ_sec_idx = sh->sh_info; /* points to other section */ 4072 4073 if (sh->sh_entsize != sizeof(Elf64_Rel) || 4074 targ_sec_idx >= obj->efile.sec_cnt) 4075 return -LIBBPF_ERRNO__FORMAT; 4076 4077 /* Only do relo for section with exec instructions */ 4078 if (!section_have_execinstr(obj, targ_sec_idx) && 4079 strcmp(name, ".rel" STRUCT_OPS_SEC) && 4080 strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) && 4081 strcmp(name, ".rel?" STRUCT_OPS_SEC) && 4082 strcmp(name, ".rel?" STRUCT_OPS_LINK_SEC) && 4083 strcmp(name, ".rel" MAPS_ELF_SEC)) { 4084 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", 4085 idx, name, targ_sec_idx, 4086 elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>"); 4087 continue; 4088 } 4089 4090 sec_desc->sec_type = SEC_RELO; 4091 sec_desc->shdr = sh; 4092 sec_desc->data = data; 4093 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 || 4094 str_has_pfx(name, BSS_SEC "."))) { 4095 sec_desc->sec_type = SEC_BSS; 4096 sec_desc->shdr = sh; 4097 sec_desc->data = data; 4098 } else { 4099 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name, 4100 (size_t)sh->sh_size); 4101 } 4102 } 4103 4104 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { 4105 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); 4106 return -LIBBPF_ERRNO__FORMAT; 4107 } 4108 4109 /* change BPF program insns to native endianness for introspection */ 4110 if (!is_native_endianness(obj)) 4111 bpf_object_bswap_progs(obj); 4112 4113 /* sort BPF programs by section name and in-section instruction offset 4114 * for faster search 4115 */ 4116 if (obj->nr_programs) 4117 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); 4118 4119 return bpf_object__init_btf(obj, btf_data, btf_ext_data); 4120 } 4121 4122 static bool sym_is_extern(const Elf64_Sym *sym) 4123 { 4124 int bind = ELF64_ST_BIND(sym->st_info); 4125 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ 4126 return sym->st_shndx == SHN_UNDEF && 4127 (bind == STB_GLOBAL || bind == STB_WEAK) && 4128 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE; 4129 } 4130 4131 static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx) 4132 { 4133 int bind = ELF64_ST_BIND(sym->st_info); 4134 int type = ELF64_ST_TYPE(sym->st_info); 4135 4136 /* in .text section */ 4137 if (sym->st_shndx != text_shndx) 4138 return false; 4139 4140 /* local function */ 4141 if (bind == STB_LOCAL && type == STT_SECTION) 4142 return true; 4143 4144 /* global function */ 4145 return (bind == STB_GLOBAL || bind == STB_WEAK) && type == STT_FUNC; 4146 } 4147 4148 static int find_extern_btf_id(const struct btf *btf, const char *ext_name) 4149 { 4150 const struct btf_type *t; 4151 const char *tname; 4152 int i, n; 4153 4154 if (!btf) 4155 return -ESRCH; 4156 4157 n = btf__type_cnt(btf); 4158 for (i = 1; i < n; i++) { 4159 t = btf__type_by_id(btf, i); 4160 4161 if (!btf_is_var(t) && !btf_is_func(t)) 4162 continue; 4163 4164 tname = btf__name_by_offset(btf, t->name_off); 4165 if (strcmp(tname, ext_name)) 4166 continue; 4167 4168 if (btf_is_var(t) && 4169 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) 4170 return -EINVAL; 4171 4172 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN) 4173 return -EINVAL; 4174 4175 return i; 4176 } 4177 4178 return -ENOENT; 4179 } 4180 4181 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) { 4182 const struct btf_var_secinfo *vs; 4183 const struct btf_type *t; 4184 int i, j, n; 4185 4186 if (!btf) 4187 return -ESRCH; 4188 4189 n = btf__type_cnt(btf); 4190 for (i = 1; i < n; i++) { 4191 t = btf__type_by_id(btf, i); 4192 4193 if (!btf_is_datasec(t)) 4194 continue; 4195 4196 vs = btf_var_secinfos(t); 4197 for (j = 0; j < btf_vlen(t); j++, vs++) { 4198 if (vs->type == ext_btf_id) 4199 return i; 4200 } 4201 } 4202 4203 return -ENOENT; 4204 } 4205 4206 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, 4207 bool *is_signed) 4208 { 4209 const struct btf_type *t; 4210 const char *name; 4211 4212 t = skip_mods_and_typedefs(btf, id, NULL); 4213 name = btf__name_by_offset(btf, t->name_off); 4214 4215 if (is_signed) 4216 *is_signed = false; 4217 switch (btf_kind(t)) { 4218 case BTF_KIND_INT: { 4219 int enc = btf_int_encoding(t); 4220 4221 if (enc & BTF_INT_BOOL) 4222 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; 4223 if (is_signed) 4224 *is_signed = enc & BTF_INT_SIGNED; 4225 if (t->size == 1) 4226 return KCFG_CHAR; 4227 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) 4228 return KCFG_UNKNOWN; 4229 return KCFG_INT; 4230 } 4231 case BTF_KIND_ENUM: 4232 if (t->size != 4) 4233 return KCFG_UNKNOWN; 4234 if (strcmp(name, "libbpf_tristate")) 4235 return KCFG_UNKNOWN; 4236 return KCFG_TRISTATE; 4237 case BTF_KIND_ENUM64: 4238 if (strcmp(name, "libbpf_tristate")) 4239 return KCFG_UNKNOWN; 4240 return KCFG_TRISTATE; 4241 case BTF_KIND_ARRAY: 4242 if (btf_array(t)->nelems == 0) 4243 return KCFG_UNKNOWN; 4244 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) 4245 return KCFG_UNKNOWN; 4246 return KCFG_CHAR_ARR; 4247 default: 4248 return KCFG_UNKNOWN; 4249 } 4250 } 4251 4252 static int cmp_externs(const void *_a, const void *_b) 4253 { 4254 const struct extern_desc *a = _a; 4255 const struct extern_desc *b = _b; 4256 4257 if (a->type != b->type) 4258 return a->type < b->type ? -1 : 1; 4259 4260 if (a->type == EXT_KCFG) { 4261 /* descending order by alignment requirements */ 4262 if (a->kcfg.align != b->kcfg.align) 4263 return a->kcfg.align > b->kcfg.align ? -1 : 1; 4264 /* ascending order by size, within same alignment class */ 4265 if (a->kcfg.sz != b->kcfg.sz) 4266 return a->kcfg.sz < b->kcfg.sz ? -1 : 1; 4267 } 4268 4269 /* resolve ties by name */ 4270 return strcmp(a->name, b->name); 4271 } 4272 4273 static int find_int_btf_id(const struct btf *btf) 4274 { 4275 const struct btf_type *t; 4276 int i, n; 4277 4278 n = btf__type_cnt(btf); 4279 for (i = 1; i < n; i++) { 4280 t = btf__type_by_id(btf, i); 4281 4282 if (btf_is_int(t) && btf_int_bits(t) == 32) 4283 return i; 4284 } 4285 4286 return 0; 4287 } 4288 4289 static int add_dummy_ksym_var(struct btf *btf) 4290 { 4291 int i, int_btf_id, sec_btf_id, dummy_var_btf_id; 4292 const struct btf_var_secinfo *vs; 4293 const struct btf_type *sec; 4294 4295 if (!btf) 4296 return 0; 4297 4298 sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC, 4299 BTF_KIND_DATASEC); 4300 if (sec_btf_id < 0) 4301 return 0; 4302 4303 sec = btf__type_by_id(btf, sec_btf_id); 4304 vs = btf_var_secinfos(sec); 4305 for (i = 0; i < btf_vlen(sec); i++, vs++) { 4306 const struct btf_type *vt; 4307 4308 vt = btf__type_by_id(btf, vs->type); 4309 if (btf_is_func(vt)) 4310 break; 4311 } 4312 4313 /* No func in ksyms sec. No need to add dummy var. */ 4314 if (i == btf_vlen(sec)) 4315 return 0; 4316 4317 int_btf_id = find_int_btf_id(btf); 4318 dummy_var_btf_id = btf__add_var(btf, 4319 "dummy_ksym", 4320 BTF_VAR_GLOBAL_ALLOCATED, 4321 int_btf_id); 4322 if (dummy_var_btf_id < 0) 4323 pr_warn("cannot create a dummy_ksym var\n"); 4324 4325 return dummy_var_btf_id; 4326 } 4327 4328 static int bpf_object__collect_externs(struct bpf_object *obj) 4329 { 4330 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL; 4331 const struct btf_type *t; 4332 struct extern_desc *ext; 4333 int i, n, off, dummy_var_btf_id; 4334 const char *ext_name, *sec_name; 4335 size_t ext_essent_len; 4336 Elf_Scn *scn; 4337 Elf64_Shdr *sh; 4338 4339 if (!obj->efile.symbols) 4340 return 0; 4341 4342 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); 4343 sh = elf_sec_hdr(obj, scn); 4344 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym)) 4345 return -LIBBPF_ERRNO__FORMAT; 4346 4347 dummy_var_btf_id = add_dummy_ksym_var(obj->btf); 4348 if (dummy_var_btf_id < 0) 4349 return dummy_var_btf_id; 4350 4351 n = sh->sh_size / sh->sh_entsize; 4352 pr_debug("looking for externs among %d symbols...\n", n); 4353 4354 for (i = 0; i < n; i++) { 4355 Elf64_Sym *sym = elf_sym_by_idx(obj, i); 4356 4357 if (!sym) 4358 return -LIBBPF_ERRNO__FORMAT; 4359 if (!sym_is_extern(sym)) 4360 continue; 4361 ext_name = elf_sym_str(obj, sym->st_name); 4362 if (str_is_empty(ext_name)) 4363 continue; 4364 4365 ext = obj->externs; 4366 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); 4367 if (!ext) 4368 return -ENOMEM; 4369 obj->externs = ext; 4370 ext = &ext[obj->nr_extern]; 4371 memset(ext, 0, sizeof(*ext)); 4372 obj->nr_extern++; 4373 4374 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); 4375 if (ext->btf_id <= 0) { 4376 pr_warn("failed to find BTF for extern '%s': %d\n", 4377 ext_name, ext->btf_id); 4378 return ext->btf_id; 4379 } 4380 t = btf__type_by_id(obj->btf, ext->btf_id); 4381 ext->name = strdup(btf__name_by_offset(obj->btf, t->name_off)); 4382 if (!ext->name) 4383 return -ENOMEM; 4384 ext->sym_idx = i; 4385 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK; 4386 4387 ext_essent_len = bpf_core_essential_name_len(ext->name); 4388 ext->essent_name = NULL; 4389 if (ext_essent_len != strlen(ext->name)) { 4390 ext->essent_name = strndup(ext->name, ext_essent_len); 4391 if (!ext->essent_name) 4392 return -ENOMEM; 4393 } 4394 4395 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); 4396 if (ext->sec_btf_id <= 0) { 4397 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n", 4398 ext_name, ext->btf_id, ext->sec_btf_id); 4399 return ext->sec_btf_id; 4400 } 4401 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); 4402 sec_name = btf__name_by_offset(obj->btf, sec->name_off); 4403 4404 if (strcmp(sec_name, KCONFIG_SEC) == 0) { 4405 if (btf_is_func(t)) { 4406 pr_warn("extern function %s is unsupported under %s section\n", 4407 ext->name, KCONFIG_SEC); 4408 return -ENOTSUP; 4409 } 4410 kcfg_sec = sec; 4411 ext->type = EXT_KCFG; 4412 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); 4413 if (ext->kcfg.sz <= 0) { 4414 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n", 4415 ext_name, ext->kcfg.sz); 4416 return ext->kcfg.sz; 4417 } 4418 ext->kcfg.align = btf__align_of(obj->btf, t->type); 4419 if (ext->kcfg.align <= 0) { 4420 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n", 4421 ext_name, ext->kcfg.align); 4422 return -EINVAL; 4423 } 4424 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, 4425 &ext->kcfg.is_signed); 4426 if (ext->kcfg.type == KCFG_UNKNOWN) { 4427 pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name); 4428 return -ENOTSUP; 4429 } 4430 } else if (strcmp(sec_name, KSYMS_SEC) == 0) { 4431 ksym_sec = sec; 4432 ext->type = EXT_KSYM; 4433 skip_mods_and_typedefs(obj->btf, t->type, 4434 &ext->ksym.type_id); 4435 } else { 4436 pr_warn("unrecognized extern section '%s'\n", sec_name); 4437 return -ENOTSUP; 4438 } 4439 } 4440 pr_debug("collected %d externs total\n", obj->nr_extern); 4441 4442 if (!obj->nr_extern) 4443 return 0; 4444 4445 /* sort externs by type, for kcfg ones also by (align, size, name) */ 4446 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); 4447 4448 /* for .ksyms section, we need to turn all externs into allocated 4449 * variables in BTF to pass kernel verification; we do this by 4450 * pretending that each extern is a 8-byte variable 4451 */ 4452 if (ksym_sec) { 4453 /* find existing 4-byte integer type in BTF to use for fake 4454 * extern variables in DATASEC 4455 */ 4456 int int_btf_id = find_int_btf_id(obj->btf); 4457 /* For extern function, a dummy_var added earlier 4458 * will be used to replace the vs->type and 4459 * its name string will be used to refill 4460 * the missing param's name. 4461 */ 4462 const struct btf_type *dummy_var; 4463 4464 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); 4465 for (i = 0; i < obj->nr_extern; i++) { 4466 ext = &obj->externs[i]; 4467 if (ext->type != EXT_KSYM) 4468 continue; 4469 pr_debug("extern (ksym) #%d: symbol %d, name %s\n", 4470 i, ext->sym_idx, ext->name); 4471 } 4472 4473 sec = ksym_sec; 4474 n = btf_vlen(sec); 4475 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) { 4476 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 4477 struct btf_type *vt; 4478 4479 vt = (void *)btf__type_by_id(obj->btf, vs->type); 4480 ext_name = btf__name_by_offset(obj->btf, vt->name_off); 4481 ext = find_extern_by_name(obj, ext_name); 4482 if (!ext) { 4483 pr_warn("failed to find extern definition for BTF %s '%s'\n", 4484 btf_kind_str(vt), ext_name); 4485 return -ESRCH; 4486 } 4487 if (btf_is_func(vt)) { 4488 const struct btf_type *func_proto; 4489 struct btf_param *param; 4490 int j; 4491 4492 func_proto = btf__type_by_id(obj->btf, 4493 vt->type); 4494 param = btf_params(func_proto); 4495 /* Reuse the dummy_var string if the 4496 * func proto does not have param name. 4497 */ 4498 for (j = 0; j < btf_vlen(func_proto); j++) 4499 if (param[j].type && !param[j].name_off) 4500 param[j].name_off = 4501 dummy_var->name_off; 4502 vs->type = dummy_var_btf_id; 4503 vt->info &= ~0xffff; 4504 vt->info |= BTF_FUNC_GLOBAL; 4505 } else { 4506 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 4507 vt->type = int_btf_id; 4508 } 4509 vs->offset = off; 4510 vs->size = sizeof(int); 4511 } 4512 sec->size = off; 4513 } 4514 4515 if (kcfg_sec) { 4516 sec = kcfg_sec; 4517 /* for kcfg externs calculate their offsets within a .kconfig map */ 4518 off = 0; 4519 for (i = 0; i < obj->nr_extern; i++) { 4520 ext = &obj->externs[i]; 4521 if (ext->type != EXT_KCFG) 4522 continue; 4523 4524 ext->kcfg.data_off = roundup(off, ext->kcfg.align); 4525 off = ext->kcfg.data_off + ext->kcfg.sz; 4526 pr_debug("extern (kcfg) #%d: symbol %d, off %d, name %s\n", 4527 i, ext->sym_idx, ext->kcfg.data_off, ext->name); 4528 } 4529 sec->size = off; 4530 n = btf_vlen(sec); 4531 for (i = 0; i < n; i++) { 4532 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 4533 4534 t = btf__type_by_id(obj->btf, vs->type); 4535 ext_name = btf__name_by_offset(obj->btf, t->name_off); 4536 ext = find_extern_by_name(obj, ext_name); 4537 if (!ext) { 4538 pr_warn("failed to find extern definition for BTF var '%s'\n", 4539 ext_name); 4540 return -ESRCH; 4541 } 4542 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 4543 vs->offset = ext->kcfg.data_off; 4544 } 4545 } 4546 return 0; 4547 } 4548 4549 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog) 4550 { 4551 return prog->sec_idx == obj->efile.text_shndx; 4552 } 4553 4554 struct bpf_program * 4555 bpf_object__find_program_by_name(const struct bpf_object *obj, 4556 const char *name) 4557 { 4558 struct bpf_program *prog; 4559 4560 bpf_object__for_each_program(prog, obj) { 4561 if (prog_is_subprog(obj, prog)) 4562 continue; 4563 if (!strcmp(prog->name, name)) 4564 return prog; 4565 } 4566 return errno = ENOENT, NULL; 4567 } 4568 4569 static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 4570 int shndx) 4571 { 4572 switch (obj->efile.secs[shndx].sec_type) { 4573 case SEC_BSS: 4574 case SEC_DATA: 4575 case SEC_RODATA: 4576 case SEC_PERCPU: 4577 return true; 4578 default: 4579 return false; 4580 } 4581 } 4582 4583 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 4584 int shndx) 4585 { 4586 return shndx == obj->efile.btf_maps_shndx; 4587 } 4588 4589 static enum libbpf_map_type 4590 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 4591 { 4592 if (shndx == obj->efile.symbols_shndx) 4593 return LIBBPF_MAP_KCONFIG; 4594 4595 switch (obj->efile.secs[shndx].sec_type) { 4596 case SEC_BSS: 4597 return LIBBPF_MAP_BSS; 4598 case SEC_DATA: 4599 return LIBBPF_MAP_DATA; 4600 case SEC_RODATA: 4601 return LIBBPF_MAP_RODATA; 4602 case SEC_PERCPU: 4603 return LIBBPF_MAP_PERCPU; 4604 default: 4605 return LIBBPF_MAP_UNSPEC; 4606 } 4607 } 4608 4609 static int bpf_prog_compute_hash(struct bpf_program *prog) 4610 { 4611 struct bpf_insn *purged; 4612 int i, err = 0; 4613 4614 purged = calloc(prog->insns_cnt, BPF_INSN_SZ); 4615 if (!purged) 4616 return -ENOMEM; 4617 4618 /* If relocations have been done, the map_fd needs to be 4619 * discarded for the digest calculation. 4620 */ 4621 for (i = 0; i < prog->insns_cnt; i++) { 4622 purged[i] = prog->insns[i]; 4623 if (purged[i].code == (BPF_LD | BPF_IMM | BPF_DW) && 4624 (purged[i].src_reg == BPF_PSEUDO_MAP_FD || 4625 purged[i].src_reg == BPF_PSEUDO_MAP_VALUE)) { 4626 purged[i].imm = 0; 4627 i++; 4628 if (i >= prog->insns_cnt || 4629 prog->insns[i].code != 0 || 4630 prog->insns[i].dst_reg != 0 || 4631 prog->insns[i].src_reg != 0 || 4632 prog->insns[i].off != 0) { 4633 err = -EINVAL; 4634 goto out; 4635 } 4636 purged[i] = prog->insns[i]; 4637 purged[i].imm = 0; 4638 } 4639 } 4640 libbpf_sha256(purged, prog->insns_cnt * sizeof(struct bpf_insn), 4641 prog->hash); 4642 out: 4643 free(purged); 4644 return err; 4645 } 4646 4647 static int bpf_program__record_reloc(struct bpf_program *prog, 4648 struct reloc_desc *reloc_desc, 4649 __u32 insn_idx, const char *sym_name, 4650 const Elf64_Sym *sym, const Elf64_Rel *rel) 4651 { 4652 struct bpf_insn *insn = &prog->insns[insn_idx]; 4653 size_t map_idx, nr_maps = prog->obj->nr_maps; 4654 struct bpf_object *obj = prog->obj; 4655 __u32 shdr_idx = sym->st_shndx; 4656 enum libbpf_map_type type; 4657 const char *sym_sec_name; 4658 struct bpf_map *map; 4659 4660 if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { 4661 pr_warn("prog '%s': invalid relo against '%s' for insns[%u].code 0x%x\n", 4662 prog->name, sym_name, insn_idx, insn->code); 4663 return -LIBBPF_ERRNO__RELOC; 4664 } 4665 4666 if (sym_is_extern(sym)) { 4667 int sym_idx = ELF64_R_SYM(rel->r_info); 4668 int i, n = obj->nr_extern; 4669 struct extern_desc *ext; 4670 4671 for (i = 0; i < n; i++) { 4672 ext = &obj->externs[i]; 4673 if (ext->sym_idx == sym_idx) 4674 break; 4675 } 4676 if (i >= n) { 4677 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n", 4678 prog->name, sym_name, sym_idx); 4679 return -LIBBPF_ERRNO__RELOC; 4680 } 4681 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n", 4682 prog->name, i, ext->name, ext->sym_idx, insn_idx); 4683 if (insn->code == (BPF_JMP | BPF_CALL)) 4684 reloc_desc->type = RELO_EXTERN_CALL; 4685 else 4686 reloc_desc->type = RELO_EXTERN_LD64; 4687 reloc_desc->insn_idx = insn_idx; 4688 reloc_desc->ext_idx = i; 4689 return 0; 4690 } 4691 4692 /* sub-program call relocation */ 4693 if (is_call_insn(insn)) { 4694 if (insn->src_reg != BPF_PSEUDO_CALL) { 4695 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); 4696 return -LIBBPF_ERRNO__RELOC; 4697 } 4698 /* text_shndx can be 0, if no default "main" program exists */ 4699 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { 4700 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 4701 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n", 4702 prog->name, sym_name, sym_sec_name); 4703 return -LIBBPF_ERRNO__RELOC; 4704 } 4705 if (sym->st_value % BPF_INSN_SZ) { 4706 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n", 4707 prog->name, sym_name, (size_t)sym->st_value); 4708 return -LIBBPF_ERRNO__RELOC; 4709 } 4710 reloc_desc->type = RELO_CALL; 4711 reloc_desc->insn_idx = insn_idx; 4712 reloc_desc->sym_off = sym->st_value; 4713 return 0; 4714 } 4715 4716 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { 4717 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n", 4718 prog->name, sym_name, shdr_idx); 4719 return -LIBBPF_ERRNO__RELOC; 4720 } 4721 4722 /* loading subprog addresses */ 4723 if (sym_is_subprog(sym, obj->efile.text_shndx)) { 4724 /* global_func: sym->st_value = offset in the section, insn->imm = 0. 4725 * local_func: sym->st_value = 0, insn->imm = offset in the section. 4726 */ 4727 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { 4728 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n", 4729 prog->name, sym_name, (size_t)sym->st_value, insn->imm); 4730 return -LIBBPF_ERRNO__RELOC; 4731 } 4732 4733 reloc_desc->type = RELO_SUBPROG_ADDR; 4734 reloc_desc->insn_idx = insn_idx; 4735 reloc_desc->sym_off = sym->st_value; 4736 return 0; 4737 } 4738 4739 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 4740 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 4741 4742 /* arena data relocation */ 4743 if (shdr_idx == obj->efile.arena_data_shndx) { 4744 if (obj->arena_map_idx < 0) { 4745 pr_warn("prog '%s': bad arena data relocation at insn %u, no arena maps defined\n", 4746 prog->name, insn_idx); 4747 return -LIBBPF_ERRNO__RELOC; 4748 } 4749 reloc_desc->type = RELO_DATA; 4750 reloc_desc->insn_idx = insn_idx; 4751 reloc_desc->map_idx = obj->arena_map_idx; 4752 reloc_desc->sym_off = sym->st_value; 4753 4754 map = &obj->maps[obj->arena_map_idx]; 4755 pr_debug("prog '%s': found arena map %d (%s, sec %d, off %zu) for insn %u\n", 4756 prog->name, obj->arena_map_idx, map->name, map->sec_idx, 4757 map->sec_offset, insn_idx); 4758 return 0; 4759 } 4760 4761 /* jump table data relocation */ 4762 if (shdr_idx == obj->efile.jumptables_data_shndx) { 4763 reloc_desc->type = RELO_INSN_ARRAY; 4764 reloc_desc->insn_idx = insn_idx; 4765 reloc_desc->map_idx = -1; 4766 reloc_desc->sym_off = sym->st_value; 4767 reloc_desc->sym_size = sym->st_size; 4768 return 0; 4769 } 4770 4771 /* generic map reference relocation */ 4772 if (type == LIBBPF_MAP_UNSPEC) { 4773 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { 4774 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n", 4775 prog->name, sym_name, sym_sec_name); 4776 return -LIBBPF_ERRNO__RELOC; 4777 } 4778 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 4779 map = &obj->maps[map_idx]; 4780 if (map->libbpf_type != type || 4781 map->sec_idx != sym->st_shndx || 4782 map->sec_offset != sym->st_value) 4783 continue; 4784 pr_debug("prog '%s': found map %zu (%s, sec %d, off %zu) for insn #%u\n", 4785 prog->name, map_idx, map->name, map->sec_idx, 4786 map->sec_offset, insn_idx); 4787 break; 4788 } 4789 if (map_idx >= nr_maps) { 4790 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n", 4791 prog->name, sym_sec_name, (size_t)sym->st_value); 4792 return -LIBBPF_ERRNO__RELOC; 4793 } 4794 reloc_desc->type = RELO_LD64; 4795 reloc_desc->insn_idx = insn_idx; 4796 reloc_desc->map_idx = map_idx; 4797 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ 4798 return 0; 4799 } 4800 4801 /* global data map relocation */ 4802 if (!bpf_object__shndx_is_data(obj, shdr_idx)) { 4803 pr_warn("prog '%s': bad data relo against section '%s'\n", 4804 prog->name, sym_sec_name); 4805 return -LIBBPF_ERRNO__RELOC; 4806 } 4807 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 4808 map = &obj->maps[map_idx]; 4809 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx) 4810 continue; 4811 pr_debug("prog '%s': found data map %zu (%s, sec %d, off %zu) for insn %u\n", 4812 prog->name, map_idx, map->name, map->sec_idx, 4813 map->sec_offset, insn_idx); 4814 break; 4815 } 4816 if (map_idx >= nr_maps) { 4817 pr_warn("prog '%s': data relo failed to find map for section '%s'\n", 4818 prog->name, sym_sec_name); 4819 return -LIBBPF_ERRNO__RELOC; 4820 } 4821 4822 reloc_desc->type = RELO_DATA; 4823 reloc_desc->insn_idx = insn_idx; 4824 reloc_desc->map_idx = map_idx; 4825 reloc_desc->sym_off = sym->st_value; 4826 return 0; 4827 } 4828 4829 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx) 4830 { 4831 return insn_idx >= prog->sec_insn_off && 4832 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; 4833 } 4834 4835 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, 4836 size_t sec_idx, size_t insn_idx) 4837 { 4838 int l = 0, r = obj->nr_programs - 1, m; 4839 struct bpf_program *prog; 4840 4841 if (!obj->nr_programs) 4842 return NULL; 4843 4844 while (l < r) { 4845 m = l + (r - l + 1) / 2; 4846 prog = &obj->programs[m]; 4847 4848 if (prog->sec_idx < sec_idx || 4849 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) 4850 l = m; 4851 else 4852 r = m - 1; 4853 } 4854 /* matching program could be at index l, but it still might be the 4855 * wrong one, so we need to double check conditions for the last time 4856 */ 4857 prog = &obj->programs[l]; 4858 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) 4859 return prog; 4860 return NULL; 4861 } 4862 4863 static int 4864 bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) 4865 { 4866 const char *relo_sec_name, *sec_name; 4867 size_t sec_idx = shdr->sh_info, sym_idx; 4868 struct bpf_program *prog; 4869 struct reloc_desc *relos; 4870 int err, i, nrels; 4871 const char *sym_name; 4872 __u32 insn_idx; 4873 Elf_Scn *scn; 4874 Elf_Data *scn_data; 4875 Elf64_Sym *sym; 4876 Elf64_Rel *rel; 4877 4878 if (sec_idx >= obj->efile.sec_cnt) 4879 return -EINVAL; 4880 4881 scn = elf_sec_by_idx(obj, sec_idx); 4882 scn_data = elf_sec_data(obj, scn); 4883 if (!scn_data) 4884 return -LIBBPF_ERRNO__FORMAT; 4885 4886 relo_sec_name = elf_sec_str(obj, shdr->sh_name); 4887 sec_name = elf_sec_name(obj, scn); 4888 if (!relo_sec_name || !sec_name) 4889 return -EINVAL; 4890 4891 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n", 4892 relo_sec_name, sec_idx, sec_name); 4893 nrels = shdr->sh_size / shdr->sh_entsize; 4894 4895 for (i = 0; i < nrels; i++) { 4896 rel = elf_rel_by_idx(data, i); 4897 if (!rel) { 4898 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); 4899 return -LIBBPF_ERRNO__FORMAT; 4900 } 4901 4902 sym_idx = ELF64_R_SYM(rel->r_info); 4903 sym = elf_sym_by_idx(obj, sym_idx); 4904 if (!sym) { 4905 pr_warn("sec '%s': symbol #%zu not found for relo #%d\n", 4906 relo_sec_name, sym_idx, i); 4907 return -LIBBPF_ERRNO__FORMAT; 4908 } 4909 4910 if (sym->st_shndx >= obj->efile.sec_cnt) { 4911 pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n", 4912 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i); 4913 return -LIBBPF_ERRNO__FORMAT; 4914 } 4915 4916 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) { 4917 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n", 4918 relo_sec_name, (size_t)rel->r_offset, i); 4919 return -LIBBPF_ERRNO__FORMAT; 4920 } 4921 4922 insn_idx = rel->r_offset / BPF_INSN_SZ; 4923 /* relocations against static functions are recorded as 4924 * relocations against the section that contains a function; 4925 * in such case, symbol will be STT_SECTION and sym.st_name 4926 * will point to empty string (0), so fetch section name 4927 * instead 4928 */ 4929 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0) 4930 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)); 4931 else 4932 sym_name = elf_sym_str(obj, sym->st_name); 4933 sym_name = sym_name ?: "<?"; 4934 4935 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n", 4936 relo_sec_name, i, insn_idx, sym_name); 4937 4938 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 4939 if (!prog) { 4940 pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n", 4941 relo_sec_name, i, sec_name, insn_idx); 4942 continue; 4943 } 4944 4945 relos = libbpf_reallocarray(prog->reloc_desc, 4946 prog->nr_reloc + 1, sizeof(*relos)); 4947 if (!relos) 4948 return -ENOMEM; 4949 prog->reloc_desc = relos; 4950 4951 /* adjust insn_idx to local BPF program frame of reference */ 4952 insn_idx -= prog->sec_insn_off; 4953 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], 4954 insn_idx, sym_name, sym, rel); 4955 if (err) 4956 return err; 4957 4958 prog->nr_reloc++; 4959 } 4960 return 0; 4961 } 4962 4963 static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map) 4964 { 4965 int id; 4966 4967 if (!obj->btf) 4968 return -ENOENT; 4969 4970 /* if it's BTF-defined map, we don't need to search for type IDs. 4971 * For struct_ops map, it does not need btf_key_type_id and 4972 * btf_value_type_id. 4973 */ 4974 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map)) 4975 return 0; 4976 4977 /* 4978 * LLVM annotates global data differently in BTF, that is, 4979 * only as '.data', '.bss', '.percpu' or '.rodata'. 4980 */ 4981 if (!bpf_map__is_internal(map)) 4982 return -ENOENT; 4983 4984 id = btf__find_by_name(obj->btf, map->real_name); 4985 if (id < 0) 4986 return id; 4987 4988 map->btf_key_type_id = 0; 4989 map->btf_value_type_id = id; 4990 return 0; 4991 } 4992 4993 static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) 4994 { 4995 char file[PATH_MAX], buff[4096]; 4996 FILE *fp; 4997 __u32 val; 4998 int err; 4999 5000 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd); 5001 memset(info, 0, sizeof(*info)); 5002 5003 fp = fopen(file, "re"); 5004 if (!fp) { 5005 err = -errno; 5006 pr_warn("failed to open %s: %s. No procfs support?\n", file, 5007 errstr(err)); 5008 return err; 5009 } 5010 5011 while (fgets(buff, sizeof(buff), fp)) { 5012 if (sscanf(buff, "map_type:\t%u", &val) == 1) 5013 info->type = val; 5014 else if (sscanf(buff, "key_size:\t%u", &val) == 1) 5015 info->key_size = val; 5016 else if (sscanf(buff, "value_size:\t%u", &val) == 1) 5017 info->value_size = val; 5018 else if (sscanf(buff, "max_entries:\t%u", &val) == 1) 5019 info->max_entries = val; 5020 else if (sscanf(buff, "map_flags:\t%x", &val) == 1) 5021 info->map_flags = val; 5022 } 5023 5024 fclose(fp); 5025 5026 return 0; 5027 } 5028 5029 static bool map_is_created(const struct bpf_map *map) 5030 { 5031 return map->obj->state >= OBJ_PREPARED || map->reused; 5032 } 5033 5034 bool bpf_map__autocreate(const struct bpf_map *map) 5035 { 5036 return map->autocreate; 5037 } 5038 5039 int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) 5040 { 5041 if (map_is_created(map)) 5042 return libbpf_err(-EBUSY); 5043 5044 map->autocreate = autocreate; 5045 return 0; 5046 } 5047 5048 int bpf_map__set_autoattach(struct bpf_map *map, bool autoattach) 5049 { 5050 if (!bpf_map__is_struct_ops(map)) 5051 return libbpf_err(-EINVAL); 5052 5053 map->autoattach = autoattach; 5054 return 0; 5055 } 5056 5057 bool bpf_map__autoattach(const struct bpf_map *map) 5058 { 5059 return map->autoattach; 5060 } 5061 5062 int bpf_map__reuse_fd(struct bpf_map *map, int fd) 5063 { 5064 struct bpf_map_info info; 5065 __u32 len = sizeof(info), name_len; 5066 int new_fd, err; 5067 char *new_name; 5068 5069 memset(&info, 0, len); 5070 err = bpf_map_get_info_by_fd(fd, &info, &len); 5071 if (err && errno == EINVAL) 5072 err = bpf_get_map_info_from_fdinfo(fd, &info); 5073 if (err) 5074 return libbpf_err(err); 5075 5076 name_len = strlen(info.name); 5077 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0) 5078 new_name = strdup(map->name); 5079 else 5080 new_name = strdup(info.name); 5081 5082 if (!new_name) 5083 return libbpf_err(-errno); 5084 5085 /* 5086 * Like dup(), but make sure new FD is >= 3 and has O_CLOEXEC set. 5087 * This is similar to what we do in ensure_good_fd(), but without 5088 * closing original FD. 5089 */ 5090 new_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); 5091 if (new_fd < 0) { 5092 err = -errno; 5093 goto err_free_new_name; 5094 } 5095 5096 err = reuse_fd(map->fd, new_fd); 5097 if (err) 5098 goto err_free_new_name; 5099 5100 free(map->name); 5101 5102 map->name = new_name; 5103 map->def.type = info.type; 5104 map->def.key_size = info.key_size; 5105 map->def.value_size = info.value_size; 5106 map->def.max_entries = info.max_entries; 5107 map->def.map_flags = info.map_flags; 5108 map->btf_key_type_id = info.btf_key_type_id; 5109 map->btf_value_type_id = info.btf_value_type_id; 5110 map->reused = true; 5111 map->map_extra = info.map_extra; 5112 5113 return 0; 5114 5115 err_free_new_name: 5116 free(new_name); 5117 return libbpf_err(err); 5118 } 5119 5120 __u32 bpf_map__max_entries(const struct bpf_map *map) 5121 { 5122 return map->def.max_entries; 5123 } 5124 5125 struct bpf_map *bpf_map__inner_map(struct bpf_map *map) 5126 { 5127 if (!bpf_map_type__is_map_in_map(map->def.type)) 5128 return errno = EINVAL, NULL; 5129 5130 return map->inner_map; 5131 } 5132 5133 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) 5134 { 5135 if (map_is_created(map)) 5136 return libbpf_err(-EBUSY); 5137 5138 map->def.max_entries = max_entries; 5139 5140 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ 5141 if (map_is_ringbuf(map)) 5142 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); 5143 5144 return 0; 5145 } 5146 5147 static int bpf_object_prepare_token(struct bpf_object *obj) 5148 { 5149 const char *bpffs_path; 5150 int bpffs_fd = -1, token_fd, err; 5151 bool mandatory; 5152 enum libbpf_print_level level; 5153 5154 /* token is explicitly prevented */ 5155 if (obj->token_path && obj->token_path[0] == '\0') { 5156 pr_debug("object '%s': token is prevented, skipping...\n", obj->name); 5157 return 0; 5158 } 5159 5160 mandatory = obj->token_path != NULL; 5161 level = mandatory ? LIBBPF_WARN : LIBBPF_DEBUG; 5162 5163 bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH; 5164 bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR); 5165 if (bpffs_fd < 0) { 5166 err = -errno; 5167 __pr(level, "object '%s': failed (%s) to open BPF FS mount at '%s'%s\n", 5168 obj->name, errstr(err), bpffs_path, 5169 mandatory ? "" : ", skipping optional step..."); 5170 return mandatory ? err : 0; 5171 } 5172 5173 token_fd = bpf_token_create(bpffs_fd, 0); 5174 close(bpffs_fd); 5175 if (token_fd < 0) { 5176 if (!mandatory && token_fd == -ENOENT) { 5177 pr_debug("object '%s': BPF FS at '%s' doesn't have BPF token delegation set up, skipping...\n", 5178 obj->name, bpffs_path); 5179 return 0; 5180 } 5181 __pr(level, "object '%s': failed (%d) to create BPF token from '%s'%s\n", 5182 obj->name, token_fd, bpffs_path, 5183 mandatory ? "" : ", skipping optional step..."); 5184 return mandatory ? token_fd : 0; 5185 } 5186 5187 obj->feat_cache = calloc(1, sizeof(*obj->feat_cache)); 5188 if (!obj->feat_cache) { 5189 close(token_fd); 5190 return -ENOMEM; 5191 } 5192 5193 obj->token_fd = token_fd; 5194 obj->feat_cache->token_fd = token_fd; 5195 5196 return 0; 5197 } 5198 5199 static int 5200 bpf_object__probe_loading(struct bpf_object *obj) 5201 { 5202 struct bpf_insn insns[] = { 5203 BPF_MOV64_IMM(BPF_REG_0, 0), 5204 BPF_EXIT_INSN(), 5205 }; 5206 int ret, insn_cnt = ARRAY_SIZE(insns); 5207 5208 if (obj->gen_loader || obj->token_fd) 5209 return 0; 5210 5211 ret = bump_rlimit_memlock(); 5212 if (ret) 5213 pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %s), you might need to do it explicitly!\n", 5214 errstr(ret)); 5215 5216 /* make sure basic loading works */ 5217 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); 5218 if (ret < 0) 5219 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); 5220 if (ret < 0) { 5221 ret = errno; 5222 pr_warn("Error in %s(): %s. Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.\n", 5223 __func__, errstr(ret)); 5224 return -ret; 5225 } 5226 close(ret); 5227 5228 return 0; 5229 } 5230 5231 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) 5232 { 5233 if (obj->gen_loader) 5234 /* To generate loader program assume the latest kernel 5235 * to avoid doing extra prog_load, map_create syscalls. 5236 */ 5237 return true; 5238 5239 if (obj->feat_cache) 5240 return feat_supported(obj->feat_cache, feat_id); 5241 5242 return feat_supported(NULL, feat_id); 5243 } 5244 5245 /* Used in testing to simulate missing features. */ 5246 void bpf_object_set_feat_cache(struct bpf_object *obj, struct kern_feature_cache *cache) 5247 { 5248 if (obj->feat_cache) 5249 free(obj->feat_cache); 5250 obj->feat_cache = cache; 5251 } 5252 5253 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) 5254 { 5255 struct bpf_map_info map_info; 5256 __u32 map_info_len = sizeof(map_info); 5257 int err; 5258 5259 memset(&map_info, 0, map_info_len); 5260 err = bpf_map_get_info_by_fd(map_fd, &map_info, &map_info_len); 5261 if (err && errno == EINVAL) 5262 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info); 5263 if (err) { 5264 pr_warn("failed to get map info for map FD %d: %s\n", map_fd, 5265 errstr(err)); 5266 return false; 5267 } 5268 5269 /* 5270 * bpf_get_map_info_by_fd() for DEVMAP will always return flags with 5271 * BPF_F_RDONLY_PROG set, but it generally is not set at map creation time. 5272 * Thus, ignore the BPF_F_RDONLY_PROG flag in the flags returned from 5273 * bpf_get_map_info_by_fd() when checking for compatibility with an 5274 * existing DEVMAP. 5275 */ 5276 if (map->def.type == BPF_MAP_TYPE_DEVMAP || map->def.type == BPF_MAP_TYPE_DEVMAP_HASH) 5277 map_info.map_flags &= ~BPF_F_RDONLY_PROG; 5278 5279 return (map_info.type == map->def.type && 5280 map_info.key_size == map->def.key_size && 5281 map_info.value_size == map->def.value_size && 5282 map_info.max_entries == map->def.max_entries && 5283 map_info.map_flags == map->def.map_flags && 5284 map_info.map_extra == map->map_extra); 5285 } 5286 5287 static int 5288 bpf_object__reuse_map(struct bpf_map *map) 5289 { 5290 int err, pin_fd; 5291 5292 pin_fd = bpf_obj_get(map->pin_path); 5293 if (pin_fd < 0) { 5294 err = -errno; 5295 if (err == -ENOENT) { 5296 pr_debug("found no pinned map to reuse at '%s'\n", 5297 map->pin_path); 5298 return 0; 5299 } 5300 5301 pr_warn("couldn't retrieve pinned map '%s': %s\n", 5302 map->pin_path, errstr(err)); 5303 return err; 5304 } 5305 5306 if (!map_is_reuse_compat(map, pin_fd)) { 5307 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", 5308 map->pin_path); 5309 close(pin_fd); 5310 return -EINVAL; 5311 } 5312 5313 err = bpf_map__reuse_fd(map, pin_fd); 5314 close(pin_fd); 5315 if (err) 5316 return err; 5317 5318 map->pinned = true; 5319 pr_debug("reused pinned map at '%s'\n", map->pin_path); 5320 5321 return 0; 5322 } 5323 5324 static int 5325 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 5326 { 5327 enum libbpf_map_type map_type = map->libbpf_type; 5328 bool is_percpu = map_type == LIBBPF_MAP_PERCPU; 5329 const __u64 update_flags = is_percpu ? BPF_F_ALL_CPUS : 0; 5330 int err, zero = 0; 5331 size_t mmap_sz; 5332 5333 if (obj->gen_loader) { 5334 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, 5335 map->mmaped, map->def.value_size, update_flags); 5336 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) 5337 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); 5338 return 0; 5339 } 5340 5341 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, update_flags); 5342 if (err) { 5343 err = -errno; 5344 pr_warn("map '%s': failed to set initial contents: %s\n", 5345 bpf_map__name(map), errstr(err)); 5346 return err; 5347 } 5348 5349 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ 5350 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { 5351 err = bpf_map_freeze(map->fd); 5352 if (err) { 5353 err = -errno; 5354 pr_warn("map '%s': failed to freeze as read-only: %s\n", 5355 bpf_map__name(map), errstr(err)); 5356 return err; 5357 } 5358 } 5359 5360 /* Remap anonymous mmap()-ed "map initialization image" as 5361 * a BPF map-backed mmap()-ed memory, but preserving the same 5362 * memory address. This will cause kernel to change process' 5363 * page table to point to a different piece of kernel memory, 5364 * but from userspace point of view memory address (and its 5365 * contents, being identical at this point) will stay the 5366 * same. This mapping will be released by bpf_object__close() 5367 * as per normal clean up procedure. 5368 */ 5369 mmap_sz = bpf_map_mmap_sz(map); 5370 if (map->def.map_flags & BPF_F_MMAPABLE) { 5371 void *mmaped; 5372 int prot; 5373 5374 if (map->def.map_flags & BPF_F_RDONLY_PROG) 5375 prot = PROT_READ; 5376 else 5377 prot = PROT_READ | PROT_WRITE; 5378 mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map->fd, 0); 5379 if (mmaped == MAP_FAILED) { 5380 err = -errno; 5381 pr_warn("map '%s': failed to re-mmap() contents: %s\n", 5382 bpf_map__name(map), errstr(err)); 5383 return err; 5384 } 5385 map->mmaped = mmaped; 5386 } else if (is_percpu) { 5387 if (mprotect(map->mmaped, mmap_sz, PROT_READ)) { 5388 err = -errno; 5389 pr_warn("map '%s': failed to mprotect() contents: %s\n", 5390 bpf_map__name(map), errstr(err)); 5391 return err; 5392 } 5393 } else if (map->mmaped) { 5394 munmap(map->mmaped, mmap_sz); 5395 map->mmaped = NULL; 5396 } 5397 5398 return 0; 5399 } 5400 5401 static void bpf_map__destroy(struct bpf_map *map); 5402 5403 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner) 5404 { 5405 LIBBPF_OPTS(bpf_map_create_opts, create_attr); 5406 struct bpf_map_def *def = &map->def; 5407 const char *map_name = NULL; 5408 int err = 0, map_fd; 5409 5410 if (kernel_supports(obj, FEAT_PROG_NAME)) 5411 map_name = map->name; 5412 create_attr.map_ifindex = map->map_ifindex; 5413 create_attr.map_flags = def->map_flags; 5414 create_attr.numa_node = map->numa_node; 5415 create_attr.map_extra = map->map_extra; 5416 create_attr.token_fd = obj->token_fd; 5417 if (obj->token_fd) 5418 create_attr.map_flags |= BPF_F_TOKEN_FD; 5419 if (map->excl_prog) { 5420 err = bpf_prog_compute_hash(map->excl_prog); 5421 if (err) 5422 return err; 5423 5424 create_attr.excl_prog_hash = map->excl_prog->hash; 5425 create_attr.excl_prog_hash_size = SHA256_DIGEST_LENGTH; 5426 } 5427 5428 if (bpf_map__is_struct_ops(map)) { 5429 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 5430 if (map->mod_btf_fd >= 0) { 5431 create_attr.value_type_btf_obj_fd = map->mod_btf_fd; 5432 create_attr.map_flags |= BPF_F_VTYPE_BTF_OBJ_FD; 5433 } 5434 } 5435 5436 if (obj->btf && btf__fd(obj->btf) >= 0) { 5437 create_attr.btf_fd = btf__fd(obj->btf); 5438 create_attr.btf_key_type_id = map->btf_key_type_id; 5439 create_attr.btf_value_type_id = map->btf_value_type_id; 5440 } 5441 5442 if (bpf_map_type__is_map_in_map(def->type)) { 5443 if (map->inner_map) { 5444 err = map_set_def_max_entries(map->inner_map); 5445 if (err) 5446 return err; 5447 err = bpf_object__create_map(obj, map->inner_map, true); 5448 if (err) { 5449 pr_warn("map '%s': failed to create inner map: %s\n", 5450 map->name, errstr(err)); 5451 return err; 5452 } 5453 map->inner_map_fd = map->inner_map->fd; 5454 } 5455 if (map->inner_map_fd >= 0) 5456 create_attr.inner_map_fd = map->inner_map_fd; 5457 } 5458 5459 switch (def->type) { 5460 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 5461 case BPF_MAP_TYPE_CGROUP_ARRAY: 5462 case BPF_MAP_TYPE_STACK_TRACE: 5463 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 5464 case BPF_MAP_TYPE_HASH_OF_MAPS: 5465 case BPF_MAP_TYPE_DEVMAP: 5466 case BPF_MAP_TYPE_DEVMAP_HASH: 5467 case BPF_MAP_TYPE_CPUMAP: 5468 case BPF_MAP_TYPE_XSKMAP: 5469 case BPF_MAP_TYPE_SOCKMAP: 5470 case BPF_MAP_TYPE_SOCKHASH: 5471 case BPF_MAP_TYPE_QUEUE: 5472 case BPF_MAP_TYPE_STACK: 5473 case BPF_MAP_TYPE_ARENA: 5474 create_attr.btf_fd = 0; 5475 create_attr.btf_key_type_id = 0; 5476 create_attr.btf_value_type_id = 0; 5477 map->btf_key_type_id = 0; 5478 map->btf_value_type_id = 0; 5479 break; 5480 case BPF_MAP_TYPE_STRUCT_OPS: 5481 create_attr.btf_value_type_id = 0; 5482 break; 5483 default: 5484 break; 5485 } 5486 5487 if (obj->gen_loader) { 5488 bpf_gen__map_create(obj->gen_loader, def->type, map_name, 5489 def->key_size, def->value_size, def->max_entries, 5490 &create_attr, is_inner ? -1 : map - obj->maps); 5491 /* We keep pretenting we have valid FD to pass various fd >= 0 5492 * checks by just keeping original placeholder FDs in place. 5493 * See bpf_object__add_map() comment. 5494 * This placeholder fd will not be used with any syscall and 5495 * will be reset to -1 eventually. 5496 */ 5497 map_fd = map->fd; 5498 } else { 5499 map_fd = bpf_map_create(def->type, map_name, 5500 def->key_size, def->value_size, 5501 def->max_entries, &create_attr); 5502 } 5503 if (map_fd < 0 && (create_attr.btf_key_type_id || create_attr.btf_value_type_id)) { 5504 err = -errno; 5505 pr_warn("Error in bpf_create_map_xattr(%s): %s. Retrying without BTF.\n", 5506 map->name, errstr(err)); 5507 create_attr.btf_fd = 0; 5508 create_attr.btf_key_type_id = 0; 5509 create_attr.btf_value_type_id = 0; 5510 map->btf_key_type_id = 0; 5511 map->btf_value_type_id = 0; 5512 map_fd = bpf_map_create(def->type, map_name, 5513 def->key_size, def->value_size, 5514 def->max_entries, &create_attr); 5515 } 5516 5517 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { 5518 if (obj->gen_loader) 5519 map->inner_map->fd = -1; 5520 bpf_map__destroy(map->inner_map); 5521 zfree(&map->inner_map); 5522 } 5523 5524 if (map_fd < 0) 5525 return map_fd; 5526 5527 /* obj->gen_loader case, prevent reuse_fd() from closing map_fd */ 5528 if (map->fd == map_fd) 5529 return 0; 5530 5531 /* Keep placeholder FD value but now point it to the BPF map object. 5532 * This way everything that relied on this map's FD (e.g., relocated 5533 * ldimm64 instructions) will stay valid and won't need adjustments. 5534 * map->fd stays valid but now point to what map_fd points to. 5535 */ 5536 return reuse_fd(map->fd, map_fd); 5537 } 5538 5539 static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) 5540 { 5541 const struct bpf_map *targ_map; 5542 unsigned int i; 5543 int fd, err = 0; 5544 5545 for (i = 0; i < map->init_slots_sz; i++) { 5546 if (!map->init_slots[i]) 5547 continue; 5548 5549 targ_map = map->init_slots[i]; 5550 fd = targ_map->fd; 5551 5552 if (obj->gen_loader) { 5553 bpf_gen__populate_outer_map(obj->gen_loader, 5554 map - obj->maps, i, 5555 targ_map - obj->maps); 5556 } else { 5557 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 5558 } 5559 if (err) { 5560 err = -errno; 5561 pr_warn("map '%s': failed to initialize slot [%u] to map '%s' fd=%d: %s\n", 5562 map->name, i, targ_map->name, fd, errstr(err)); 5563 return err; 5564 } 5565 pr_debug("map '%s': slot [%u] set to map '%s' fd=%d\n", 5566 map->name, i, targ_map->name, fd); 5567 } 5568 5569 zfree(&map->init_slots); 5570 map->init_slots_sz = 0; 5571 5572 return 0; 5573 } 5574 5575 static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) 5576 { 5577 const struct bpf_program *targ_prog; 5578 unsigned int i; 5579 int fd, err; 5580 5581 if (obj->gen_loader) 5582 return -ENOTSUP; 5583 5584 for (i = 0; i < map->init_slots_sz; i++) { 5585 if (!map->init_slots[i]) 5586 continue; 5587 5588 targ_prog = map->init_slots[i]; 5589 fd = bpf_program__fd(targ_prog); 5590 5591 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 5592 if (err) { 5593 err = -errno; 5594 pr_warn("map '%s': failed to initialize slot [%u] to prog '%s' fd=%d: %s\n", 5595 map->name, i, targ_prog->name, fd, errstr(err)); 5596 return err; 5597 } 5598 pr_debug("map '%s': slot [%u] set to prog '%s' fd=%d\n", 5599 map->name, i, targ_prog->name, fd); 5600 } 5601 5602 zfree(&map->init_slots); 5603 map->init_slots_sz = 0; 5604 5605 return 0; 5606 } 5607 5608 static int bpf_object_init_prog_arrays(struct bpf_object *obj) 5609 { 5610 struct bpf_map *map; 5611 int i, err; 5612 5613 for (i = 0; i < obj->nr_maps; i++) { 5614 map = &obj->maps[i]; 5615 5616 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY) 5617 continue; 5618 5619 err = init_prog_array_slots(obj, map); 5620 if (err < 0) 5621 return err; 5622 } 5623 return 0; 5624 } 5625 5626 static int map_set_def_max_entries(struct bpf_map *map) 5627 { 5628 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) { 5629 int nr_cpus; 5630 5631 nr_cpus = libbpf_num_possible_cpus(); 5632 if (nr_cpus < 0) { 5633 pr_warn("map '%s': failed to determine number of system CPUs: %d\n", 5634 map->name, nr_cpus); 5635 return nr_cpus; 5636 } 5637 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); 5638 map->def.max_entries = nr_cpus; 5639 } 5640 5641 return 0; 5642 } 5643 5644 static int 5645 bpf_object__create_maps(struct bpf_object *obj) 5646 { 5647 struct bpf_map *map; 5648 unsigned int i, j; 5649 int err; 5650 bool retried; 5651 5652 for (i = 0; i < obj->nr_maps; i++) { 5653 map = &obj->maps[i]; 5654 5655 /* To support old kernels, we skip creating global data maps 5656 * (.rodata, .data, .kconfig, etc); later on, during program 5657 * loading, if we detect that at least one of the to-be-loaded 5658 * programs is referencing any global data map, we'll error 5659 * out with program name and relocation index logged. 5660 * This approach allows to accommodate Clang emitting 5661 * unnecessary .rodata.str1.1 sections for string literals, 5662 * but also it allows to have CO-RE applications that use 5663 * global variables in some of BPF programs, but not others. 5664 * If those global variable-using programs are not loaded at 5665 * runtime due to bpf_program__set_autoload(prog, false), 5666 * bpf_object loading will succeed just fine even on old 5667 * kernels. 5668 * Same skipping applies to percpu data. 5669 */ 5670 if (bpf_map__is_internal(map)) { 5671 bool is_percpu = map->libbpf_type == LIBBPF_MAP_PERCPU; 5672 enum kern_feature_id feat_id; 5673 5674 feat_id = is_percpu ? FEAT_PERCPU_DATA : FEAT_GLOBAL_DATA; 5675 if (!kernel_supports(obj, feat_id)) 5676 map->autocreate = false; 5677 } 5678 5679 if (!map->autocreate) { 5680 pr_debug("map '%s': skipped auto-creating...\n", map->name); 5681 continue; 5682 } 5683 5684 err = map_set_def_max_entries(map); 5685 if (err) 5686 goto err_out; 5687 5688 retried = false; 5689 retry: 5690 if (map->pin_path) { 5691 err = bpf_object__reuse_map(map); 5692 if (err) { 5693 pr_warn("map '%s': error reusing pinned map\n", 5694 map->name); 5695 goto err_out; 5696 } 5697 if (retried && map->fd < 0) { 5698 pr_warn("map '%s': cannot find pinned map\n", 5699 map->name); 5700 err = -ENOENT; 5701 goto err_out; 5702 } 5703 } 5704 5705 if (map->reused) { 5706 pr_debug("map '%s': skipping creation (preset fd=%d)\n", 5707 map->name, map->fd); 5708 } else { 5709 err = bpf_object__create_map(obj, map, false); 5710 if (err) 5711 goto err_out; 5712 5713 pr_debug("map '%s': created successfully, fd=%d\n", 5714 map->name, map->fd); 5715 5716 if (bpf_map__is_internal(map)) { 5717 err = bpf_object__populate_internal_map(obj, map); 5718 if (err < 0) 5719 goto err_out; 5720 } else if (map->def.type == BPF_MAP_TYPE_ARENA) { 5721 map->mmaped = mmap((void *)(long)map->map_extra, 5722 bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, 5723 map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, 5724 map->fd, 0); 5725 if (map->mmaped == MAP_FAILED) { 5726 err = -errno; 5727 map->mmaped = NULL; 5728 pr_warn("map '%s': failed to mmap arena: %s\n", 5729 map->name, errstr(err)); 5730 return err; 5731 } 5732 if (obj->arena_data) { 5733 memcpy(map->mmaped + obj->arena_data_off, obj->arena_data, 5734 obj->arena_data_sz); 5735 zfree(&obj->arena_data); 5736 } 5737 } 5738 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { 5739 err = init_map_in_map_slots(obj, map); 5740 if (err < 0) 5741 goto err_out; 5742 } 5743 } 5744 5745 if (map->pin_path && !map->pinned) { 5746 err = bpf_map__pin(map, NULL); 5747 if (err) { 5748 if (!retried && err == -EEXIST) { 5749 retried = true; 5750 goto retry; 5751 } 5752 pr_warn("map '%s': failed to auto-pin at '%s': %s\n", 5753 map->name, map->pin_path, errstr(err)); 5754 goto err_out; 5755 } 5756 } 5757 } 5758 5759 return 0; 5760 5761 err_out: 5762 pr_warn("map '%s': failed to create: %s\n", map->name, errstr(err)); 5763 pr_perm_msg(err); 5764 for (j = 0; j < i; j++) 5765 zclose(obj->maps[j].fd); 5766 return err; 5767 } 5768 5769 static bool bpf_core_is_flavor_sep(const char *s) 5770 { 5771 /* check X___Y name pattern, where X and Y are not underscores */ 5772 return s[0] != '_' && /* X */ 5773 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 5774 s[4] != '_'; /* Y */ 5775 } 5776 5777 /* Given 'some_struct_name___with_flavor' return the length of a name prefix 5778 * before last triple underscore. Struct name part after last triple 5779 * underscore is ignored by BPF CO-RE relocation during relocation matching. 5780 */ 5781 size_t bpf_core_essential_name_len(const char *name) 5782 { 5783 size_t n = strlen(name); 5784 int i; 5785 5786 for (i = n - 5; i >= 0; i--) { 5787 if (bpf_core_is_flavor_sep(name + i)) 5788 return i + 1; 5789 } 5790 return n; 5791 } 5792 5793 void bpf_core_free_cands(struct bpf_core_cand_list *cands) 5794 { 5795 if (!cands) 5796 return; 5797 5798 free(cands->cands); 5799 free(cands); 5800 } 5801 5802 int bpf_core_add_cands(struct bpf_core_cand *local_cand, 5803 size_t local_essent_len, 5804 const struct btf *targ_btf, 5805 const char *targ_btf_name, 5806 int targ_start_id, 5807 struct bpf_core_cand_list *cands) 5808 { 5809 struct bpf_core_cand *new_cands, *cand; 5810 const struct btf_type *t, *local_t; 5811 const char *targ_name, *local_name; 5812 size_t targ_essent_len; 5813 int n, i; 5814 5815 local_t = btf__type_by_id(local_cand->btf, local_cand->id); 5816 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off); 5817 5818 n = btf__type_cnt(targ_btf); 5819 for (i = targ_start_id; i < n; i++) { 5820 t = btf__type_by_id(targ_btf, i); 5821 if (!btf_kind_core_compat(t, local_t)) 5822 continue; 5823 5824 targ_name = btf__name_by_offset(targ_btf, t->name_off); 5825 if (str_is_empty(targ_name)) 5826 continue; 5827 5828 targ_essent_len = bpf_core_essential_name_len(targ_name); 5829 if (targ_essent_len != local_essent_len) 5830 continue; 5831 5832 if (strncmp(local_name, targ_name, local_essent_len) != 0) 5833 continue; 5834 5835 pr_debug("CO-RE relocating [%u] %s %s: found target candidate [%d] %s %s in [%s]\n", 5836 local_cand->id, btf_kind_str(local_t), 5837 local_name, i, btf_kind_str(t), targ_name, 5838 targ_btf_name); 5839 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, 5840 sizeof(*cands->cands)); 5841 if (!new_cands) 5842 return -ENOMEM; 5843 5844 cand = &new_cands[cands->len]; 5845 cand->btf = targ_btf; 5846 cand->id = i; 5847 5848 cands->cands = new_cands; 5849 cands->len++; 5850 } 5851 return 0; 5852 } 5853 5854 static int load_module_btfs(struct bpf_object *obj) 5855 { 5856 struct bpf_btf_info info; 5857 struct module_btf *mod_btf; 5858 struct btf *btf; 5859 char name[64]; 5860 __u32 id = 0, len; 5861 int err, fd; 5862 5863 if (obj->btf_modules_loaded) 5864 return 0; 5865 5866 if (obj->gen_loader) 5867 return 0; 5868 5869 /* don't do this again, even if we find no module BTFs */ 5870 obj->btf_modules_loaded = true; 5871 5872 /* kernel too old to support module BTFs */ 5873 if (!kernel_supports(obj, FEAT_MODULE_BTF)) 5874 return 0; 5875 5876 while (true) { 5877 err = bpf_btf_get_next_id(id, &id); 5878 if (err && errno == ENOENT) 5879 return 0; 5880 if (err && errno == EPERM) { 5881 pr_debug("skipping module BTFs loading, missing privileges\n"); 5882 return 0; 5883 } 5884 if (err) { 5885 err = -errno; 5886 pr_warn("failed to iterate BTF objects: %s\n", errstr(err)); 5887 return err; 5888 } 5889 5890 fd = bpf_btf_get_fd_by_id(id); 5891 if (fd < 0) { 5892 if (errno == ENOENT) 5893 continue; /* expected race: BTF was unloaded */ 5894 err = -errno; 5895 pr_warn("failed to get BTF object #%u FD: %s\n", id, errstr(err)); 5896 return err; 5897 } 5898 5899 len = sizeof(info); 5900 memset(&info, 0, sizeof(info)); 5901 info.name = ptr_to_u64(name); 5902 info.name_len = sizeof(name); 5903 5904 btf = NULL; 5905 err = bpf_btf_get_info_by_fd(fd, &info, &len); 5906 if (err) { 5907 err = -errno; 5908 pr_warn("failed to get BTF object #%u info: %s\n", id, errstr(err)); 5909 break; 5910 } 5911 5912 /* ignore non-module BTFs */ 5913 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) { 5914 close(fd); 5915 continue; 5916 } 5917 5918 btf = btf_get_from_fd(fd, obj->btf_vmlinux); 5919 err = libbpf_get_error(btf); 5920 if (err) { 5921 pr_warn("failed to load module [%s]'s BTF object #%u: %s\n", 5922 name, id, errstr(err)); 5923 break; 5924 } 5925 5926 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, 5927 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); 5928 if (err) 5929 break; 5930 5931 mod_btf = &obj->btf_modules[obj->btf_module_cnt]; 5932 5933 mod_btf->btf = btf; 5934 mod_btf->id = id; 5935 mod_btf->fd = fd; 5936 mod_btf->name = strdup(name); 5937 if (!mod_btf->name) { 5938 err = -ENOMEM; 5939 break; 5940 } 5941 obj->btf_module_cnt++; 5942 } 5943 5944 if (err) { 5945 btf__free(btf); 5946 close(fd); 5947 } 5948 return err; 5949 } 5950 5951 static struct bpf_core_cand_list * 5952 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id) 5953 { 5954 struct bpf_core_cand local_cand = {}; 5955 struct bpf_core_cand_list *cands; 5956 const struct btf *main_btf; 5957 const struct btf_type *local_t; 5958 const char *local_name; 5959 size_t local_essent_len; 5960 int err, i; 5961 5962 local_cand.btf = local_btf; 5963 local_cand.id = local_type_id; 5964 local_t = btf__type_by_id(local_btf, local_type_id); 5965 if (!local_t) 5966 return ERR_PTR(-EINVAL); 5967 5968 local_name = btf__name_by_offset(local_btf, local_t->name_off); 5969 if (str_is_empty(local_name)) 5970 return ERR_PTR(-EINVAL); 5971 local_essent_len = bpf_core_essential_name_len(local_name); 5972 5973 cands = calloc(1, sizeof(*cands)); 5974 if (!cands) 5975 return ERR_PTR(-ENOMEM); 5976 5977 /* Attempt to find target candidates in vmlinux BTF first */ 5978 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; 5979 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands); 5980 if (err) 5981 goto err_out; 5982 5983 /* if vmlinux BTF has any candidate, don't got for module BTFs */ 5984 if (cands->len) 5985 return cands; 5986 5987 /* if vmlinux BTF was overridden, don't attempt to load module BTFs */ 5988 if (obj->btf_vmlinux_override) 5989 return cands; 5990 5991 /* now look through module BTFs, trying to still find candidates */ 5992 err = load_module_btfs(obj); 5993 if (err) 5994 goto err_out; 5995 5996 for (i = 0; i < obj->btf_module_cnt; i++) { 5997 err = bpf_core_add_cands(&local_cand, local_essent_len, 5998 obj->btf_modules[i].btf, 5999 obj->btf_modules[i].name, 6000 btf__type_cnt(obj->btf_vmlinux), 6001 cands); 6002 if (err) 6003 goto err_out; 6004 } 6005 6006 return cands; 6007 err_out: 6008 bpf_core_free_cands(cands); 6009 return ERR_PTR(err); 6010 } 6011 6012 /* Check local and target types for compatibility. This check is used for 6013 * type-based CO-RE relocations and follow slightly different rules than 6014 * field-based relocations. This function assumes that root types were already 6015 * checked for name match. Beyond that initial root-level name check, names 6016 * are completely ignored. Compatibility rules are as follows: 6017 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but 6018 * kind should match for local and target types (i.e., STRUCT is not 6019 * compatible with UNION); 6020 * - for ENUMs, the size is ignored; 6021 * - for INT, size and signedness are ignored; 6022 * - for ARRAY, dimensionality is ignored, element types are checked for 6023 * compatibility recursively; 6024 * - CONST/VOLATILE/RESTRICT modifiers are ignored; 6025 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; 6026 * - FUNC_PROTOs are compatible if they have compatible signature: same 6027 * number of input args and compatible return and argument types. 6028 * These rules are not set in stone and probably will be adjusted as we get 6029 * more experience with using BPF CO-RE relocations. 6030 */ 6031 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, 6032 const struct btf *targ_btf, __u32 targ_id) 6033 { 6034 return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32); 6035 } 6036 6037 int bpf_core_types_match(const struct btf *local_btf, __u32 local_id, 6038 const struct btf *targ_btf, __u32 targ_id) 6039 { 6040 return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32); 6041 } 6042 6043 static size_t bpf_core_hash_fn(const long key, void *ctx) 6044 { 6045 return key; 6046 } 6047 6048 static bool bpf_core_equal_fn(const long k1, const long k2, void *ctx) 6049 { 6050 return k1 == k2; 6051 } 6052 6053 static int record_relo_core(struct bpf_program *prog, 6054 const struct bpf_core_relo *core_relo, int insn_idx) 6055 { 6056 struct reloc_desc *relos, *relo; 6057 6058 relos = libbpf_reallocarray(prog->reloc_desc, 6059 prog->nr_reloc + 1, sizeof(*relos)); 6060 if (!relos) 6061 return -ENOMEM; 6062 relo = &relos[prog->nr_reloc]; 6063 relo->type = RELO_CORE; 6064 relo->insn_idx = insn_idx; 6065 relo->core_relo = core_relo; 6066 prog->reloc_desc = relos; 6067 prog->nr_reloc++; 6068 return 0; 6069 } 6070 6071 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx) 6072 { 6073 struct reloc_desc *relo; 6074 int i; 6075 6076 for (i = 0; i < prog->nr_reloc; i++) { 6077 relo = &prog->reloc_desc[i]; 6078 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx) 6079 continue; 6080 6081 return relo->core_relo; 6082 } 6083 6084 return NULL; 6085 } 6086 6087 static int bpf_core_resolve_relo(struct bpf_program *prog, 6088 const struct bpf_core_relo *relo, 6089 int relo_idx, 6090 const struct btf *local_btf, 6091 struct hashmap *cand_cache, 6092 struct bpf_core_relo_res *targ_res) 6093 { 6094 struct bpf_core_spec specs_scratch[3] = {}; 6095 struct bpf_core_cand_list *cands = NULL; 6096 const char *prog_name = prog->name; 6097 const struct btf_type *local_type; 6098 const char *local_name; 6099 __u32 local_id = relo->type_id; 6100 int err; 6101 6102 local_type = btf__type_by_id(local_btf, local_id); 6103 if (!local_type) 6104 return -EINVAL; 6105 6106 local_name = btf__name_by_offset(local_btf, local_type->name_off); 6107 if (!local_name) 6108 return -EINVAL; 6109 6110 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL && 6111 !hashmap__find(cand_cache, local_id, &cands)) { 6112 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); 6113 if (IS_ERR(cands)) { 6114 pr_warn("prog '%s': relo #%d: target candidate search failed for [%u] %s %s: %ld\n", 6115 prog_name, relo_idx, local_id, btf_kind_str(local_type), 6116 local_name, PTR_ERR(cands)); 6117 return PTR_ERR(cands); 6118 } 6119 err = hashmap__set(cand_cache, local_id, cands, NULL, NULL); 6120 if (err) { 6121 bpf_core_free_cands(cands); 6122 return err; 6123 } 6124 } 6125 6126 return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch, 6127 targ_res); 6128 } 6129 6130 static int 6131 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) 6132 { 6133 const struct btf_ext_info_sec *sec; 6134 struct bpf_core_relo_res targ_res; 6135 const struct bpf_core_relo *rec; 6136 const struct btf_ext_info *seg; 6137 struct hashmap_entry *entry; 6138 struct hashmap *cand_cache = NULL; 6139 struct bpf_program *prog; 6140 struct bpf_insn *insn; 6141 const char *sec_name; 6142 int i, err = 0, insn_idx, sec_idx, sec_num; 6143 6144 if (obj->btf_ext->core_relo_info.len == 0) 6145 return 0; 6146 6147 if (targ_btf_path) { 6148 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); 6149 err = libbpf_get_error(obj->btf_vmlinux_override); 6150 if (err) { 6151 pr_warn("failed to parse target BTF: %s\n", errstr(err)); 6152 return err; 6153 } 6154 } 6155 6156 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); 6157 if (IS_ERR(cand_cache)) { 6158 err = PTR_ERR(cand_cache); 6159 goto out; 6160 } 6161 6162 seg = &obj->btf_ext->core_relo_info; 6163 sec_num = 0; 6164 for_each_btf_ext_sec(seg, sec) { 6165 sec_idx = seg->sec_idxs[sec_num]; 6166 sec_num++; 6167 6168 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 6169 if (str_is_empty(sec_name)) { 6170 err = -EINVAL; 6171 goto out; 6172 } 6173 6174 pr_debug("sec '%s': found %u CO-RE relocations\n", sec_name, sec->num_info); 6175 6176 for_each_btf_ext_rec(seg, sec, i, rec) { 6177 if (rec->insn_off % BPF_INSN_SZ) 6178 return -EINVAL; 6179 insn_idx = rec->insn_off / BPF_INSN_SZ; 6180 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 6181 if (!prog) { 6182 /* When __weak subprog is "overridden" by another instance 6183 * of the subprog from a different object file, linker still 6184 * appends all the .BTF.ext info that used to belong to that 6185 * eliminated subprogram. 6186 * This is similar to what x86-64 linker does for relocations. 6187 * So just ignore such relocations just like we ignore 6188 * subprog instructions when discovering subprograms. 6189 */ 6190 pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n", 6191 sec_name, i, insn_idx); 6192 continue; 6193 } 6194 /* no need to apply CO-RE relocation if the program is 6195 * not going to be loaded 6196 */ 6197 if (!prog->autoload) 6198 continue; 6199 6200 /* adjust insn_idx from section frame of reference to the local 6201 * program's frame of reference; (sub-)program code is not yet 6202 * relocated, so it's enough to just subtract in-section offset 6203 */ 6204 insn_idx = insn_idx - prog->sec_insn_off; 6205 if (insn_idx >= prog->insns_cnt) 6206 return -EINVAL; 6207 insn = &prog->insns[insn_idx]; 6208 6209 if (is_ldimm64_insn(insn) && (size_t)insn_idx + 1 >= prog->insns_cnt) { 6210 pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) is truncated\n", 6211 prog->name, i, insn_idx); 6212 err = -EINVAL; 6213 goto out; 6214 } 6215 6216 err = record_relo_core(prog, rec, insn_idx); 6217 if (err) { 6218 pr_warn("prog '%s': relo #%d: failed to record relocation: %s\n", 6219 prog->name, i, errstr(err)); 6220 goto out; 6221 } 6222 6223 if (prog->obj->gen_loader) 6224 continue; 6225 6226 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res); 6227 if (err) { 6228 pr_warn("prog '%s': relo #%d: failed to relocate: %s\n", 6229 prog->name, i, errstr(err)); 6230 goto out; 6231 } 6232 6233 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); 6234 if (err) { 6235 pr_warn("prog '%s': relo #%d: failed to patch insn #%d: %s\n", 6236 prog->name, i, insn_idx, errstr(err)); 6237 goto out; 6238 } 6239 } 6240 } 6241 6242 out: 6243 /* obj->btf_vmlinux and module BTFs are freed after object load */ 6244 btf__free(obj->btf_vmlinux_override); 6245 obj->btf_vmlinux_override = NULL; 6246 6247 if (!IS_ERR_OR_NULL(cand_cache)) { 6248 hashmap__for_each_entry(cand_cache, entry, i) { 6249 bpf_core_free_cands(entry->pvalue); 6250 } 6251 hashmap__free(cand_cache); 6252 } 6253 return err; 6254 } 6255 6256 /* base map load ldimm64 special constant, used also for log fixup logic */ 6257 #define POISON_LDIMM64_MAP_BASE 2001000000 6258 #define POISON_LDIMM64_MAP_PFX "200100" 6259 6260 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx, 6261 int insn_idx, struct bpf_insn *insn, 6262 int map_idx, const struct bpf_map *map) 6263 { 6264 int i; 6265 6266 pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n", 6267 prog->name, relo_idx, insn_idx, map_idx, map->name); 6268 6269 /* we turn single ldimm64 into two identical invalid calls */ 6270 for (i = 0; i < 2; i++) { 6271 insn->code = BPF_JMP | BPF_CALL; 6272 insn->dst_reg = 0; 6273 insn->src_reg = 0; 6274 insn->off = 0; 6275 /* if this instruction is reachable (not a dead code), 6276 * verifier will complain with something like: 6277 * invalid func unknown#2001000123 6278 * where lower 123 is map index into obj->maps[] array 6279 */ 6280 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx; 6281 6282 insn++; 6283 } 6284 } 6285 6286 /* unresolved kfunc call special constant, used also for log fixup logic */ 6287 #define POISON_CALL_KFUNC_BASE 2002000000 6288 #define POISON_CALL_KFUNC_PFX "2002" 6289 6290 static void poison_kfunc_call(struct bpf_program *prog, int relo_idx, 6291 int insn_idx, struct bpf_insn *insn, 6292 int ext_idx, const struct extern_desc *ext) 6293 { 6294 pr_debug("prog '%s': relo #%d: poisoning insn #%d that calls kfunc '%s'\n", 6295 prog->name, relo_idx, insn_idx, ext->name); 6296 6297 /* we turn kfunc call into invalid helper call with identifiable constant */ 6298 insn->code = BPF_JMP | BPF_CALL; 6299 insn->dst_reg = 0; 6300 insn->src_reg = 0; 6301 insn->off = 0; 6302 /* if this instruction is reachable (not a dead code), 6303 * verifier will complain with something like: 6304 * invalid func unknown#2001000123 6305 * where lower 123 is extern index into obj->externs[] array 6306 */ 6307 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx; 6308 } 6309 6310 static int find_jt_map(struct bpf_object *obj, struct bpf_program *prog, unsigned int sym_off) 6311 { 6312 size_t i; 6313 6314 for (i = 0; i < obj->jumptable_map_cnt; i++) { 6315 /* 6316 * This might happen that same offset is used for two different 6317 * programs (as jump tables can be the same). However, for 6318 * different programs different maps should be created. 6319 */ 6320 if (obj->jumptable_maps[i].sym_off == sym_off && 6321 obj->jumptable_maps[i].prog == prog) 6322 return obj->jumptable_maps[i].fd; 6323 } 6324 6325 return -ENOENT; 6326 } 6327 6328 static int add_jt_map(struct bpf_object *obj, struct bpf_program *prog, unsigned int sym_off, int map_fd) 6329 { 6330 size_t cnt = obj->jumptable_map_cnt; 6331 size_t size = sizeof(obj->jumptable_maps[0]); 6332 void *tmp; 6333 6334 tmp = libbpf_reallocarray(obj->jumptable_maps, cnt + 1, size); 6335 if (!tmp) 6336 return -ENOMEM; 6337 6338 obj->jumptable_maps = tmp; 6339 obj->jumptable_maps[cnt].prog = prog; 6340 obj->jumptable_maps[cnt].sym_off = sym_off; 6341 obj->jumptable_maps[cnt].fd = map_fd; 6342 obj->jumptable_map_cnt++; 6343 6344 return 0; 6345 } 6346 6347 static int find_subprog_idx(struct bpf_program *prog, int insn_idx) 6348 { 6349 int i; 6350 6351 for (i = prog->subprog_cnt - 1; i >= 0; i--) { 6352 if (insn_idx >= prog->subprogs[i].sub_insn_off) 6353 return i; 6354 } 6355 6356 return -1; 6357 } 6358 6359 static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struct reloc_desc *relo) 6360 { 6361 const __u32 jt_entry_size = 8; 6362 unsigned int sym_off = relo->sym_off; 6363 int jt_size = relo->sym_size; 6364 __u32 max_entries = jt_size / jt_entry_size; 6365 __u32 value_size = sizeof(struct bpf_insn_array_value); 6366 struct bpf_insn_array_value val = {}; 6367 int subprog_idx; 6368 int map_fd, err; 6369 __u64 insn_off; 6370 __u64 *jt; 6371 __u32 i; 6372 6373 map_fd = find_jt_map(obj, prog, sym_off); 6374 if (map_fd >= 0) 6375 return map_fd; 6376 6377 if (sym_off % jt_entry_size) { 6378 pr_warn("map '.jumptables': jumptable start %u should be multiple of %u\n", 6379 sym_off, jt_entry_size); 6380 return -EINVAL; 6381 } 6382 6383 if (jt_size % jt_entry_size) { 6384 pr_warn("map '.jumptables': jumptable size %d should be multiple of %u\n", 6385 jt_size, jt_entry_size); 6386 return -EINVAL; 6387 } 6388 6389 map_fd = bpf_map_create(BPF_MAP_TYPE_INSN_ARRAY, ".jumptables", 6390 4, value_size, max_entries, NULL); 6391 if (map_fd < 0) 6392 return map_fd; 6393 6394 if (!obj->jumptables_data) { 6395 pr_warn("map '.jumptables': ELF file is missing jump table data\n"); 6396 err = -EINVAL; 6397 goto err_close; 6398 } 6399 if (sym_off + jt_size > obj->jumptables_data_sz) { 6400 pr_warn("map '.jumptables': jumptables_data size is %zu, trying to access %u\n", 6401 obj->jumptables_data_sz, sym_off + jt_size); 6402 err = -EINVAL; 6403 goto err_close; 6404 } 6405 6406 subprog_idx = -1; /* main program */ 6407 if (relo->insn_idx < 0 || relo->insn_idx >= prog->insns_cnt) { 6408 pr_warn("map '.jumptables': invalid instruction index %d\n", relo->insn_idx); 6409 err = -EINVAL; 6410 goto err_close; 6411 } 6412 if (prog->subprogs) 6413 subprog_idx = find_subprog_idx(prog, relo->insn_idx); 6414 6415 jt = (__u64 *)(obj->jumptables_data + sym_off); 6416 for (i = 0; i < max_entries; i++) { 6417 /* 6418 * The offset should be made to be relative to the beginning of 6419 * the main function, not the subfunction. 6420 */ 6421 insn_off = jt[i]/sizeof(struct bpf_insn); 6422 if (subprog_idx >= 0) { 6423 insn_off -= prog->subprogs[subprog_idx].sec_insn_off; 6424 insn_off += prog->subprogs[subprog_idx].sub_insn_off; 6425 } else { 6426 insn_off -= prog->sec_insn_off; 6427 } 6428 6429 /* 6430 * LLVM-generated jump tables contain u64 records, however 6431 * should contain values that fit in u32. 6432 */ 6433 if (insn_off > UINT32_MAX) { 6434 pr_warn("map '.jumptables': invalid jump table value 0x%llx at offset %u\n", 6435 (unsigned long long)jt[i], sym_off + i * jt_entry_size); 6436 err = -EINVAL; 6437 goto err_close; 6438 } 6439 6440 val.orig_off = insn_off; 6441 err = bpf_map_update_elem(map_fd, &i, &val, 0); 6442 if (err) 6443 goto err_close; 6444 } 6445 6446 err = bpf_map_freeze(map_fd); 6447 if (err) 6448 goto err_close; 6449 6450 err = add_jt_map(obj, prog, sym_off, map_fd); 6451 if (err) 6452 goto err_close; 6453 6454 return map_fd; 6455 6456 err_close: 6457 close(map_fd); 6458 return err; 6459 } 6460 6461 /* Relocate data references within program code: 6462 * - map references; 6463 * - global variable references; 6464 * - extern references. 6465 */ 6466 static int 6467 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) 6468 { 6469 int i; 6470 6471 for (i = 0; i < prog->nr_reloc; i++) { 6472 struct reloc_desc *relo = &prog->reloc_desc[i]; 6473 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 6474 const struct bpf_map *map; 6475 struct extern_desc *ext; 6476 6477 switch (relo->type) { 6478 case RELO_LD64: 6479 map = &obj->maps[relo->map_idx]; 6480 if (obj->gen_loader) { 6481 insn[0].src_reg = BPF_PSEUDO_MAP_IDX; 6482 insn[0].imm = relo->map_idx; 6483 } else if (map->autocreate) { 6484 insn[0].src_reg = BPF_PSEUDO_MAP_FD; 6485 insn[0].imm = map->fd; 6486 } else { 6487 poison_map_ldimm64(prog, i, relo->insn_idx, insn, 6488 relo->map_idx, map); 6489 } 6490 break; 6491 case RELO_DATA: 6492 map = &obj->maps[relo->map_idx]; 6493 insn[1].imm = insn[0].imm + relo->sym_off; 6494 6495 if (relo->map_idx == obj->arena_map_idx) 6496 insn[1].imm += obj->arena_data_off; 6497 6498 if (obj->gen_loader) { 6499 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 6500 insn[0].imm = relo->map_idx; 6501 } else if (map->autocreate) { 6502 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 6503 insn[0].imm = map->fd; 6504 } else { 6505 poison_map_ldimm64(prog, i, relo->insn_idx, insn, 6506 relo->map_idx, map); 6507 } 6508 break; 6509 case RELO_EXTERN_LD64: 6510 ext = &obj->externs[relo->ext_idx]; 6511 if (ext->type == EXT_KCFG) { 6512 if (obj->gen_loader) { 6513 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 6514 insn[0].imm = obj->kconfig_map_idx; 6515 } else { 6516 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 6517 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; 6518 } 6519 insn[1].imm = ext->kcfg.data_off; 6520 } else /* EXT_KSYM */ { 6521 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */ 6522 insn[0].src_reg = BPF_PSEUDO_BTF_ID; 6523 insn[0].imm = ext->ksym.kernel_btf_id; 6524 insn[1].imm = ext->ksym.kernel_btf_obj_fd; 6525 } else { /* typeless ksyms or unresolved typed ksyms */ 6526 insn[0].imm = (__u32)ext->ksym.addr; 6527 insn[1].imm = ext->ksym.addr >> 32; 6528 } 6529 } 6530 break; 6531 case RELO_EXTERN_CALL: 6532 ext = &obj->externs[relo->ext_idx]; 6533 insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL; 6534 if (ext->is_set) { 6535 insn[0].imm = ext->ksym.kernel_btf_id; 6536 insn[0].off = ext->ksym.btf_fd_idx; 6537 } else { /* unresolved weak kfunc call */ 6538 poison_kfunc_call(prog, i, relo->insn_idx, insn, 6539 relo->ext_idx, ext); 6540 } 6541 break; 6542 case RELO_SUBPROG_ADDR: 6543 if (insn[0].src_reg != BPF_PSEUDO_FUNC) { 6544 pr_warn("prog '%s': relo #%d: bad insn\n", 6545 prog->name, i); 6546 return -EINVAL; 6547 } 6548 /* handled already */ 6549 break; 6550 case RELO_CALL: 6551 /* handled already */ 6552 break; 6553 case RELO_CORE: 6554 /* will be handled by bpf_program_record_relos() */ 6555 break; 6556 case RELO_INSN_ARRAY: { 6557 int map_fd; 6558 6559 map_fd = create_jt_map(obj, prog, relo); 6560 if (map_fd < 0) { 6561 pr_warn("prog '%s': relo #%d: can't create jump table: sym_off %u\n", 6562 prog->name, i, relo->sym_off); 6563 return map_fd; 6564 } 6565 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 6566 insn->imm = map_fd; 6567 insn->off = 0; 6568 } 6569 break; 6570 default: 6571 pr_warn("prog '%s': relo #%d: bad relo type %u\n", 6572 prog->name, i, relo->type); 6573 return -EINVAL; 6574 } 6575 } 6576 6577 return 0; 6578 } 6579 6580 static int adjust_prog_btf_ext_info(const struct bpf_object *obj, 6581 const struct bpf_program *prog, 6582 const struct btf_ext_info *ext_info, 6583 void **prog_info, __u32 *prog_rec_cnt, 6584 __u32 *prog_rec_sz) 6585 { 6586 void *copy_start = NULL, *copy_end = NULL; 6587 void *rec, *rec_end, *new_prog_info; 6588 const struct btf_ext_info_sec *sec; 6589 size_t old_sz, new_sz; 6590 int i, sec_num, sec_idx, off_adj; 6591 6592 sec_num = 0; 6593 for_each_btf_ext_sec(ext_info, sec) { 6594 sec_idx = ext_info->sec_idxs[sec_num]; 6595 sec_num++; 6596 if (prog->sec_idx != sec_idx) 6597 continue; 6598 6599 for_each_btf_ext_rec(ext_info, sec, i, rec) { 6600 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ; 6601 6602 if (insn_off < prog->sec_insn_off) 6603 continue; 6604 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) 6605 break; 6606 6607 if (!copy_start) 6608 copy_start = rec; 6609 copy_end = rec + ext_info->rec_size; 6610 } 6611 6612 if (!copy_start) 6613 return -ENOENT; 6614 6615 /* append func/line info of a given (sub-)program to the main 6616 * program func/line info 6617 */ 6618 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size; 6619 new_sz = old_sz + (copy_end - copy_start); 6620 new_prog_info = realloc(*prog_info, new_sz); 6621 if (!new_prog_info) 6622 return -ENOMEM; 6623 *prog_info = new_prog_info; 6624 *prog_rec_cnt = new_sz / ext_info->rec_size; 6625 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start); 6626 6627 /* Kernel instruction offsets are in units of 8-byte 6628 * instructions, while .BTF.ext instruction offsets generated 6629 * by Clang are in units of bytes. So convert Clang offsets 6630 * into kernel offsets and adjust offset according to program 6631 * relocated position. 6632 */ 6633 off_adj = prog->sub_insn_off - prog->sec_insn_off; 6634 rec = new_prog_info + old_sz; 6635 rec_end = new_prog_info + new_sz; 6636 for (; rec < rec_end; rec += ext_info->rec_size) { 6637 __u32 *insn_off = rec; 6638 6639 *insn_off = *insn_off / BPF_INSN_SZ + off_adj; 6640 } 6641 *prog_rec_sz = ext_info->rec_size; 6642 return 0; 6643 } 6644 6645 return -ENOENT; 6646 } 6647 6648 static int 6649 reloc_prog_func_and_line_info(const struct bpf_object *obj, 6650 struct bpf_program *main_prog, 6651 const struct bpf_program *prog) 6652 { 6653 int err; 6654 6655 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't 6656 * support func/line info 6657 */ 6658 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC)) 6659 return 0; 6660 6661 /* only attempt func info relocation if main program's func_info 6662 * relocation was successful 6663 */ 6664 if (main_prog != prog && !main_prog->func_info) 6665 goto line_info; 6666 6667 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, 6668 &main_prog->func_info, 6669 &main_prog->func_info_cnt, 6670 &main_prog->func_info_rec_size); 6671 if (err) { 6672 if (err != -ENOENT) { 6673 pr_warn("prog '%s': error relocating .BTF.ext function info: %s\n", 6674 prog->name, errstr(err)); 6675 return err; 6676 } 6677 if (main_prog->func_info) { 6678 /* 6679 * Some info has already been found but has problem 6680 * in the last btf_ext reloc. Must have to error out. 6681 */ 6682 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); 6683 return err; 6684 } 6685 /* Have problem loading the very first info. Ignore the rest. */ 6686 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n", 6687 prog->name); 6688 } 6689 6690 line_info: 6691 /* don't relocate line info if main program's relocation failed */ 6692 if (main_prog != prog && !main_prog->line_info) 6693 return 0; 6694 6695 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, 6696 &main_prog->line_info, 6697 &main_prog->line_info_cnt, 6698 &main_prog->line_info_rec_size); 6699 if (err) { 6700 if (err != -ENOENT) { 6701 pr_warn("prog '%s': error relocating .BTF.ext line info: %s\n", 6702 prog->name, errstr(err)); 6703 return err; 6704 } 6705 if (main_prog->line_info) { 6706 /* 6707 * Some info has already been found but has problem 6708 * in the last btf_ext reloc. Must have to error out. 6709 */ 6710 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); 6711 return err; 6712 } 6713 /* Have problem loading the very first info. Ignore the rest. */ 6714 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n", 6715 prog->name); 6716 } 6717 return 0; 6718 } 6719 6720 static int cmp_relo_by_insn_idx(const void *key, const void *elem) 6721 { 6722 size_t insn_idx = *(const size_t *)key; 6723 const struct reloc_desc *relo = elem; 6724 6725 if (insn_idx == relo->insn_idx) 6726 return 0; 6727 return insn_idx < relo->insn_idx ? -1 : 1; 6728 } 6729 6730 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx) 6731 { 6732 if (!prog->nr_reloc) 6733 return NULL; 6734 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, 6735 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); 6736 } 6737 6738 static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog) 6739 { 6740 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc; 6741 struct reloc_desc *relos; 6742 int i; 6743 6744 if (main_prog == subprog) 6745 return 0; 6746 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); 6747 /* if new count is zero, reallocarray can return a valid NULL result; 6748 * in this case the previous pointer will be freed, so we *have to* 6749 * reassign old pointer to the new value (even if it's NULL) 6750 */ 6751 if (!relos && new_cnt) 6752 return -ENOMEM; 6753 if (subprog->nr_reloc) 6754 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, 6755 sizeof(*relos) * subprog->nr_reloc); 6756 6757 for (i = main_prog->nr_reloc; i < new_cnt; i++) 6758 relos[i].insn_idx += subprog->sub_insn_off; 6759 /* After insn_idx adjustment the 'relos' array is still sorted 6760 * by insn_idx and doesn't break bsearch. 6761 */ 6762 main_prog->reloc_desc = relos; 6763 main_prog->nr_reloc = new_cnt; 6764 return 0; 6765 } 6766 6767 static int save_subprog_offsets(struct bpf_program *main_prog, struct bpf_program *subprog) 6768 { 6769 size_t size = sizeof(main_prog->subprogs[0]); 6770 int cnt = main_prog->subprog_cnt; 6771 void *tmp; 6772 6773 tmp = libbpf_reallocarray(main_prog->subprogs, cnt + 1, size); 6774 if (!tmp) 6775 return -ENOMEM; 6776 6777 main_prog->subprogs = tmp; 6778 main_prog->subprogs[cnt].sec_insn_off = subprog->sec_insn_off; 6779 main_prog->subprogs[cnt].sub_insn_off = subprog->sub_insn_off; 6780 main_prog->subprog_cnt++; 6781 6782 return 0; 6783 } 6784 6785 static int 6786 bpf_object__append_subprog_code(struct bpf_object *obj, struct bpf_program *main_prog, 6787 struct bpf_program *subprog) 6788 { 6789 struct bpf_insn *insns; 6790 size_t new_cnt; 6791 int err; 6792 6793 subprog->sub_insn_off = main_prog->insns_cnt; 6794 6795 new_cnt = main_prog->insns_cnt + subprog->insns_cnt; 6796 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); 6797 if (!insns) { 6798 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); 6799 return -ENOMEM; 6800 } 6801 main_prog->insns = insns; 6802 main_prog->insns_cnt = new_cnt; 6803 6804 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, 6805 subprog->insns_cnt * sizeof(*insns)); 6806 6807 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", 6808 main_prog->name, subprog->insns_cnt, subprog->name); 6809 6810 /* The subprog insns are now appended. Append its relos too. */ 6811 err = append_subprog_relos(main_prog, subprog); 6812 if (err) 6813 return err; 6814 6815 err = save_subprog_offsets(main_prog, subprog); 6816 if (err) { 6817 pr_warn("prog '%s': failed to add subprog offsets: %s\n", 6818 main_prog->name, errstr(err)); 6819 return err; 6820 } 6821 6822 return 0; 6823 } 6824 6825 static int 6826 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, 6827 struct bpf_program *prog) 6828 { 6829 size_t sub_insn_idx, insn_idx; 6830 struct bpf_program *subprog; 6831 struct reloc_desc *relo; 6832 struct bpf_insn *insn; 6833 int err; 6834 6835 err = reloc_prog_func_and_line_info(obj, main_prog, prog); 6836 if (err) 6837 return err; 6838 6839 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { 6840 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 6841 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn)) 6842 continue; 6843 6844 relo = find_prog_insn_relo(prog, insn_idx); 6845 if (relo && relo->type == RELO_EXTERN_CALL) 6846 /* kfunc relocations will be handled later 6847 * in bpf_object__relocate_data() 6848 */ 6849 continue; 6850 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { 6851 pr_warn("prog '%s': unexpected relo for insn #%zu, type %u\n", 6852 prog->name, insn_idx, relo->type); 6853 return -LIBBPF_ERRNO__RELOC; 6854 } 6855 if (relo) { 6856 /* sub-program instruction index is a combination of 6857 * an offset of a symbol pointed to by relocation and 6858 * call instruction's imm field; for global functions, 6859 * call always has imm = -1, but for static functions 6860 * relocation is against STT_SECTION and insn->imm 6861 * points to a start of a static function 6862 * 6863 * for subprog addr relocation, the relo->sym_off + insn->imm is 6864 * the byte offset in the corresponding section. 6865 */ 6866 if (relo->type == RELO_CALL) 6867 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; 6868 else 6869 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; 6870 } else if (insn_is_pseudo_func(insn)) { 6871 /* 6872 * RELO_SUBPROG_ADDR relo is always emitted even if both 6873 * functions are in the same section, so it shouldn't reach here. 6874 */ 6875 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n", 6876 prog->name, insn_idx); 6877 return -LIBBPF_ERRNO__RELOC; 6878 } else { 6879 /* if subprogram call is to a static function within 6880 * the same ELF section, there won't be any relocation 6881 * emitted, but it also means there is no additional 6882 * offset necessary, insns->imm is relative to 6883 * instruction's original position within the section 6884 */ 6885 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; 6886 } 6887 6888 /* we enforce that sub-programs should be in .text section */ 6889 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); 6890 if (!subprog) { 6891 pr_warn("prog '%s': no .text section found yet sub-program call exists\n", 6892 prog->name); 6893 return -LIBBPF_ERRNO__RELOC; 6894 } 6895 6896 /* if it's the first call instruction calling into this 6897 * subprogram (meaning this subprog hasn't been processed 6898 * yet) within the context of current main program: 6899 * - append it at the end of main program's instructions blog; 6900 * - process is recursively, while current program is put on hold; 6901 * - if that subprogram calls some other not yet processes 6902 * subprogram, same thing will happen recursively until 6903 * there are no more unprocesses subprograms left to append 6904 * and relocate. 6905 */ 6906 if (subprog->sub_insn_off == 0) { 6907 err = bpf_object__append_subprog_code(obj, main_prog, subprog); 6908 if (err) 6909 return err; 6910 err = bpf_object__reloc_code(obj, main_prog, subprog); 6911 if (err) 6912 return err; 6913 } 6914 6915 /* main_prog->insns memory could have been re-allocated, so 6916 * calculate pointer again 6917 */ 6918 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 6919 /* calculate correct instruction position within current main 6920 * prog; each main prog can have a different set of 6921 * subprograms appended (potentially in different order as 6922 * well), so position of any subprog can be different for 6923 * different main programs 6924 */ 6925 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; 6926 6927 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n", 6928 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); 6929 } 6930 6931 return 0; 6932 } 6933 6934 /* 6935 * Relocate sub-program calls. 6936 * 6937 * Algorithm operates as follows. Each entry-point BPF program (referred to as 6938 * main prog) is processed separately. For each subprog (non-entry functions, 6939 * that can be called from either entry progs or other subprogs) gets their 6940 * sub_insn_off reset to zero. This serves as indicator that this subprogram 6941 * hasn't been yet appended and relocated within current main prog. Once its 6942 * relocated, sub_insn_off will point at the position within current main prog 6943 * where given subprog was appended. This will further be used to relocate all 6944 * the call instructions jumping into this subprog. 6945 * 6946 * We start with main program and process all call instructions. If the call 6947 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off 6948 * is zero), subprog instructions are appended at the end of main program's 6949 * instruction array. Then main program is "put on hold" while we recursively 6950 * process newly appended subprogram. If that subprogram calls into another 6951 * subprogram that hasn't been appended, new subprogram is appended again to 6952 * the *main* prog's instructions (subprog's instructions are always left 6953 * untouched, as they need to be in unmodified state for subsequent main progs 6954 * and subprog instructions are always sent only as part of a main prog) and 6955 * the process continues recursively. Once all the subprogs called from a main 6956 * prog or any of its subprogs are appended (and relocated), all their 6957 * positions within finalized instructions array are known, so it's easy to 6958 * rewrite call instructions with correct relative offsets, corresponding to 6959 * desired target subprog. 6960 * 6961 * Its important to realize that some subprogs might not be called from some 6962 * main prog and any of its called/used subprogs. Those will keep their 6963 * subprog->sub_insn_off as zero at all times and won't be appended to current 6964 * main prog and won't be relocated within the context of current main prog. 6965 * They might still be used from other main progs later. 6966 * 6967 * Visually this process can be shown as below. Suppose we have two main 6968 * programs mainA and mainB and BPF object contains three subprogs: subA, 6969 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and 6970 * subC both call subB: 6971 * 6972 * +--------+ +-------+ 6973 * | v v | 6974 * +--+---+ +--+-+-+ +---+--+ 6975 * | subA | | subB | | subC | 6976 * +--+---+ +------+ +---+--+ 6977 * ^ ^ 6978 * | | 6979 * +---+-------+ +------+----+ 6980 * | mainA | | mainB | 6981 * +-----------+ +-----------+ 6982 * 6983 * We'll start relocating mainA, will find subA, append it and start 6984 * processing sub A recursively: 6985 * 6986 * +-----------+------+ 6987 * | mainA | subA | 6988 * +-----------+------+ 6989 * 6990 * At this point we notice that subB is used from subA, so we append it and 6991 * relocate (there are no further subcalls from subB): 6992 * 6993 * +-----------+------+------+ 6994 * | mainA | subA | subB | 6995 * +-----------+------+------+ 6996 * 6997 * At this point, we relocate subA calls, then go one level up and finish with 6998 * relocation mainA calls. mainA is done. 6999 * 7000 * For mainB process is similar but results in different order. We start with 7001 * mainB and skip subA and subB, as mainB never calls them (at least 7002 * directly), but we see subC is needed, so we append and start processing it: 7003 * 7004 * +-----------+------+ 7005 * | mainB | subC | 7006 * +-----------+------+ 7007 * Now we see subC needs subB, so we go back to it, append and relocate it: 7008 * 7009 * +-----------+------+------+ 7010 * | mainB | subC | subB | 7011 * +-----------+------+------+ 7012 * 7013 * At this point we unwind recursion, relocate calls in subC, then in mainB. 7014 */ 7015 static int 7016 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) 7017 { 7018 struct bpf_program *subprog; 7019 int i, err; 7020 7021 /* mark all subprogs as not relocated (yet) within the context of 7022 * current main program 7023 */ 7024 for (i = 0; i < obj->nr_programs; i++) { 7025 subprog = &obj->programs[i]; 7026 if (!prog_is_subprog(obj, subprog)) 7027 continue; 7028 7029 subprog->sub_insn_off = 0; 7030 } 7031 7032 err = bpf_object__reloc_code(obj, prog, prog); 7033 if (err) 7034 return err; 7035 7036 return 0; 7037 } 7038 7039 static void 7040 bpf_object__free_relocs(struct bpf_object *obj) 7041 { 7042 struct bpf_program *prog; 7043 int i; 7044 7045 /* free up relocation descriptors */ 7046 for (i = 0; i < obj->nr_programs; i++) { 7047 prog = &obj->programs[i]; 7048 zfree(&prog->reloc_desc); 7049 prog->nr_reloc = 0; 7050 } 7051 } 7052 7053 static int cmp_relocs(const void *_a, const void *_b) 7054 { 7055 const struct reloc_desc *a = _a; 7056 const struct reloc_desc *b = _b; 7057 7058 if (a->insn_idx != b->insn_idx) 7059 return a->insn_idx < b->insn_idx ? -1 : 1; 7060 7061 /* no two relocations should have the same insn_idx, but ... */ 7062 if (a->type != b->type) 7063 return a->type < b->type ? -1 : 1; 7064 7065 return 0; 7066 } 7067 7068 static void bpf_object__sort_relos(struct bpf_object *obj) 7069 { 7070 int i; 7071 7072 for (i = 0; i < obj->nr_programs; i++) { 7073 struct bpf_program *p = &obj->programs[i]; 7074 7075 if (!p->nr_reloc) 7076 continue; 7077 7078 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); 7079 } 7080 } 7081 7082 static int bpf_prog_assign_exc_cb(struct bpf_object *obj, struct bpf_program *prog) 7083 { 7084 const char *str = "exception_callback:"; 7085 size_t pfx_len = strlen(str); 7086 int i, j, n; 7087 7088 if (!obj->btf || !kernel_supports(obj, FEAT_BTF_DECL_TAG)) 7089 return 0; 7090 7091 n = btf__type_cnt(obj->btf); 7092 for (i = 1; i < n; i++) { 7093 const char *name; 7094 struct btf_type *t; 7095 7096 t = btf_type_by_id(obj->btf, i); 7097 if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1) 7098 continue; 7099 7100 name = btf__str_by_offset(obj->btf, t->name_off); 7101 if (strncmp(name, str, pfx_len) != 0) 7102 continue; 7103 7104 t = btf_type_by_id(obj->btf, t->type); 7105 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) { 7106 pr_warn("prog '%s': exception_callback:<value> decl tag not applied to the main program\n", 7107 prog->name); 7108 return -EINVAL; 7109 } 7110 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0) 7111 continue; 7112 /* Multiple callbacks are specified for the same prog, 7113 * the verifier will eventually return an error for this 7114 * case, hence simply skip appending a subprog. 7115 */ 7116 if (prog->exception_cb_idx >= 0) { 7117 prog->exception_cb_idx = -1; 7118 break; 7119 } 7120 7121 name += pfx_len; 7122 if (str_is_empty(name)) { 7123 pr_warn("prog '%s': exception_callback:<value> decl tag contains empty value\n", 7124 prog->name); 7125 return -EINVAL; 7126 } 7127 7128 for (j = 0; j < obj->nr_programs; j++) { 7129 struct bpf_program *subprog = &obj->programs[j]; 7130 7131 if (!prog_is_subprog(obj, subprog)) 7132 continue; 7133 if (strcmp(name, subprog->name) != 0) 7134 continue; 7135 /* Enforce non-hidden, as from verifier point of 7136 * view it expects global functions, whereas the 7137 * mark_btf_static fixes up linkage as static. 7138 */ 7139 if (!subprog->sym_global || subprog->mark_btf_static) { 7140 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n", 7141 prog->name, subprog->name); 7142 return -EINVAL; 7143 } 7144 /* Let's see if we already saw a static exception callback with the same name */ 7145 if (prog->exception_cb_idx >= 0) { 7146 pr_warn("prog '%s': multiple subprogs with same name as exception callback '%s'\n", 7147 prog->name, subprog->name); 7148 return -EINVAL; 7149 } 7150 prog->exception_cb_idx = j; 7151 break; 7152 } 7153 7154 if (prog->exception_cb_idx >= 0) 7155 continue; 7156 7157 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name); 7158 return -ENOENT; 7159 } 7160 7161 return 0; 7162 } 7163 7164 static struct { 7165 enum bpf_prog_type prog_type; 7166 const char *ctx_name; 7167 } global_ctx_map[] = { 7168 { BPF_PROG_TYPE_CGROUP_DEVICE, "bpf_cgroup_dev_ctx" }, 7169 { BPF_PROG_TYPE_CGROUP_SKB, "__sk_buff" }, 7170 { BPF_PROG_TYPE_CGROUP_SOCK, "bpf_sock" }, 7171 { BPF_PROG_TYPE_CGROUP_SOCK_ADDR, "bpf_sock_addr" }, 7172 { BPF_PROG_TYPE_CGROUP_SOCKOPT, "bpf_sockopt" }, 7173 { BPF_PROG_TYPE_CGROUP_SYSCTL, "bpf_sysctl" }, 7174 { BPF_PROG_TYPE_FLOW_DISSECTOR, "__sk_buff" }, 7175 { BPF_PROG_TYPE_KPROBE, "bpf_user_pt_regs_t" }, 7176 { BPF_PROG_TYPE_LWT_IN, "__sk_buff" }, 7177 { BPF_PROG_TYPE_LWT_OUT, "__sk_buff" }, 7178 { BPF_PROG_TYPE_LWT_SEG6LOCAL, "__sk_buff" }, 7179 { BPF_PROG_TYPE_LWT_XMIT, "__sk_buff" }, 7180 { BPF_PROG_TYPE_NETFILTER, "bpf_nf_ctx" }, 7181 { BPF_PROG_TYPE_PERF_EVENT, "bpf_perf_event_data" }, 7182 { BPF_PROG_TYPE_RAW_TRACEPOINT, "bpf_raw_tracepoint_args" }, 7183 { BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, "bpf_raw_tracepoint_args" }, 7184 { BPF_PROG_TYPE_SCHED_ACT, "__sk_buff" }, 7185 { BPF_PROG_TYPE_SCHED_CLS, "__sk_buff" }, 7186 { BPF_PROG_TYPE_SK_LOOKUP, "bpf_sk_lookup" }, 7187 { BPF_PROG_TYPE_SK_MSG, "sk_msg_md" }, 7188 { BPF_PROG_TYPE_SK_REUSEPORT, "sk_reuseport_md" }, 7189 { BPF_PROG_TYPE_SK_SKB, "__sk_buff" }, 7190 { BPF_PROG_TYPE_SOCK_OPS, "bpf_sock_ops" }, 7191 { BPF_PROG_TYPE_SOCKET_FILTER, "__sk_buff" }, 7192 { BPF_PROG_TYPE_XDP, "xdp_md" }, 7193 /* all other program types don't have "named" context structs */ 7194 }; 7195 7196 /* forward declarations for arch-specific underlying types of bpf_user_pt_regs_t typedef, 7197 * for below __builtin_types_compatible_p() checks; 7198 * with this approach we don't need any extra arch-specific #ifdef guards 7199 */ 7200 struct pt_regs; 7201 struct user_pt_regs; 7202 struct user_regs_struct; 7203 7204 static bool need_func_arg_type_fixup(const struct btf *btf, const struct bpf_program *prog, 7205 const char *subprog_name, int arg_idx, 7206 int arg_type_id, const char *ctx_name) 7207 { 7208 const struct btf_type *t; 7209 const char *tname; 7210 7211 /* check if existing parameter already matches verifier expectations */ 7212 t = skip_mods_and_typedefs(btf, arg_type_id, NULL); 7213 if (!btf_is_ptr(t)) 7214 goto out_warn; 7215 7216 /* typedef bpf_user_pt_regs_t is a special PITA case, valid for kprobe 7217 * and perf_event programs, so check this case early on and forget 7218 * about it for subsequent checks 7219 */ 7220 while (btf_is_mod(t)) 7221 t = btf__type_by_id(btf, t->type); 7222 if (btf_is_typedef(t) && 7223 (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) { 7224 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>"; 7225 if (strcmp(tname, "bpf_user_pt_regs_t") == 0) 7226 return false; /* canonical type for kprobe/perf_event */ 7227 } 7228 7229 /* now we can ignore typedefs moving forward */ 7230 t = skip_mods_and_typedefs(btf, t->type, NULL); 7231 7232 /* if it's `void *`, definitely fix up BTF info */ 7233 if (btf_is_void(t)) 7234 return true; 7235 7236 /* if it's already proper canonical type, no need to fix up */ 7237 tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>"; 7238 if (btf_is_struct(t) && strcmp(tname, ctx_name) == 0) 7239 return false; 7240 7241 /* special cases */ 7242 switch (prog->type) { 7243 case BPF_PROG_TYPE_KPROBE: 7244 /* `struct pt_regs *` is expected, but we need to fix up */ 7245 if (btf_is_struct(t) && strcmp(tname, "pt_regs") == 0) 7246 return true; 7247 break; 7248 case BPF_PROG_TYPE_PERF_EVENT: 7249 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct pt_regs) && 7250 btf_is_struct(t) && strcmp(tname, "pt_regs") == 0) 7251 return true; 7252 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_pt_regs) && 7253 btf_is_struct(t) && strcmp(tname, "user_pt_regs") == 0) 7254 return true; 7255 if (__builtin_types_compatible_p(bpf_user_pt_regs_t, struct user_regs_struct) && 7256 btf_is_struct(t) && strcmp(tname, "user_regs_struct") == 0) 7257 return true; 7258 break; 7259 case BPF_PROG_TYPE_RAW_TRACEPOINT: 7260 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 7261 /* allow u64* as ctx */ 7262 if (btf_is_int(t) && t->size == 8) 7263 return true; 7264 break; 7265 default: 7266 break; 7267 } 7268 7269 out_warn: 7270 pr_warn("prog '%s': subprog '%s' arg#%d is expected to be of `struct %s *` type\n", 7271 prog->name, subprog_name, arg_idx, ctx_name); 7272 return false; 7273 } 7274 7275 static int clone_func_btf_info(struct btf *btf, int orig_fn_id, struct bpf_program *prog) 7276 { 7277 int fn_id, fn_proto_id, ret_type_id, orig_proto_id; 7278 int i, err, arg_cnt, fn_name_off, linkage; 7279 struct btf_type *fn_t, *fn_proto_t, *t; 7280 struct btf_param *p; 7281 7282 /* caller already validated FUNC -> FUNC_PROTO validity */ 7283 fn_t = btf_type_by_id(btf, orig_fn_id); 7284 fn_proto_t = btf_type_by_id(btf, fn_t->type); 7285 7286 /* Note that each btf__add_xxx() operation invalidates 7287 * all btf_type and string pointers, so we need to be 7288 * very careful when cloning BTF types. BTF type 7289 * pointers have to be always refetched. And to avoid 7290 * problems with invalidated string pointers, we 7291 * add empty strings initially, then just fix up 7292 * name_off offsets in place. Offsets are stable for 7293 * existing strings, so that works out. 7294 */ 7295 fn_name_off = fn_t->name_off; /* we are about to invalidate fn_t */ 7296 linkage = btf_func_linkage(fn_t); 7297 orig_proto_id = fn_t->type; /* original FUNC_PROTO ID */ 7298 ret_type_id = fn_proto_t->type; /* fn_proto_t will be invalidated */ 7299 arg_cnt = btf_vlen(fn_proto_t); 7300 7301 /* clone FUNC_PROTO and its params */ 7302 fn_proto_id = btf__add_func_proto(btf, ret_type_id); 7303 if (fn_proto_id < 0) 7304 return -EINVAL; 7305 7306 for (i = 0; i < arg_cnt; i++) { 7307 int name_off; 7308 7309 /* copy original parameter data */ 7310 t = btf_type_by_id(btf, orig_proto_id); 7311 p = &btf_params(t)[i]; 7312 name_off = p->name_off; 7313 7314 err = btf__add_func_param(btf, "", p->type); 7315 if (err) 7316 return err; 7317 7318 fn_proto_t = btf_type_by_id(btf, fn_proto_id); 7319 p = &btf_params(fn_proto_t)[i]; 7320 p->name_off = name_off; /* use remembered str offset */ 7321 } 7322 7323 /* clone FUNC now, btf__add_func() enforces non-empty name, so use 7324 * entry program's name as a placeholder, which we replace immediately 7325 * with original name_off 7326 */ 7327 fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id); 7328 if (fn_id < 0) 7329 return -EINVAL; 7330 7331 fn_t = btf_type_by_id(btf, fn_id); 7332 fn_t->name_off = fn_name_off; /* reuse original string */ 7333 7334 return fn_id; 7335 } 7336 7337 /* Check if main program or global subprog's function prototype has `arg:ctx` 7338 * argument tags, and, if necessary, substitute correct type to match what BPF 7339 * verifier would expect, taking into account specific program type. This 7340 * allows to support __arg_ctx tag transparently on old kernels that don't yet 7341 * have a native support for it in the verifier, making user's life much 7342 * easier. 7343 */ 7344 static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_program *prog) 7345 { 7346 const char *ctx_name = NULL, *ctx_tag = "arg:ctx", *fn_name; 7347 struct bpf_func_info_min *func_rec; 7348 struct btf_type *fn_t, *fn_proto_t; 7349 struct btf *btf = obj->btf; 7350 const struct btf_type *t; 7351 struct btf_param *p; 7352 int ptr_id = 0, struct_id, tag_id, orig_fn_id; 7353 int i, n, arg_idx, arg_cnt, err, rec_idx; 7354 int *orig_ids; 7355 7356 /* no .BTF.ext, no problem */ 7357 if (!obj->btf_ext || !prog->func_info) 7358 return 0; 7359 7360 /* don't do any fix ups if kernel natively supports __arg_ctx */ 7361 if (kernel_supports(obj, FEAT_ARG_CTX_TAG)) 7362 return 0; 7363 7364 /* some BPF program types just don't have named context structs, so 7365 * this fallback mechanism doesn't work for them 7366 */ 7367 for (i = 0; i < ARRAY_SIZE(global_ctx_map); i++) { 7368 if (global_ctx_map[i].prog_type != prog->type) 7369 continue; 7370 ctx_name = global_ctx_map[i].ctx_name; 7371 break; 7372 } 7373 if (!ctx_name) 7374 return 0; 7375 7376 /* remember original func BTF IDs to detect if we already cloned them */ 7377 orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids)); 7378 if (!orig_ids) 7379 return -ENOMEM; 7380 for (i = 0; i < prog->func_info_cnt; i++) { 7381 func_rec = prog->func_info + prog->func_info_rec_size * i; 7382 orig_ids[i] = func_rec->type_id; 7383 } 7384 7385 /* go through each DECL_TAG with "arg:ctx" and see if it points to one 7386 * of our subprogs; if yes and subprog is global and needs adjustment, 7387 * clone and adjust FUNC -> FUNC_PROTO combo 7388 */ 7389 for (i = 1, n = btf__type_cnt(btf); i < n; i++) { 7390 /* only DECL_TAG with "arg:ctx" value are interesting */ 7391 t = btf__type_by_id(btf, i); 7392 if (!btf_is_decl_tag(t)) 7393 continue; 7394 if (strcmp(btf__str_by_offset(btf, t->name_off), ctx_tag) != 0) 7395 continue; 7396 7397 /* only global funcs need adjustment, if at all */ 7398 orig_fn_id = t->type; 7399 fn_t = btf_type_by_id(btf, orig_fn_id); 7400 if (!btf_is_func(fn_t) || btf_func_linkage(fn_t) != BTF_FUNC_GLOBAL) 7401 continue; 7402 7403 /* sanity check FUNC -> FUNC_PROTO chain, just in case */ 7404 fn_proto_t = btf_type_by_id(btf, fn_t->type); 7405 if (!fn_proto_t || !btf_is_func_proto(fn_proto_t)) 7406 continue; 7407 7408 /* find corresponding func_info record */ 7409 func_rec = NULL; 7410 for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) { 7411 if (orig_ids[rec_idx] == t->type) { 7412 func_rec = prog->func_info + prog->func_info_rec_size * rec_idx; 7413 break; 7414 } 7415 } 7416 /* current main program doesn't call into this subprog */ 7417 if (!func_rec) 7418 continue; 7419 7420 /* some more sanity checking of DECL_TAG */ 7421 arg_cnt = btf_vlen(fn_proto_t); 7422 arg_idx = btf_decl_tag(t)->component_idx; 7423 if (arg_idx < 0 || arg_idx >= arg_cnt) 7424 continue; 7425 7426 /* check if we should fix up argument type */ 7427 p = &btf_params(fn_proto_t)[arg_idx]; 7428 fn_name = btf__str_by_offset(btf, fn_t->name_off) ?: "<anon>"; 7429 if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name)) 7430 continue; 7431 7432 /* clone fn/fn_proto, unless we already did it for another arg */ 7433 if (func_rec->type_id == orig_fn_id) { 7434 int fn_id; 7435 7436 fn_id = clone_func_btf_info(btf, orig_fn_id, prog); 7437 if (fn_id < 0) { 7438 err = fn_id; 7439 goto err_out; 7440 } 7441 7442 /* point func_info record to a cloned FUNC type */ 7443 func_rec->type_id = fn_id; 7444 } 7445 7446 /* create PTR -> STRUCT type chain to mark PTR_TO_CTX argument; 7447 * we do it just once per main BPF program, as all global 7448 * funcs share the same program type, so need only PTR -> 7449 * STRUCT type chain 7450 */ 7451 if (ptr_id == 0) { 7452 struct_id = btf__add_struct(btf, ctx_name, 0); 7453 ptr_id = btf__add_ptr(btf, struct_id); 7454 if (ptr_id < 0 || struct_id < 0) { 7455 err = -EINVAL; 7456 goto err_out; 7457 } 7458 } 7459 7460 /* for completeness, clone DECL_TAG and point it to cloned param */ 7461 tag_id = btf__add_decl_tag(btf, ctx_tag, func_rec->type_id, arg_idx); 7462 if (tag_id < 0) { 7463 err = -EINVAL; 7464 goto err_out; 7465 } 7466 7467 /* all the BTF manipulations invalidated pointers, refetch them */ 7468 fn_t = btf_type_by_id(btf, func_rec->type_id); 7469 fn_proto_t = btf_type_by_id(btf, fn_t->type); 7470 7471 /* fix up type ID pointed to by param */ 7472 p = &btf_params(fn_proto_t)[arg_idx]; 7473 p->type = ptr_id; 7474 } 7475 7476 free(orig_ids); 7477 return 0; 7478 err_out: 7479 free(orig_ids); 7480 return err; 7481 } 7482 7483 static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) 7484 { 7485 struct bpf_program *prog; 7486 size_t i, j; 7487 int err; 7488 7489 if (obj->btf_ext) { 7490 err = bpf_object__relocate_core(obj, targ_btf_path); 7491 if (err) { 7492 pr_warn("failed to perform CO-RE relocations: %s\n", 7493 errstr(err)); 7494 return err; 7495 } 7496 bpf_object__sort_relos(obj); 7497 } 7498 7499 /* place globals at the end of the arena (if supported) */ 7500 if (obj->arena_map_idx >= 0 && kernel_supports(obj, FEAT_LDIMM64_FULL_RANGE_OFF)) { 7501 struct bpf_map *arena_map = &obj->maps[obj->arena_map_idx]; 7502 7503 obj->arena_data_off = bpf_map_mmap_sz(arena_map) - 7504 roundup(obj->arena_data_sz, sysconf(_SC_PAGE_SIZE)); 7505 } 7506 7507 /* Before relocating calls pre-process relocations and mark 7508 * few ld_imm64 instructions that points to subprogs. 7509 * Otherwise bpf_object__reloc_code() later would have to consider 7510 * all ld_imm64 insns as relocation candidates. That would 7511 * reduce relocation speed, since amount of find_prog_insn_relo() 7512 * would increase and most of them will fail to find a relo. 7513 */ 7514 for (i = 0; i < obj->nr_programs; i++) { 7515 prog = &obj->programs[i]; 7516 for (j = 0; j < prog->nr_reloc; j++) { 7517 struct reloc_desc *relo = &prog->reloc_desc[j]; 7518 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 7519 7520 /* mark the insn, so it's recognized by insn_is_pseudo_func() */ 7521 if (relo->type == RELO_SUBPROG_ADDR) 7522 insn[0].src_reg = BPF_PSEUDO_FUNC; 7523 } 7524 } 7525 7526 /* relocate subprogram calls and append used subprograms to main 7527 * programs; each copy of subprogram code needs to be relocated 7528 * differently for each main program, because its code location might 7529 * have changed. 7530 * Append subprog relos to main programs to allow data relos to be 7531 * processed after text is completely relocated. 7532 */ 7533 for (i = 0; i < obj->nr_programs; i++) { 7534 prog = &obj->programs[i]; 7535 /* sub-program's sub-calls are relocated within the context of 7536 * its main program only 7537 */ 7538 if (prog_is_subprog(obj, prog)) 7539 continue; 7540 if (!prog->autoload) 7541 continue; 7542 7543 err = bpf_object__relocate_calls(obj, prog); 7544 if (err) { 7545 pr_warn("prog '%s': failed to relocate calls: %s\n", 7546 prog->name, errstr(err)); 7547 return err; 7548 } 7549 7550 err = bpf_prog_assign_exc_cb(obj, prog); 7551 if (err) 7552 return err; 7553 /* Now, also append exception callback if it has not been done already. */ 7554 if (prog->exception_cb_idx >= 0) { 7555 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx]; 7556 7557 /* Calling exception callback directly is disallowed, which the 7558 * verifier will reject later. In case it was processed already, 7559 * we can skip this step, otherwise for all other valid cases we 7560 * have to append exception callback now. 7561 */ 7562 if (subprog->sub_insn_off == 0) { 7563 err = bpf_object__append_subprog_code(obj, prog, subprog); 7564 if (err) 7565 return err; 7566 err = bpf_object__reloc_code(obj, prog, subprog); 7567 if (err) 7568 return err; 7569 } 7570 } 7571 } 7572 for (i = 0; i < obj->nr_programs; i++) { 7573 prog = &obj->programs[i]; 7574 if (prog_is_subprog(obj, prog)) 7575 continue; 7576 if (!prog->autoload) 7577 continue; 7578 7579 /* Process data relos for main programs */ 7580 err = bpf_object__relocate_data(obj, prog); 7581 if (err) { 7582 pr_warn("prog '%s': failed to relocate data references: %s\n", 7583 prog->name, errstr(err)); 7584 return err; 7585 } 7586 7587 /* Fix up .BTF.ext information, if necessary */ 7588 err = bpf_program_fixup_func_info(obj, prog); 7589 if (err) { 7590 pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %s\n", 7591 prog->name, errstr(err)); 7592 return err; 7593 } 7594 } 7595 7596 return 0; 7597 } 7598 7599 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 7600 Elf64_Shdr *shdr, Elf_Data *data); 7601 7602 static int bpf_object__collect_map_relos(struct bpf_object *obj, 7603 Elf64_Shdr *shdr, Elf_Data *data) 7604 { 7605 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *); 7606 int i, j, nrels, new_sz; 7607 const struct btf_var_secinfo *vi = NULL; 7608 const struct btf_type *sec, *var, *def; 7609 struct bpf_map *map = NULL, *targ_map = NULL; 7610 struct bpf_program *targ_prog = NULL; 7611 bool is_prog_array, is_map_in_map; 7612 const struct btf_member *member; 7613 const char *name, *mname, *type; 7614 unsigned int moff; 7615 Elf64_Sym *sym; 7616 Elf64_Rel *rel; 7617 void *tmp; 7618 7619 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) 7620 return -EINVAL; 7621 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); 7622 if (!sec) 7623 return -EINVAL; 7624 7625 nrels = shdr->sh_size / shdr->sh_entsize; 7626 for (i = 0; i < nrels; i++) { 7627 rel = elf_rel_by_idx(data, i); 7628 if (!rel) { 7629 pr_warn(".maps relo #%d: failed to get ELF relo\n", i); 7630 return -LIBBPF_ERRNO__FORMAT; 7631 } 7632 7633 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); 7634 if (!sym) { 7635 pr_warn(".maps relo #%d: symbol %zx not found\n", 7636 i, (size_t)ELF64_R_SYM(rel->r_info)); 7637 return -LIBBPF_ERRNO__FORMAT; 7638 } 7639 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; 7640 7641 pr_debug(".maps relo #%d: for %zd value %zu rel->r_offset %zu name %u ('%s')\n", 7642 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value, 7643 (size_t)rel->r_offset, sym->st_name, name); 7644 7645 for (j = 0; j < obj->nr_maps; j++) { 7646 map = &obj->maps[j]; 7647 if (map->sec_idx != obj->efile.btf_maps_shndx) 7648 continue; 7649 7650 vi = btf_var_secinfos(sec) + map->btf_var_idx; 7651 if (vi->offset <= rel->r_offset && 7652 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size) 7653 break; 7654 } 7655 if (j == obj->nr_maps) { 7656 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n", 7657 i, name, (size_t)rel->r_offset); 7658 return -EINVAL; 7659 } 7660 7661 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type); 7662 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY; 7663 type = is_map_in_map ? "map" : "prog"; 7664 if (is_map_in_map) { 7665 if (sym->st_shndx != obj->efile.btf_maps_shndx) { 7666 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", 7667 i, name); 7668 return -LIBBPF_ERRNO__RELOC; 7669 } 7670 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && 7671 map->def.key_size != sizeof(int)) { 7672 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", 7673 i, map->name, sizeof(int)); 7674 return -EINVAL; 7675 } 7676 targ_map = bpf_object__find_map_by_name(obj, name); 7677 if (!targ_map) { 7678 pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n", 7679 i, name); 7680 return -ESRCH; 7681 } 7682 } else if (is_prog_array) { 7683 targ_prog = bpf_object__find_program_by_name(obj, name); 7684 if (!targ_prog) { 7685 pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n", 7686 i, name); 7687 return -ESRCH; 7688 } 7689 if (targ_prog->sec_idx != sym->st_shndx || 7690 targ_prog->sec_insn_off * 8 != sym->st_value || 7691 prog_is_subprog(obj, targ_prog)) { 7692 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n", 7693 i, name); 7694 return -LIBBPF_ERRNO__RELOC; 7695 } 7696 } else { 7697 return -EINVAL; 7698 } 7699 7700 var = btf__type_by_id(obj->btf, vi->type); 7701 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 7702 if (btf_vlen(def) == 0) 7703 return -EINVAL; 7704 member = btf_members(def) + btf_vlen(def) - 1; 7705 mname = btf__name_by_offset(obj->btf, member->name_off); 7706 if (strcmp(mname, "values")) 7707 return -EINVAL; 7708 7709 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; 7710 if (rel->r_offset - vi->offset < moff) 7711 return -EINVAL; 7712 7713 moff = rel->r_offset - vi->offset - moff; 7714 /* here we use BPF pointer size, which is always 64 bit, as we 7715 * are parsing ELF that was built for BPF target 7716 */ 7717 if (moff % bpf_ptr_sz) 7718 return -EINVAL; 7719 moff /= bpf_ptr_sz; 7720 if (moff >= map->init_slots_sz) { 7721 new_sz = moff + 1; 7722 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz); 7723 if (!tmp) 7724 return -ENOMEM; 7725 map->init_slots = tmp; 7726 memset(map->init_slots + map->init_slots_sz, 0, 7727 (new_sz - map->init_slots_sz) * host_ptr_sz); 7728 map->init_slots_sz = new_sz; 7729 } 7730 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog; 7731 7732 pr_debug(".maps relo #%d: map '%s' slot [%u] points to %s '%s'\n", 7733 i, map->name, moff, type, name); 7734 } 7735 7736 return 0; 7737 } 7738 7739 static int bpf_object__collect_relos(struct bpf_object *obj) 7740 { 7741 int i, err; 7742 7743 for (i = 0; i < obj->efile.sec_cnt; i++) { 7744 struct elf_sec_desc *sec_desc = &obj->efile.secs[i]; 7745 Elf64_Shdr *shdr; 7746 Elf_Data *data; 7747 int idx; 7748 7749 if (sec_desc->sec_type != SEC_RELO) 7750 continue; 7751 7752 shdr = sec_desc->shdr; 7753 data = sec_desc->data; 7754 idx = shdr->sh_info; 7755 7756 if (shdr->sh_type != SHT_REL || idx < 0 || idx >= obj->efile.sec_cnt) { 7757 pr_warn("internal error at %d\n", __LINE__); 7758 return -LIBBPF_ERRNO__INTERNAL; 7759 } 7760 7761 if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) 7762 err = bpf_object__collect_st_ops_relos(obj, shdr, data); 7763 else if (idx == obj->efile.btf_maps_shndx) 7764 err = bpf_object__collect_map_relos(obj, shdr, data); 7765 else 7766 err = bpf_object__collect_prog_relos(obj, shdr, data); 7767 if (err) 7768 return err; 7769 } 7770 7771 bpf_object__sort_relos(obj); 7772 return 0; 7773 } 7774 7775 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id) 7776 { 7777 if (BPF_CLASS(insn->code) == BPF_JMP && 7778 BPF_OP(insn->code) == BPF_CALL && 7779 BPF_SRC(insn->code) == BPF_K && 7780 insn->src_reg == 0 && 7781 insn->dst_reg == 0) { 7782 *func_id = insn->imm; 7783 return true; 7784 } 7785 return false; 7786 } 7787 7788 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) 7789 { 7790 struct bpf_insn *insn = prog->insns; 7791 enum bpf_func_id func_id; 7792 int i; 7793 7794 if (obj->gen_loader) 7795 return 0; 7796 7797 for (i = 0; i < prog->insns_cnt; i++, insn++) { 7798 if (!insn_is_helper_call(insn, &func_id)) 7799 continue; 7800 7801 /* on kernels that don't yet support 7802 * bpf_probe_read_{kernel,user}[_str] helpers, fall back 7803 * to bpf_probe_read() which works well for old kernels 7804 */ 7805 switch (func_id) { 7806 case BPF_FUNC_probe_read_kernel: 7807 case BPF_FUNC_probe_read_user: 7808 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 7809 insn->imm = BPF_FUNC_probe_read; 7810 break; 7811 case BPF_FUNC_probe_read_kernel_str: 7812 case BPF_FUNC_probe_read_user_str: 7813 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 7814 insn->imm = BPF_FUNC_probe_read_str; 7815 break; 7816 default: 7817 break; 7818 } 7819 } 7820 return 0; 7821 } 7822 7823 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, 7824 int *btf_obj_fd, int *btf_type_id); 7825 7826 static inline bool is_tracing_multi(enum bpf_attach_type type) 7827 { 7828 return type == BPF_TRACE_FENTRY_MULTI || type == BPF_TRACE_FEXIT_MULTI || 7829 type == BPF_TRACE_FSESSION_MULTI; 7830 } 7831 7832 static const struct module_btf *find_attach_module(struct bpf_object *obj, const char *attach) 7833 { 7834 const char *sep, *mod_name = NULL; 7835 int i, mod_len, err; 7836 7837 /* 7838 * We expect attach string in the form of either 7839 * - function_pattern or 7840 * - <module>:function_pattern 7841 */ 7842 sep = strchr(attach, ':'); 7843 if (sep) { 7844 mod_name = attach; 7845 mod_len = sep - mod_name; 7846 } 7847 if (!mod_name) 7848 return NULL; 7849 7850 err = load_module_btfs(obj); 7851 if (err) 7852 return NULL; 7853 7854 for (i = 0; i < obj->btf_module_cnt; i++) { 7855 const struct module_btf *mod = &obj->btf_modules[i]; 7856 7857 if (strncmp(mod->name, mod_name, mod_len) == 0 && mod->name[mod_len] == '\0') 7858 return mod; 7859 } 7860 return NULL; 7861 } 7862 7863 static int tracing_multi_mod_fd(struct bpf_program *prog, int *btf_obj_fd) 7864 { 7865 const char *attach_name, *sep; 7866 const struct module_btf *mod; 7867 7868 *btf_obj_fd = 0; 7869 attach_name = strchr(prog->sec_name, '/'); 7870 7871 /* Program with no details in spec, using kernel btf. */ 7872 if (!attach_name) 7873 return 0; 7874 7875 /* Program with no module section, using kernel btf. */ 7876 sep = strchr(++attach_name, ':'); 7877 if (!sep) 7878 return 0; 7879 7880 /* Program with module specified, get its btf fd. */ 7881 mod = find_attach_module(prog->obj, attach_name); 7882 if (!mod) 7883 return -EINVAL; 7884 7885 *btf_obj_fd = mod->fd; 7886 return 0; 7887 } 7888 7889 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */ 7890 static int libbpf_prepare_prog_load(struct bpf_program *prog, 7891 struct bpf_prog_load_opts *opts, long cookie) 7892 { 7893 enum sec_def_flags def = cookie; 7894 7895 /* old kernels might not support specifying expected_attach_type */ 7896 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) 7897 opts->expected_attach_type = 0; 7898 7899 if (def & SEC_SLEEPABLE) 7900 opts->prog_flags |= BPF_F_SLEEPABLE; 7901 7902 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) 7903 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; 7904 7905 /* special check for usdt to use uprobe_multi link */ 7906 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) { 7907 /* for BPF_TRACE_UPROBE_MULTI, user might want to query expected_attach_type 7908 * in prog, and expected_attach_type we set in kernel is from opts, so we 7909 * update both. 7910 */ 7911 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; 7912 opts->expected_attach_type = BPF_TRACE_UPROBE_MULTI; 7913 } 7914 7915 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { 7916 int btf_obj_fd = 0, btf_type_id = 0, err; 7917 const char *attach_name; 7918 7919 attach_name = strchr(prog->sec_name, '/'); 7920 if (!attach_name) { 7921 /* if BPF program is annotated with just SEC("fentry") 7922 * (or similar) without declaratively specifying 7923 * target, then it is expected that target will be 7924 * specified with bpf_program__set_attach_target() at 7925 * runtime before BPF object load step. If not, then 7926 * there is nothing to load into the kernel as BPF 7927 * verifier won't be able to validate BPF program 7928 * correctness anyways. 7929 */ 7930 pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n", 7931 prog->name); 7932 return -EINVAL; 7933 } 7934 attach_name++; /* skip over / */ 7935 7936 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id); 7937 if (err) 7938 return err; 7939 7940 /* cache resolved BTF FD and BTF type ID in the prog */ 7941 prog->attach_btf_obj_fd = btf_obj_fd; 7942 prog->attach_btf_id = btf_type_id; 7943 7944 /* but by now libbpf common logic is not utilizing 7945 * prog->attach_btf_obj_fd/prog->attach_btf_id anymore because 7946 * this callback is called after opts were populated by 7947 * libbpf, so this callback has to update opts explicitly here 7948 */ 7949 opts->attach_btf_obj_fd = btf_obj_fd; 7950 opts->attach_btf_id = btf_type_id; 7951 } 7952 7953 if (is_tracing_multi(prog->expected_attach_type)) { 7954 int err, btf_obj_fd = 0; 7955 7956 err = tracing_multi_mod_fd(prog, &btf_obj_fd); 7957 if (err < 0) 7958 return err; 7959 7960 prog->attach_btf_obj_fd = btf_obj_fd; 7961 opts->attach_btf_obj_fd = btf_obj_fd; 7962 } 7963 7964 return 0; 7965 } 7966 7967 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz); 7968 7969 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, 7970 struct bpf_insn *insns, int insns_cnt, 7971 const char *license, __u32 kern_version, int *prog_fd) 7972 { 7973 LIBBPF_OPTS(bpf_prog_load_opts, load_attr); 7974 const char *prog_name = NULL; 7975 size_t log_buf_size = 0; 7976 char *log_buf = NULL, *tmp; 7977 bool own_log_buf = true; 7978 __u32 log_level = prog->log_level; 7979 int ret, err; 7980 7981 /* Be more helpful by rejecting programs that can't be validated early 7982 * with more meaningful and actionable error message. 7983 */ 7984 switch (prog->type) { 7985 case BPF_PROG_TYPE_UNSPEC: 7986 /* 7987 * The program type must be set. Most likely we couldn't find a proper 7988 * section definition at load time, and thus we didn't infer the type. 7989 */ 7990 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n", 7991 prog->name, prog->sec_name); 7992 return -EINVAL; 7993 case BPF_PROG_TYPE_STRUCT_OPS: 7994 if (prog->attach_btf_id == 0) { 7995 pr_warn("prog '%s': SEC(\"struct_ops\") program isn't referenced anywhere, did you forget to use it?\n", 7996 prog->name); 7997 return -EINVAL; 7998 } 7999 break; 8000 default: 8001 break; 8002 } 8003 8004 if (!insns || !insns_cnt) 8005 return -EINVAL; 8006 8007 if (kernel_supports(obj, FEAT_PROG_NAME)) 8008 prog_name = prog->name; 8009 load_attr.attach_prog_fd = prog->attach_prog_fd; 8010 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; 8011 load_attr.attach_btf_id = prog->attach_btf_id; 8012 load_attr.kern_version = kern_version; 8013 load_attr.prog_ifindex = prog->prog_ifindex; 8014 load_attr.expected_attach_type = prog->expected_attach_type; 8015 8016 /* specify func_info/line_info only if kernel supports them */ 8017 if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) { 8018 load_attr.prog_btf_fd = btf__fd(obj->btf); 8019 load_attr.func_info = prog->func_info; 8020 load_attr.func_info_rec_size = prog->func_info_rec_size; 8021 load_attr.func_info_cnt = prog->func_info_cnt; 8022 load_attr.line_info = prog->line_info; 8023 load_attr.line_info_rec_size = prog->line_info_rec_size; 8024 load_attr.line_info_cnt = prog->line_info_cnt; 8025 } 8026 load_attr.log_level = log_level; 8027 load_attr.prog_flags = prog->prog_flags; 8028 load_attr.fd_array = obj->fd_array; 8029 8030 load_attr.token_fd = obj->token_fd; 8031 if (obj->token_fd) 8032 load_attr.prog_flags |= BPF_F_TOKEN_FD; 8033 8034 /* adjust load_attr if sec_def provides custom preload callback */ 8035 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) { 8036 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie); 8037 if (err < 0) { 8038 pr_warn("prog '%s': failed to prepare load attributes: %s\n", 8039 prog->name, errstr(err)); 8040 return err; 8041 } 8042 insns = prog->insns; 8043 insns_cnt = prog->insns_cnt; 8044 } 8045 8046 if (obj->gen_loader) { 8047 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, 8048 license, insns, insns_cnt, &load_attr, 8049 prog - obj->programs); 8050 *prog_fd = -1; 8051 return 0; 8052 } 8053 8054 retry_load: 8055 /* if log_level is zero, we don't request logs initially even if 8056 * custom log_buf is specified; if the program load fails, then we'll 8057 * bump log_level to 1 and use either custom log_buf or we'll allocate 8058 * our own and retry the load to get details on what failed 8059 */ 8060 if (log_level) { 8061 if (prog->log_buf) { 8062 log_buf = prog->log_buf; 8063 log_buf_size = prog->log_size; 8064 own_log_buf = false; 8065 } else if (obj->log_buf) { 8066 log_buf = obj->log_buf; 8067 log_buf_size = obj->log_size; 8068 own_log_buf = false; 8069 } else { 8070 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2); 8071 tmp = realloc(log_buf, log_buf_size); 8072 if (!tmp) { 8073 ret = -ENOMEM; 8074 goto out; 8075 } 8076 log_buf = tmp; 8077 log_buf[0] = '\0'; 8078 own_log_buf = true; 8079 } 8080 } 8081 8082 load_attr.log_buf = log_buf; 8083 load_attr.log_size = log_buf_size; 8084 load_attr.log_level = log_level; 8085 8086 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); 8087 if (ret >= 0) { 8088 if (log_level && own_log_buf) { 8089 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", 8090 prog->name, log_buf); 8091 } 8092 8093 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) { 8094 struct bpf_map *map; 8095 int i; 8096 8097 for (i = 0; i < obj->nr_maps; i++) { 8098 map = &prog->obj->maps[i]; 8099 if (map->libbpf_type != LIBBPF_MAP_RODATA) 8100 continue; 8101 8102 if (bpf_prog_bind_map(ret, map->fd, NULL)) { 8103 pr_warn("prog '%s': failed to bind map '%s': %s\n", 8104 prog->name, map->real_name, errstr(errno)); 8105 /* Don't fail hard if can't bind rodata. */ 8106 } 8107 } 8108 } 8109 8110 *prog_fd = ret; 8111 ret = 0; 8112 goto out; 8113 } 8114 8115 if (log_level == 0) { 8116 log_level = 1; 8117 goto retry_load; 8118 } 8119 /* On ENOSPC, increase log buffer size and retry, unless custom 8120 * log_buf is specified. 8121 * Be careful to not overflow u32, though. Kernel's log buf size limit 8122 * isn't part of UAPI so it can always be bumped to full 4GB. So don't 8123 * multiply by 2 unless we are sure we'll fit within 32 bits. 8124 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2). 8125 */ 8126 if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2) 8127 goto retry_load; 8128 8129 ret = -errno; 8130 8131 /* post-process verifier log to improve error descriptions */ 8132 fixup_verifier_log(prog, log_buf, log_buf_size); 8133 8134 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, errstr(errno)); 8135 pr_perm_msg(ret); 8136 8137 if (own_log_buf && log_buf && log_buf[0] != '\0') { 8138 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", 8139 prog->name, log_buf); 8140 } 8141 8142 out: 8143 if (own_log_buf) 8144 free(log_buf); 8145 return ret; 8146 } 8147 8148 static char *find_prev_line(char *buf, char *cur) 8149 { 8150 char *p; 8151 8152 if (cur == buf) /* end of a log buf */ 8153 return NULL; 8154 8155 p = cur - 1; 8156 while (p - 1 >= buf && *(p - 1) != '\n') 8157 p--; 8158 8159 return p; 8160 } 8161 8162 static void patch_log(char *buf, size_t buf_sz, size_t log_sz, 8163 char *orig, size_t orig_sz, const char *patch) 8164 { 8165 /* size of the remaining log content to the right from the to-be-replaced part */ 8166 size_t rem_sz = (buf + log_sz) - (orig + orig_sz); 8167 size_t patch_sz = strlen(patch); 8168 8169 if (patch_sz != orig_sz) { 8170 /* If patch line(s) are longer than original piece of verifier log, 8171 * shift log contents by (patch_sz - orig_sz) bytes to the right 8172 * starting from after to-be-replaced part of the log. 8173 * 8174 * If patch line(s) are shorter than original piece of verifier log, 8175 * shift log contents by (orig_sz - patch_sz) bytes to the left 8176 * starting from after to-be-replaced part of the log 8177 * 8178 * We need to be careful about not overflowing available 8179 * buf_sz capacity. If that's the case, we'll truncate the end 8180 * of the original log, as necessary. 8181 */ 8182 if (patch_sz > orig_sz) { 8183 if (orig + patch_sz >= buf + buf_sz) { 8184 /* patch is big enough to cover remaining space completely */ 8185 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1; 8186 rem_sz = 0; 8187 } else if (patch_sz - orig_sz > buf_sz - log_sz) { 8188 /* patch causes part of remaining log to be truncated */ 8189 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz); 8190 } 8191 } 8192 /* shift remaining log to the right by calculated amount */ 8193 memmove(orig + patch_sz, orig + orig_sz, rem_sz); 8194 } 8195 8196 memcpy(orig, patch, patch_sz); 8197 } 8198 8199 static void fixup_log_failed_core_relo(struct bpf_program *prog, 8200 char *buf, size_t buf_sz, size_t log_sz, 8201 char *line1, char *line2, char *line3) 8202 { 8203 /* Expected log for failed and not properly guarded CO-RE relocation: 8204 * line1 -> 123: (85) call unknown#195896080 8205 * line2 -> invalid func unknown#195896080 8206 * line3 -> <anything else or end of buffer> 8207 * 8208 * "123" is the index of the instruction that was poisoned. We extract 8209 * instruction index to find corresponding CO-RE relocation and 8210 * replace this part of the log with more relevant information about 8211 * failed CO-RE relocation. 8212 */ 8213 const struct bpf_core_relo *relo; 8214 struct bpf_core_spec spec; 8215 char patch[512], spec_buf[256]; 8216 int insn_idx, err, spec_len; 8217 8218 if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1) 8219 return; 8220 8221 relo = find_relo_core(prog, insn_idx); 8222 if (!relo) 8223 return; 8224 8225 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec); 8226 if (err) 8227 return; 8228 8229 spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec); 8230 snprintf(patch, sizeof(patch), 8231 "%d: <invalid CO-RE relocation>\n" 8232 "failed to resolve CO-RE relocation %s%s\n", 8233 insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : ""); 8234 8235 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 8236 } 8237 8238 static void fixup_log_missing_map_load(struct bpf_program *prog, 8239 char *buf, size_t buf_sz, size_t log_sz, 8240 char *line1, char *line2, char *line3) 8241 { 8242 /* Expected log for failed and not properly guarded map reference: 8243 * line1 -> 123: (85) call unknown#2001000345 8244 * line2 -> invalid func unknown#2001000345 8245 * line3 -> <anything else or end of buffer> 8246 * 8247 * "123" is the index of the instruction that was poisoned. 8248 * "345" in "2001000345" is a map index in obj->maps to fetch map name. 8249 */ 8250 struct bpf_object *obj = prog->obj; 8251 const struct bpf_map *map; 8252 int insn_idx, map_idx; 8253 char patch[128]; 8254 8255 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2) 8256 return; 8257 8258 map_idx -= POISON_LDIMM64_MAP_BASE; 8259 if (map_idx < 0 || map_idx >= obj->nr_maps) 8260 return; 8261 map = &obj->maps[map_idx]; 8262 8263 snprintf(patch, sizeof(patch), 8264 "%d: <invalid BPF map reference>\n" 8265 "BPF map '%s' is referenced but wasn't created\n", 8266 insn_idx, map->name); 8267 8268 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 8269 } 8270 8271 static void fixup_log_missing_kfunc_call(struct bpf_program *prog, 8272 char *buf, size_t buf_sz, size_t log_sz, 8273 char *line1, char *line2, char *line3) 8274 { 8275 /* Expected log for failed and not properly guarded kfunc call: 8276 * line1 -> 123: (85) call unknown#2002000345 8277 * line2 -> invalid func unknown#2002000345 8278 * line3 -> <anything else or end of buffer> 8279 * 8280 * "123" is the index of the instruction that was poisoned. 8281 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name. 8282 */ 8283 struct bpf_object *obj = prog->obj; 8284 const struct extern_desc *ext; 8285 int insn_idx, ext_idx; 8286 char patch[128]; 8287 8288 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &ext_idx) != 2) 8289 return; 8290 8291 ext_idx -= POISON_CALL_KFUNC_BASE; 8292 if (ext_idx < 0 || ext_idx >= obj->nr_extern) 8293 return; 8294 ext = &obj->externs[ext_idx]; 8295 8296 snprintf(patch, sizeof(patch), 8297 "%d: <invalid kfunc call>\n" 8298 "kfunc '%s' is referenced but wasn't resolved\n", 8299 insn_idx, ext->name); 8300 8301 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 8302 } 8303 8304 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz) 8305 { 8306 /* look for familiar error patterns in last N lines of the log */ 8307 const size_t max_last_line_cnt = 10; 8308 char *prev_line, *cur_line, *next_line; 8309 size_t log_sz; 8310 int i; 8311 8312 if (!buf) 8313 return; 8314 8315 log_sz = strlen(buf) + 1; 8316 next_line = buf + log_sz - 1; 8317 8318 for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) { 8319 cur_line = find_prev_line(buf, next_line); 8320 if (!cur_line) 8321 return; 8322 8323 if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) { 8324 prev_line = find_prev_line(buf, cur_line); 8325 if (!prev_line) 8326 continue; 8327 8328 /* failed CO-RE relocation case */ 8329 fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz, 8330 prev_line, cur_line, next_line); 8331 return; 8332 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_LDIMM64_MAP_PFX)) { 8333 prev_line = find_prev_line(buf, cur_line); 8334 if (!prev_line) 8335 continue; 8336 8337 /* reference to uncreated BPF map */ 8338 fixup_log_missing_map_load(prog, buf, buf_sz, log_sz, 8339 prev_line, cur_line, next_line); 8340 return; 8341 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_CALL_KFUNC_PFX)) { 8342 prev_line = find_prev_line(buf, cur_line); 8343 if (!prev_line) 8344 continue; 8345 8346 /* reference to unresolved kfunc */ 8347 fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz, 8348 prev_line, cur_line, next_line); 8349 return; 8350 } 8351 } 8352 } 8353 8354 static int bpf_program_record_relos(struct bpf_program *prog) 8355 { 8356 struct bpf_object *obj = prog->obj; 8357 int i; 8358 8359 for (i = 0; i < prog->nr_reloc; i++) { 8360 struct reloc_desc *relo = &prog->reloc_desc[i]; 8361 struct extern_desc *ext = &obj->externs[relo->ext_idx]; 8362 int kind; 8363 8364 switch (relo->type) { 8365 case RELO_EXTERN_LD64: 8366 if (ext->type != EXT_KSYM) 8367 continue; 8368 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ? 8369 BTF_KIND_VAR : BTF_KIND_FUNC; 8370 bpf_gen__record_extern(obj->gen_loader, ext->name, 8371 ext->is_weak, !ext->ksym.type_id, 8372 true, kind, relo->insn_idx); 8373 break; 8374 case RELO_EXTERN_CALL: 8375 bpf_gen__record_extern(obj->gen_loader, ext->name, 8376 ext->is_weak, false, false, BTF_KIND_FUNC, 8377 relo->insn_idx); 8378 break; 8379 case RELO_CORE: { 8380 struct bpf_core_relo cr = { 8381 .insn_off = relo->insn_idx * 8, 8382 .type_id = relo->core_relo->type_id, 8383 .access_str_off = relo->core_relo->access_str_off, 8384 .kind = relo->core_relo->kind, 8385 }; 8386 8387 bpf_gen__record_relo_core(obj->gen_loader, &cr); 8388 break; 8389 } 8390 default: 8391 continue; 8392 } 8393 } 8394 return 0; 8395 } 8396 8397 static int 8398 bpf_object__load_progs(struct bpf_object *obj, int log_level) 8399 { 8400 struct bpf_program *prog; 8401 size_t i; 8402 int err; 8403 8404 for (i = 0; i < obj->nr_programs; i++) { 8405 prog = &obj->programs[i]; 8406 if (prog_is_subprog(obj, prog)) 8407 continue; 8408 if (!prog->autoload) { 8409 pr_debug("prog '%s': skipped loading\n", prog->name); 8410 continue; 8411 } 8412 prog->log_level |= log_level; 8413 8414 if (obj->gen_loader) 8415 bpf_program_record_relos(prog); 8416 8417 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, 8418 obj->license, obj->kern_version, &prog->fd); 8419 if (err) { 8420 pr_warn("prog '%s': failed to load: %s\n", prog->name, errstr(err)); 8421 return err; 8422 } 8423 } 8424 8425 bpf_object__free_relocs(obj); 8426 return 0; 8427 } 8428 8429 static int bpf_object_prepare_progs(struct bpf_object *obj) 8430 { 8431 struct bpf_program *prog; 8432 size_t i; 8433 int err; 8434 8435 for (i = 0; i < obj->nr_programs; i++) { 8436 prog = &obj->programs[i]; 8437 err = bpf_object__sanitize_prog(obj, prog); 8438 if (err) 8439 return err; 8440 } 8441 return 0; 8442 } 8443 8444 static const struct bpf_sec_def *find_sec_def(const char *sec_name); 8445 8446 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts) 8447 { 8448 struct bpf_program *prog; 8449 int err; 8450 8451 bpf_object__for_each_program(prog, obj) { 8452 prog->sec_def = find_sec_def(prog->sec_name); 8453 if (!prog->sec_def) { 8454 /* couldn't guess, but user might manually specify */ 8455 pr_debug("prog '%s': unrecognized ELF section name '%s'\n", 8456 prog->name, prog->sec_name); 8457 continue; 8458 } 8459 8460 prog->type = prog->sec_def->prog_type; 8461 prog->expected_attach_type = prog->sec_def->expected_attach_type; 8462 8463 /* sec_def can have custom callback which should be called 8464 * after bpf_program is initialized to adjust its properties 8465 */ 8466 if (prog->sec_def->prog_setup_fn) { 8467 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie); 8468 if (err < 0) { 8469 pr_warn("prog '%s': failed to initialize: %s\n", 8470 prog->name, errstr(err)); 8471 return err; 8472 } 8473 } 8474 } 8475 8476 return 0; 8477 } 8478 8479 static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz, 8480 const char *obj_name, 8481 const struct bpf_object_open_opts *opts) 8482 { 8483 const char *kconfig, *btf_tmp_path, *token_path; 8484 struct bpf_object *obj; 8485 int err; 8486 char *log_buf; 8487 size_t log_size; 8488 __u32 log_level; 8489 8490 if (obj_buf && !obj_name) 8491 return ERR_PTR(-EINVAL); 8492 8493 if (elf_version(EV_CURRENT) == EV_NONE) { 8494 pr_warn("failed to init libelf for %s\n", 8495 path ? : "(mem buf)"); 8496 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 8497 } 8498 8499 if (!OPTS_VALID(opts, bpf_object_open_opts)) 8500 return ERR_PTR(-EINVAL); 8501 8502 obj_name = OPTS_GET(opts, object_name, NULL) ?: obj_name; 8503 if (obj_buf) { 8504 path = obj_name; 8505 pr_debug("loading object '%s' from buffer\n", obj_name); 8506 } else { 8507 pr_debug("loading object from %s\n", path); 8508 } 8509 8510 log_buf = OPTS_GET(opts, kernel_log_buf, NULL); 8511 log_size = OPTS_GET(opts, kernel_log_size, 0); 8512 log_level = OPTS_GET(opts, kernel_log_level, 0); 8513 if (log_size > UINT_MAX) 8514 return ERR_PTR(-EINVAL); 8515 if (log_size && !log_buf) 8516 return ERR_PTR(-EINVAL); 8517 8518 token_path = OPTS_GET(opts, bpf_token_path, NULL); 8519 /* if user didn't specify bpf_token_path explicitly, check if 8520 * LIBBPF_BPF_TOKEN_PATH envvar was set and treat it as bpf_token_path 8521 * option 8522 */ 8523 if (!token_path) 8524 token_path = getenv("LIBBPF_BPF_TOKEN_PATH"); 8525 if (token_path && strlen(token_path) >= PATH_MAX) 8526 return ERR_PTR(-ENAMETOOLONG); 8527 8528 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); 8529 if (IS_ERR(obj)) 8530 return obj; 8531 8532 obj->log_buf = log_buf; 8533 obj->log_size = log_size; 8534 obj->log_level = log_level; 8535 8536 if (token_path) { 8537 obj->token_path = strdup(token_path); 8538 if (!obj->token_path) { 8539 err = -ENOMEM; 8540 goto out; 8541 } 8542 } 8543 8544 btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL); 8545 if (btf_tmp_path) { 8546 if (strlen(btf_tmp_path) >= PATH_MAX) { 8547 err = -ENAMETOOLONG; 8548 goto out; 8549 } 8550 obj->btf_custom_path = strdup(btf_tmp_path); 8551 if (!obj->btf_custom_path) { 8552 err = -ENOMEM; 8553 goto out; 8554 } 8555 } 8556 8557 kconfig = OPTS_GET(opts, kconfig, NULL); 8558 if (kconfig) { 8559 obj->kconfig = strdup(kconfig); 8560 if (!obj->kconfig) { 8561 err = -ENOMEM; 8562 goto out; 8563 } 8564 } 8565 8566 err = bpf_object__elf_init(obj); 8567 err = err ? : bpf_object__elf_collect(obj); 8568 err = err ? : bpf_object__collect_externs(obj); 8569 err = err ? : bpf_object_fixup_btf(obj); 8570 err = err ? : bpf_object__init_maps(obj, opts); 8571 err = err ? : bpf_object_init_progs(obj, opts); 8572 err = err ? : bpf_object__collect_relos(obj); 8573 if (err) 8574 goto out; 8575 8576 bpf_object__elf_finish(obj); 8577 8578 return obj; 8579 out: 8580 bpf_object__close(obj); 8581 return ERR_PTR(err); 8582 } 8583 8584 struct bpf_object * 8585 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) 8586 { 8587 if (!path) 8588 return libbpf_err_ptr(-EINVAL); 8589 8590 return libbpf_ptr(bpf_object_open(path, NULL, 0, NULL, opts)); 8591 } 8592 8593 struct bpf_object *bpf_object__open(const char *path) 8594 { 8595 return bpf_object__open_file(path, NULL); 8596 } 8597 8598 struct bpf_object * 8599 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, 8600 const struct bpf_object_open_opts *opts) 8601 { 8602 char tmp_name[64]; 8603 8604 if (!obj_buf || obj_buf_sz == 0) 8605 return libbpf_err_ptr(-EINVAL); 8606 8607 /* create a (quite useless) default "name" for this memory buffer object */ 8608 snprintf(tmp_name, sizeof(tmp_name), "%lx-%zx", (unsigned long)obj_buf, obj_buf_sz); 8609 8610 return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, tmp_name, opts)); 8611 } 8612 8613 static int bpf_object_unload(struct bpf_object *obj) 8614 { 8615 size_t i; 8616 8617 if (!obj) 8618 return libbpf_err(-EINVAL); 8619 8620 for (i = 0; i < obj->nr_maps; i++) { 8621 zclose(obj->maps[i].fd); 8622 if (obj->maps[i].st_ops) 8623 zfree(&obj->maps[i].st_ops->kern_vdata); 8624 } 8625 8626 for (i = 0; i < obj->nr_programs; i++) 8627 bpf_program__unload(&obj->programs[i]); 8628 8629 return 0; 8630 } 8631 8632 static int bpf_object__sanitize_maps(struct bpf_object *obj) 8633 { 8634 struct bpf_map *m; 8635 8636 bpf_object__for_each_map(m, obj) { 8637 if (!bpf_map__is_internal(m)) 8638 continue; 8639 if (!kernel_supports(obj, FEAT_ARRAY_MMAP)) 8640 m->def.map_flags &= ~BPF_F_MMAPABLE; 8641 } 8642 8643 return 0; 8644 } 8645 8646 typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type, 8647 const char *sym_name, void *ctx); 8648 8649 static int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx) 8650 { 8651 char sym_type, sym_name[500]; 8652 unsigned long long sym_addr; 8653 int ret, err = 0; 8654 FILE *f; 8655 8656 f = fopen("/proc/kallsyms", "re"); 8657 if (!f) { 8658 err = -errno; 8659 pr_warn("failed to open /proc/kallsyms: %s\n", errstr(err)); 8660 return err; 8661 } 8662 8663 while (true) { 8664 ret = fscanf(f, "%llx %c %499s%*[^\n]\n", 8665 &sym_addr, &sym_type, sym_name); 8666 if (ret == EOF && feof(f)) 8667 break; 8668 if (ret != 3) { 8669 pr_warn("failed to read kallsyms entry: %d\n", ret); 8670 err = -EINVAL; 8671 break; 8672 } 8673 8674 err = cb(sym_addr, sym_type, sym_name, ctx); 8675 if (err) 8676 break; 8677 } 8678 8679 fclose(f); 8680 return err; 8681 } 8682 8683 static int kallsyms_cb(unsigned long long sym_addr, char sym_type, 8684 const char *sym_name, void *ctx) 8685 { 8686 struct bpf_object *obj = ctx; 8687 const struct btf_type *t; 8688 struct extern_desc *ext; 8689 const char *res; 8690 8691 res = strstr(sym_name, ".llvm."); 8692 if (sym_type == 'd' && res) 8693 ext = find_extern_by_name_with_len(obj, sym_name, res - sym_name); 8694 else 8695 ext = find_extern_by_name(obj, sym_name); 8696 if (!ext || ext->type != EXT_KSYM) 8697 return 0; 8698 8699 t = btf__type_by_id(obj->btf, ext->btf_id); 8700 if (!btf_is_var(t)) 8701 return 0; 8702 8703 if (ext->is_set && ext->ksym.addr != sym_addr) { 8704 pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n", 8705 sym_name, ext->ksym.addr, sym_addr); 8706 return -EINVAL; 8707 } 8708 if (!ext->is_set) { 8709 ext->is_set = true; 8710 ext->ksym.addr = sym_addr; 8711 pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr); 8712 } 8713 return 0; 8714 } 8715 8716 static int bpf_object__read_kallsyms_file(struct bpf_object *obj) 8717 { 8718 return libbpf_kallsyms_parse(kallsyms_cb, obj); 8719 } 8720 8721 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, 8722 __u16 kind, struct btf **res_btf, 8723 struct module_btf **res_mod_btf) 8724 { 8725 struct module_btf *mod_btf; 8726 struct btf *btf; 8727 int i, id, err; 8728 8729 btf = obj->btf_vmlinux; 8730 mod_btf = NULL; 8731 id = btf__find_by_name_kind(btf, ksym_name, kind); 8732 8733 if (id == -ENOENT) { 8734 err = load_module_btfs(obj); 8735 if (err) 8736 return err; 8737 8738 for (i = 0; i < obj->btf_module_cnt; i++) { 8739 /* we assume module_btf's BTF FD is always >0 */ 8740 mod_btf = &obj->btf_modules[i]; 8741 btf = mod_btf->btf; 8742 id = btf__find_by_name_kind_own(btf, ksym_name, kind); 8743 if (id != -ENOENT) 8744 break; 8745 } 8746 } 8747 if (id <= 0) 8748 return -ESRCH; 8749 8750 *res_btf = btf; 8751 *res_mod_btf = mod_btf; 8752 return id; 8753 } 8754 8755 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, 8756 struct extern_desc *ext) 8757 { 8758 const struct btf_type *targ_var, *targ_type; 8759 __u32 targ_type_id, local_type_id; 8760 struct module_btf *mod_btf = NULL; 8761 const char *targ_var_name; 8762 struct btf *btf = NULL; 8763 int id, err; 8764 8765 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf); 8766 if (id < 0) { 8767 if (id == -ESRCH && ext->is_weak) 8768 return 0; 8769 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n", 8770 ext->name); 8771 return id; 8772 } 8773 8774 /* find local type_id */ 8775 local_type_id = ext->ksym.type_id; 8776 8777 /* find target type_id */ 8778 targ_var = btf__type_by_id(btf, id); 8779 targ_var_name = btf__name_by_offset(btf, targ_var->name_off); 8780 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id); 8781 8782 err = bpf_core_types_are_compat(obj->btf, local_type_id, 8783 btf, targ_type_id); 8784 if (err <= 0) { 8785 const struct btf_type *local_type; 8786 const char *targ_name, *local_name; 8787 8788 local_type = btf__type_by_id(obj->btf, local_type_id); 8789 local_name = btf__name_by_offset(obj->btf, local_type->name_off); 8790 targ_name = btf__name_by_offset(btf, targ_type->name_off); 8791 8792 pr_warn("extern (var ksym) '%s': incompatible types, expected [%u] %s %s, but kernel has [%u] %s %s\n", 8793 ext->name, local_type_id, 8794 btf_kind_str(local_type), local_name, targ_type_id, 8795 btf_kind_str(targ_type), targ_name); 8796 return -EINVAL; 8797 } 8798 8799 ext->is_set = true; 8800 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; 8801 ext->ksym.kernel_btf_id = id; 8802 pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n", 8803 ext->name, id, btf_kind_str(targ_var), targ_var_name); 8804 8805 return 0; 8806 } 8807 8808 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, 8809 struct extern_desc *ext) 8810 { 8811 int local_func_proto_id, kfunc_proto_id, kfunc_id; 8812 struct module_btf *mod_btf = NULL; 8813 const struct btf_type *kern_func; 8814 struct btf *kern_btf = NULL; 8815 int ret; 8816 8817 local_func_proto_id = ext->ksym.type_id; 8818 8819 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf, 8820 &mod_btf); 8821 if (kfunc_id < 0) { 8822 if (kfunc_id == -ESRCH && ext->is_weak) 8823 return 0; 8824 pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n", 8825 ext->name); 8826 return kfunc_id; 8827 } 8828 8829 kern_func = btf__type_by_id(kern_btf, kfunc_id); 8830 kfunc_proto_id = kern_func->type; 8831 8832 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, 8833 kern_btf, kfunc_proto_id); 8834 if (ret <= 0) { 8835 if (ext->is_weak) 8836 return 0; 8837 8838 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with %s [%d]\n", 8839 ext->name, local_func_proto_id, 8840 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id); 8841 return -EINVAL; 8842 } 8843 8844 /* set index for module BTF fd in fd_array, if unset */ 8845 if (mod_btf && !mod_btf->fd_array_idx) { 8846 /* insn->off is s16 */ 8847 if (obj->fd_array_cnt == INT16_MAX) { 8848 pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n", 8849 ext->name, mod_btf->fd_array_idx); 8850 return -E2BIG; 8851 } 8852 /* Cannot use index 0 for module BTF fd */ 8853 if (!obj->fd_array_cnt) 8854 obj->fd_array_cnt = 1; 8855 8856 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), 8857 obj->fd_array_cnt + 1); 8858 if (ret) 8859 return ret; 8860 mod_btf->fd_array_idx = obj->fd_array_cnt; 8861 /* we assume module BTF FD is always >0 */ 8862 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd; 8863 } 8864 8865 ext->is_set = true; 8866 ext->ksym.kernel_btf_id = kfunc_id; 8867 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0; 8868 /* Also set kernel_btf_obj_fd to make sure that bpf_object__relocate_data() 8869 * populates FD into ld_imm64 insn when it's used to point to kfunc. 8870 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call. 8871 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64. 8872 */ 8873 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; 8874 pr_debug("extern (func ksym) '%s': resolved to %s [%d]\n", 8875 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id); 8876 8877 return 0; 8878 } 8879 8880 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj) 8881 { 8882 const struct btf_type *t; 8883 struct extern_desc *ext; 8884 int i, err; 8885 8886 for (i = 0; i < obj->nr_extern; i++) { 8887 ext = &obj->externs[i]; 8888 if (ext->type != EXT_KSYM || !ext->ksym.type_id) 8889 continue; 8890 8891 if (obj->gen_loader) { 8892 ext->is_set = true; 8893 ext->ksym.kernel_btf_obj_fd = 0; 8894 ext->ksym.kernel_btf_id = 0; 8895 continue; 8896 } 8897 t = btf__type_by_id(obj->btf, ext->btf_id); 8898 if (btf_is_var(t)) 8899 err = bpf_object__resolve_ksym_var_btf_id(obj, ext); 8900 else 8901 err = bpf_object__resolve_ksym_func_btf_id(obj, ext); 8902 if (err) 8903 return err; 8904 } 8905 return 0; 8906 } 8907 8908 static int bpf_object__resolve_externs(struct bpf_object *obj, 8909 const char *extra_kconfig) 8910 { 8911 bool need_config = false, need_kallsyms = false; 8912 bool need_vmlinux_btf = false; 8913 struct extern_desc *ext; 8914 void *kcfg_data = NULL; 8915 int err, i; 8916 8917 if (obj->nr_extern == 0) 8918 return 0; 8919 8920 if (obj->kconfig_map_idx >= 0) 8921 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; 8922 8923 for (i = 0; i < obj->nr_extern; i++) { 8924 ext = &obj->externs[i]; 8925 8926 if (ext->type == EXT_KSYM) { 8927 if (ext->ksym.type_id) 8928 need_vmlinux_btf = true; 8929 else 8930 need_kallsyms = true; 8931 continue; 8932 } else if (ext->type == EXT_KCFG) { 8933 void *ext_ptr = kcfg_data + ext->kcfg.data_off; 8934 __u64 value = 0; 8935 8936 /* Kconfig externs need actual /proc/config.gz */ 8937 if (str_has_pfx(ext->name, "CONFIG_")) { 8938 need_config = true; 8939 continue; 8940 } 8941 8942 /* Virtual kcfg externs are customly handled by libbpf */ 8943 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { 8944 value = get_kernel_version(); 8945 if (!value) { 8946 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name); 8947 return -EINVAL; 8948 } 8949 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) { 8950 value = kernel_supports(obj, FEAT_BPF_COOKIE); 8951 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) { 8952 value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER); 8953 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) { 8954 /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed 8955 * __kconfig externs, where LINUX_ ones are virtual and filled out 8956 * customly by libbpf (their values don't come from Kconfig). 8957 * If LINUX_xxx variable is not recognized by libbpf, but is marked 8958 * __weak, it defaults to zero value, just like for CONFIG_xxx 8959 * externs. 8960 */ 8961 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name); 8962 return -EINVAL; 8963 } 8964 8965 err = set_kcfg_value_num(ext, ext_ptr, value); 8966 if (err) 8967 return err; 8968 pr_debug("extern (kcfg) '%s': set to 0x%llx\n", 8969 ext->name, (unsigned long long)value); 8970 } else { 8971 pr_warn("extern '%s': unrecognized extern kind\n", ext->name); 8972 return -EINVAL; 8973 } 8974 } 8975 if (need_config && extra_kconfig) { 8976 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data); 8977 if (err) 8978 return -EINVAL; 8979 need_config = false; 8980 for (i = 0; i < obj->nr_extern; i++) { 8981 ext = &obj->externs[i]; 8982 if (ext->type == EXT_KCFG && !ext->is_set) { 8983 need_config = true; 8984 break; 8985 } 8986 } 8987 } 8988 if (need_config) { 8989 err = bpf_object__read_kconfig_file(obj, kcfg_data); 8990 if (err) 8991 return -EINVAL; 8992 } 8993 if (need_kallsyms) { 8994 err = bpf_object__read_kallsyms_file(obj); 8995 if (err) 8996 return -EINVAL; 8997 } 8998 if (need_vmlinux_btf) { 8999 err = bpf_object__resolve_ksyms_btf_id(obj); 9000 if (err) 9001 return -EINVAL; 9002 } 9003 for (i = 0; i < obj->nr_extern; i++) { 9004 ext = &obj->externs[i]; 9005 9006 if (!ext->is_set && !ext->is_weak) { 9007 pr_warn("extern '%s' (strong): not resolved\n", ext->name); 9008 return -ESRCH; 9009 } else if (!ext->is_set) { 9010 pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n", 9011 ext->name); 9012 } 9013 } 9014 9015 return 0; 9016 } 9017 9018 static void bpf_map_prepare_vdata(const struct bpf_map *map) 9019 { 9020 const struct btf_type *type; 9021 struct bpf_struct_ops *st_ops; 9022 __u32 i; 9023 9024 st_ops = map->st_ops; 9025 type = btf__type_by_id(map->obj->btf, st_ops->type_id); 9026 for (i = 0; i < btf_vlen(type); i++) { 9027 struct bpf_program *prog = st_ops->progs[i]; 9028 void *kern_data; 9029 int prog_fd; 9030 9031 if (!prog) 9032 continue; 9033 9034 prog_fd = bpf_program__fd(prog); 9035 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; 9036 *(unsigned long *)kern_data = prog_fd; 9037 } 9038 } 9039 9040 static int bpf_object_prepare_struct_ops(struct bpf_object *obj) 9041 { 9042 struct bpf_map *map; 9043 int i; 9044 9045 for (i = 0; i < obj->nr_maps; i++) { 9046 map = &obj->maps[i]; 9047 9048 if (!bpf_map__is_struct_ops(map)) 9049 continue; 9050 9051 if (!map->autocreate) 9052 continue; 9053 9054 bpf_map_prepare_vdata(map); 9055 } 9056 9057 return 0; 9058 } 9059 9060 static void bpf_object_unpin(struct bpf_object *obj) 9061 { 9062 int i; 9063 9064 /* unpin any maps that were auto-pinned during load */ 9065 for (i = 0; i < obj->nr_maps; i++) 9066 if (obj->maps[i].pinned && !obj->maps[i].reused) 9067 bpf_map__unpin(&obj->maps[i], NULL); 9068 } 9069 9070 static void bpf_object_cleanup_btf(struct bpf_object *obj) 9071 { 9072 int i; 9073 9074 /* clean up module BTFs */ 9075 for (i = 0; i < obj->btf_module_cnt; i++) { 9076 close(obj->btf_modules[i].fd); 9077 btf__free(obj->btf_modules[i].btf); 9078 free(obj->btf_modules[i].name); 9079 } 9080 obj->btf_module_cnt = 0; 9081 obj->btf_module_cap = 0; 9082 obj->btf_modules_loaded = false; 9083 zfree(&obj->btf_modules); 9084 9085 /* clean up vmlinux BTF */ 9086 btf__free(obj->btf_vmlinux); 9087 obj->btf_vmlinux = NULL; 9088 } 9089 9090 static void bpf_object_post_load_cleanup(struct bpf_object *obj) 9091 { 9092 /* clean up fd_array */ 9093 zfree(&obj->fd_array); 9094 9095 /* clean up BTF */ 9096 bpf_object_cleanup_btf(obj); 9097 } 9098 9099 static int bpf_object_prepare(struct bpf_object *obj, const char *target_btf_path) 9100 { 9101 int err; 9102 9103 if (obj->state >= OBJ_PREPARED) { 9104 pr_warn("object '%s': prepare loading can't be attempted twice\n", obj->name); 9105 return -EINVAL; 9106 } 9107 9108 err = bpf_object_prepare_token(obj); 9109 err = err ? : bpf_object__probe_loading(obj); 9110 err = err ? : bpf_object__load_vmlinux_btf(obj, false); 9111 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); 9112 err = err ? : bpf_object__sanitize_maps(obj); 9113 err = err ? : bpf_object__init_kern_struct_ops_maps(obj); 9114 err = err ? : bpf_object_adjust_struct_ops_autoload(obj); 9115 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path); 9116 err = err ? : bpf_object__sanitize_and_load_btf(obj); 9117 err = err ? : bpf_object__create_maps(obj); 9118 err = err ? : bpf_object_prepare_progs(obj); 9119 9120 if (err) { 9121 bpf_object_unpin(obj); 9122 bpf_object_unload(obj); 9123 obj->state = OBJ_LOADED; 9124 return err; 9125 } 9126 9127 obj->state = OBJ_PREPARED; 9128 return 0; 9129 } 9130 9131 static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path) 9132 { 9133 int err; 9134 9135 if (!obj) 9136 return libbpf_err(-EINVAL); 9137 9138 if (obj->state >= OBJ_LOADED) { 9139 pr_warn("object '%s': load can't be attempted twice\n", obj->name); 9140 return libbpf_err(-EINVAL); 9141 } 9142 9143 /* Disallow kernel loading programs of non-native endianness but 9144 * permit cross-endian creation of "light skeleton". 9145 */ 9146 if (obj->gen_loader) { 9147 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); 9148 } else if (!is_native_endianness(obj)) { 9149 pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name); 9150 return libbpf_err(-LIBBPF_ERRNO__ENDIAN); 9151 } 9152 9153 if (obj->state < OBJ_PREPARED) { 9154 err = bpf_object_prepare(obj, target_btf_path); 9155 if (err) 9156 return libbpf_err(err); 9157 } 9158 err = bpf_object__load_progs(obj, extra_log_level); 9159 err = err ? : bpf_object_init_prog_arrays(obj); 9160 err = err ? : bpf_object_prepare_struct_ops(obj); 9161 9162 if (obj->gen_loader) { 9163 /* reset FDs */ 9164 if (obj->btf) 9165 btf__set_fd(obj->btf, -1); 9166 if (!err) 9167 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps); 9168 } 9169 9170 bpf_object_post_load_cleanup(obj); 9171 obj->state = OBJ_LOADED; /* doesn't matter if successfully or not */ 9172 9173 if (err) { 9174 bpf_object_unpin(obj); 9175 bpf_object_unload(obj); 9176 pr_warn("failed to load object '%s'\n", obj->path); 9177 return libbpf_err(err); 9178 } 9179 9180 return 0; 9181 } 9182 9183 int bpf_object__prepare(struct bpf_object *obj) 9184 { 9185 return libbpf_err(bpf_object_prepare(obj, NULL)); 9186 } 9187 9188 int bpf_object__load(struct bpf_object *obj) 9189 { 9190 return bpf_object_load(obj, 0, NULL); 9191 } 9192 9193 static int make_parent_dir(const char *path) 9194 { 9195 char *dname, *dir; 9196 int err = 0; 9197 9198 dname = strdup(path); 9199 if (dname == NULL) 9200 return -ENOMEM; 9201 9202 dir = dirname(dname); 9203 if (mkdir(dir, 0700) && errno != EEXIST) 9204 err = -errno; 9205 9206 free(dname); 9207 if (err) { 9208 pr_warn("failed to mkdir %s: %s\n", path, errstr(err)); 9209 } 9210 return err; 9211 } 9212 9213 static int check_path(const char *path) 9214 { 9215 struct statfs st_fs; 9216 char *dname, *dir; 9217 int err = 0; 9218 9219 if (path == NULL) 9220 return -EINVAL; 9221 9222 dname = strdup(path); 9223 if (dname == NULL) 9224 return -ENOMEM; 9225 9226 dir = dirname(dname); 9227 if (statfs(dir, &st_fs)) { 9228 pr_warn("failed to statfs %s: %s\n", dir, errstr(errno)); 9229 err = -errno; 9230 } 9231 free(dname); 9232 9233 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 9234 pr_warn("specified path %s is not on BPF FS\n", path); 9235 err = -EINVAL; 9236 } 9237 9238 return err; 9239 } 9240 9241 int bpf_program__pin(struct bpf_program *prog, const char *path) 9242 { 9243 int err; 9244 9245 if (prog->fd < 0) { 9246 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name); 9247 return libbpf_err(-EINVAL); 9248 } 9249 9250 err = make_parent_dir(path); 9251 if (err) 9252 return libbpf_err(err); 9253 9254 err = check_path(path); 9255 if (err) 9256 return libbpf_err(err); 9257 9258 if (bpf_obj_pin(prog->fd, path)) { 9259 err = -errno; 9260 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, errstr(err)); 9261 return libbpf_err(err); 9262 } 9263 9264 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path); 9265 return 0; 9266 } 9267 9268 int bpf_program__unpin(struct bpf_program *prog, const char *path) 9269 { 9270 int err; 9271 9272 if (prog->fd < 0) { 9273 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name); 9274 return libbpf_err(-EINVAL); 9275 } 9276 9277 err = check_path(path); 9278 if (err) 9279 return libbpf_err(err); 9280 9281 err = unlink(path); 9282 if (err) 9283 return libbpf_err(-errno); 9284 9285 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path); 9286 return 0; 9287 } 9288 9289 int bpf_map__pin(struct bpf_map *map, const char *path) 9290 { 9291 int err; 9292 9293 if (map == NULL) { 9294 pr_warn("invalid map pointer\n"); 9295 return libbpf_err(-EINVAL); 9296 } 9297 9298 if (map->fd < 0) { 9299 pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name); 9300 return libbpf_err(-EINVAL); 9301 } 9302 9303 if (map->pin_path) { 9304 if (path && strcmp(path, map->pin_path)) { 9305 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 9306 bpf_map__name(map), map->pin_path, path); 9307 return libbpf_err(-EINVAL); 9308 } else if (map->pinned) { 9309 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", 9310 bpf_map__name(map), map->pin_path); 9311 return 0; 9312 } 9313 } else { 9314 if (!path) { 9315 pr_warn("missing a path to pin map '%s' at\n", 9316 bpf_map__name(map)); 9317 return libbpf_err(-EINVAL); 9318 } else if (map->pinned) { 9319 pr_warn("map '%s' already pinned\n", bpf_map__name(map)); 9320 return libbpf_err(-EEXIST); 9321 } 9322 9323 map->pin_path = strdup(path); 9324 if (!map->pin_path) { 9325 err = -errno; 9326 goto out_err; 9327 } 9328 } 9329 9330 err = make_parent_dir(map->pin_path); 9331 if (err) 9332 return libbpf_err(err); 9333 9334 err = check_path(map->pin_path); 9335 if (err) 9336 return libbpf_err(err); 9337 9338 if (bpf_obj_pin(map->fd, map->pin_path)) { 9339 err = -errno; 9340 goto out_err; 9341 } 9342 9343 map->pinned = true; 9344 pr_debug("pinned map '%s'\n", map->pin_path); 9345 9346 return 0; 9347 9348 out_err: 9349 pr_warn("failed to pin map: %s\n", errstr(err)); 9350 return libbpf_err(err); 9351 } 9352 9353 int bpf_map__unpin(struct bpf_map *map, const char *path) 9354 { 9355 int err; 9356 9357 if (map == NULL) { 9358 pr_warn("invalid map pointer\n"); 9359 return libbpf_err(-EINVAL); 9360 } 9361 9362 if (map->pin_path) { 9363 if (path && strcmp(path, map->pin_path)) { 9364 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 9365 bpf_map__name(map), map->pin_path, path); 9366 return libbpf_err(-EINVAL); 9367 } 9368 path = map->pin_path; 9369 } else if (!path) { 9370 pr_warn("no path to unpin map '%s' from\n", 9371 bpf_map__name(map)); 9372 return libbpf_err(-EINVAL); 9373 } 9374 9375 err = check_path(path); 9376 if (err) 9377 return libbpf_err(err); 9378 9379 err = unlink(path); 9380 if (err != 0) 9381 return libbpf_err(-errno); 9382 9383 map->pinned = false; 9384 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); 9385 9386 return 0; 9387 } 9388 9389 int bpf_map__set_pin_path(struct bpf_map *map, const char *path) 9390 { 9391 char *new = NULL; 9392 9393 if (path) { 9394 new = strdup(path); 9395 if (!new) 9396 return libbpf_err(-errno); 9397 } 9398 9399 free(map->pin_path); 9400 map->pin_path = new; 9401 return 0; 9402 } 9403 9404 __alias(bpf_map__pin_path) 9405 const char *bpf_map__get_pin_path(const struct bpf_map *map); 9406 9407 const char *bpf_map__pin_path(const struct bpf_map *map) 9408 { 9409 return map->pin_path; 9410 } 9411 9412 bool bpf_map__is_pinned(const struct bpf_map *map) 9413 { 9414 return map->pinned; 9415 } 9416 9417 static void sanitize_pin_path(char *s) 9418 { 9419 /* bpffs disallows periods in path names */ 9420 while (*s) { 9421 if (*s == '.') 9422 *s = '_'; 9423 s++; 9424 } 9425 } 9426 9427 int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 9428 { 9429 struct bpf_map *map; 9430 int err; 9431 9432 if (!obj) 9433 return libbpf_err(-ENOENT); 9434 9435 if (obj->state < OBJ_PREPARED) { 9436 pr_warn("object not yet loaded; load it first\n"); 9437 return libbpf_err(-ENOENT); 9438 } 9439 9440 bpf_object__for_each_map(map, obj) { 9441 char *pin_path = NULL; 9442 char buf[PATH_MAX]; 9443 9444 if (!map->autocreate) 9445 continue; 9446 9447 if (path) { 9448 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 9449 if (err) 9450 goto err_unpin_maps; 9451 sanitize_pin_path(buf); 9452 pin_path = buf; 9453 } else if (!map->pin_path) { 9454 continue; 9455 } 9456 9457 err = bpf_map__pin(map, pin_path); 9458 if (err) 9459 goto err_unpin_maps; 9460 } 9461 9462 return 0; 9463 9464 err_unpin_maps: 9465 while ((map = bpf_object__prev_map(obj, map))) { 9466 if (!map->pin_path) 9467 continue; 9468 9469 bpf_map__unpin(map, NULL); 9470 } 9471 9472 return libbpf_err(err); 9473 } 9474 9475 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 9476 { 9477 struct bpf_map *map; 9478 int err; 9479 9480 if (!obj) 9481 return libbpf_err(-ENOENT); 9482 9483 bpf_object__for_each_map(map, obj) { 9484 char *pin_path = NULL; 9485 char buf[PATH_MAX]; 9486 9487 if (path) { 9488 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 9489 if (err) 9490 return libbpf_err(err); 9491 sanitize_pin_path(buf); 9492 pin_path = buf; 9493 } else if (!map->pin_path) { 9494 continue; 9495 } 9496 9497 err = bpf_map__unpin(map, pin_path); 9498 if (err) 9499 return libbpf_err(err); 9500 } 9501 9502 return 0; 9503 } 9504 9505 int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 9506 { 9507 struct bpf_program *prog; 9508 char buf[PATH_MAX]; 9509 int err; 9510 9511 if (!obj) 9512 return libbpf_err(-ENOENT); 9513 9514 if (obj->state < OBJ_LOADED) { 9515 pr_warn("object not yet loaded; load it first\n"); 9516 return libbpf_err(-ENOENT); 9517 } 9518 9519 bpf_object__for_each_program(prog, obj) { 9520 err = pathname_concat(buf, sizeof(buf), path, prog->name); 9521 if (err) 9522 goto err_unpin_programs; 9523 9524 err = bpf_program__pin(prog, buf); 9525 if (err) 9526 goto err_unpin_programs; 9527 } 9528 9529 return 0; 9530 9531 err_unpin_programs: 9532 while ((prog = bpf_object__prev_program(obj, prog))) { 9533 if (pathname_concat(buf, sizeof(buf), path, prog->name)) 9534 continue; 9535 9536 bpf_program__unpin(prog, buf); 9537 } 9538 9539 return libbpf_err(err); 9540 } 9541 9542 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 9543 { 9544 struct bpf_program *prog; 9545 int err; 9546 9547 if (!obj) 9548 return libbpf_err(-ENOENT); 9549 9550 bpf_object__for_each_program(prog, obj) { 9551 char buf[PATH_MAX]; 9552 9553 err = pathname_concat(buf, sizeof(buf), path, prog->name); 9554 if (err) 9555 return libbpf_err(err); 9556 9557 err = bpf_program__unpin(prog, buf); 9558 if (err) 9559 return libbpf_err(err); 9560 } 9561 9562 return 0; 9563 } 9564 9565 int bpf_object__pin(struct bpf_object *obj, const char *path) 9566 { 9567 int err; 9568 9569 err = bpf_object__pin_maps(obj, path); 9570 if (err) 9571 return libbpf_err(err); 9572 9573 err = bpf_object__pin_programs(obj, path); 9574 if (err) { 9575 bpf_object__unpin_maps(obj, path); 9576 return libbpf_err(err); 9577 } 9578 9579 return 0; 9580 } 9581 9582 int bpf_object__unpin(struct bpf_object *obj, const char *path) 9583 { 9584 int err; 9585 9586 err = bpf_object__unpin_programs(obj, path); 9587 if (err) 9588 return libbpf_err(err); 9589 9590 err = bpf_object__unpin_maps(obj, path); 9591 if (err) 9592 return libbpf_err(err); 9593 9594 return 0; 9595 } 9596 9597 static void bpf_map__destroy(struct bpf_map *map) 9598 { 9599 if (map->inner_map) { 9600 bpf_map__destroy(map->inner_map); 9601 zfree(&map->inner_map); 9602 } 9603 9604 zfree(&map->init_slots); 9605 map->init_slots_sz = 0; 9606 9607 if (map->mmaped && map->mmaped != map->obj->arena_data) 9608 munmap(map->mmaped, bpf_map_mmap_sz(map)); 9609 map->mmaped = NULL; 9610 9611 if (map->st_ops) { 9612 zfree(&map->st_ops->data); 9613 zfree(&map->st_ops->progs); 9614 zfree(&map->st_ops->kern_func_off); 9615 zfree(&map->st_ops); 9616 } 9617 9618 zfree(&map->name); 9619 zfree(&map->real_name); 9620 zfree(&map->pin_path); 9621 9622 if (map->fd >= 0) 9623 zclose(map->fd); 9624 } 9625 9626 void bpf_object__close(struct bpf_object *obj) 9627 { 9628 size_t i; 9629 9630 if (IS_ERR_OR_NULL(obj)) 9631 return; 9632 9633 /* 9634 * if user called bpf_object__prepare() without ever getting to 9635 * bpf_object__load(), we need to clean up stuff that is normally 9636 * cleaned up at the end of loading step 9637 */ 9638 bpf_object_post_load_cleanup(obj); 9639 9640 usdt_manager_free(obj->usdt_man); 9641 obj->usdt_man = NULL; 9642 9643 bpf_gen__free(obj->gen_loader); 9644 bpf_object__elf_finish(obj); 9645 bpf_object_unload(obj); 9646 btf__free(obj->btf); 9647 btf__free(obj->btf_vmlinux); 9648 btf_ext__free(obj->btf_ext); 9649 9650 for (i = 0; i < obj->nr_maps; i++) 9651 bpf_map__destroy(&obj->maps[i]); 9652 9653 zfree(&obj->btf_custom_path); 9654 zfree(&obj->kconfig); 9655 9656 for (i = 0; i < obj->nr_extern; i++) { 9657 zfree(&obj->externs[i].name); 9658 zfree(&obj->externs[i].essent_name); 9659 } 9660 9661 zfree(&obj->externs); 9662 obj->nr_extern = 0; 9663 9664 zfree(&obj->maps); 9665 obj->nr_maps = 0; 9666 9667 if (obj->programs && obj->nr_programs) { 9668 for (i = 0; i < obj->nr_programs; i++) 9669 bpf_program__exit(&obj->programs[i]); 9670 } 9671 zfree(&obj->programs); 9672 9673 zfree(&obj->feat_cache); 9674 zfree(&obj->token_path); 9675 if (obj->token_fd > 0) 9676 close(obj->token_fd); 9677 9678 zfree(&obj->arena_data); 9679 9680 zfree(&obj->jumptables_data); 9681 obj->jumptables_data_sz = 0; 9682 9683 for (i = 0; i < obj->jumptable_map_cnt; i++) 9684 close(obj->jumptable_maps[i].fd); 9685 zfree(&obj->jumptable_maps); 9686 9687 free(obj); 9688 } 9689 9690 const char *bpf_object__name(const struct bpf_object *obj) 9691 { 9692 return obj ? obj->name : libbpf_err_ptr(-EINVAL); 9693 } 9694 9695 unsigned int bpf_object__kversion(const struct bpf_object *obj) 9696 { 9697 return obj ? obj->kern_version : 0; 9698 } 9699 9700 int bpf_object__token_fd(const struct bpf_object *obj) 9701 { 9702 return obj->token_fd ?: -1; 9703 } 9704 9705 struct btf *bpf_object__btf(const struct bpf_object *obj) 9706 { 9707 return obj ? obj->btf : NULL; 9708 } 9709 9710 int bpf_object__btf_fd(const struct bpf_object *obj) 9711 { 9712 return obj->btf ? btf__fd(obj->btf) : -1; 9713 } 9714 9715 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version) 9716 { 9717 if (obj->state >= OBJ_LOADED) 9718 return libbpf_err(-EINVAL); 9719 9720 obj->kern_version = kern_version; 9721 9722 return 0; 9723 } 9724 9725 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) 9726 { 9727 struct bpf_gen *gen; 9728 9729 if (!opts) 9730 return libbpf_err(-EFAULT); 9731 if (!OPTS_VALID(opts, gen_loader_opts)) 9732 return libbpf_err(-EINVAL); 9733 gen = calloc(1, sizeof(*gen)); 9734 if (!gen) 9735 return libbpf_err(-ENOMEM); 9736 gen->opts = opts; 9737 gen->swapped_endian = !is_native_endianness(obj); 9738 obj->gen_loader = gen; 9739 return 0; 9740 } 9741 9742 static struct bpf_program * 9743 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 9744 bool forward) 9745 { 9746 size_t nr_programs = obj->nr_programs; 9747 ssize_t idx; 9748 9749 if (!nr_programs) 9750 return NULL; 9751 9752 if (!p) 9753 /* Iter from the beginning */ 9754 return forward ? &obj->programs[0] : 9755 &obj->programs[nr_programs - 1]; 9756 9757 if (p->obj != obj) { 9758 pr_warn("error: program handler doesn't match object\n"); 9759 return errno = EINVAL, NULL; 9760 } 9761 9762 idx = (p - obj->programs) + (forward ? 1 : -1); 9763 if (idx >= obj->nr_programs || idx < 0) 9764 return NULL; 9765 return &obj->programs[idx]; 9766 } 9767 9768 struct bpf_program * 9769 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev) 9770 { 9771 struct bpf_program *prog = prev; 9772 9773 do { 9774 prog = __bpf_program__iter(prog, obj, true); 9775 } while (prog && prog_is_subprog(obj, prog)); 9776 9777 return prog; 9778 } 9779 9780 struct bpf_program * 9781 bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next) 9782 { 9783 struct bpf_program *prog = next; 9784 9785 do { 9786 prog = __bpf_program__iter(prog, obj, false); 9787 } while (prog && prog_is_subprog(obj, prog)); 9788 9789 return prog; 9790 } 9791 9792 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 9793 { 9794 prog->prog_ifindex = ifindex; 9795 } 9796 9797 const char *bpf_program__name(const struct bpf_program *prog) 9798 { 9799 return prog->name; 9800 } 9801 9802 const char *bpf_program__section_name(const struct bpf_program *prog) 9803 { 9804 return prog->sec_name; 9805 } 9806 9807 bool bpf_program__autoload(const struct bpf_program *prog) 9808 { 9809 return prog->autoload; 9810 } 9811 9812 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) 9813 { 9814 if (prog->obj->state >= OBJ_LOADED) 9815 return libbpf_err(-EINVAL); 9816 9817 prog->autoload = autoload; 9818 return 0; 9819 } 9820 9821 bool bpf_program__autoattach(const struct bpf_program *prog) 9822 { 9823 return prog->autoattach; 9824 } 9825 9826 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach) 9827 { 9828 prog->autoattach = autoattach; 9829 } 9830 9831 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog) 9832 { 9833 return prog->insns; 9834 } 9835 9836 size_t bpf_program__insn_cnt(const struct bpf_program *prog) 9837 { 9838 return prog->insns_cnt; 9839 } 9840 9841 int bpf_program__set_insns(struct bpf_program *prog, 9842 struct bpf_insn *new_insns, size_t new_insn_cnt) 9843 { 9844 struct bpf_insn *insns; 9845 9846 if (prog->obj->state >= OBJ_LOADED) 9847 return libbpf_err(-EBUSY); 9848 9849 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); 9850 /* NULL is a valid return from reallocarray if the new count is zero */ 9851 if (!insns && new_insn_cnt) { 9852 pr_warn("prog '%s': failed to realloc prog code\n", prog->name); 9853 return libbpf_err(-ENOMEM); 9854 } 9855 memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns)); 9856 9857 prog->insns = insns; 9858 prog->insns_cnt = new_insn_cnt; 9859 return 0; 9860 } 9861 9862 int bpf_program__fd(const struct bpf_program *prog) 9863 { 9864 if (!prog) 9865 return libbpf_err(-EINVAL); 9866 9867 if (prog->fd < 0) 9868 return libbpf_err(-ENOENT); 9869 9870 return prog->fd; 9871 } 9872 9873 __alias(bpf_program__type) 9874 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog); 9875 9876 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog) 9877 { 9878 return prog->type; 9879 } 9880 9881 static size_t custom_sec_def_cnt; 9882 static struct bpf_sec_def *custom_sec_defs; 9883 static struct bpf_sec_def custom_fallback_def; 9884 static bool has_custom_fallback_def; 9885 static int last_custom_sec_def_handler_id; 9886 9887 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 9888 { 9889 if (prog->obj->state >= OBJ_LOADED) 9890 return libbpf_err(-EBUSY); 9891 9892 /* if type is not changed, do nothing */ 9893 if (prog->type == type) 9894 return 0; 9895 9896 prog->type = type; 9897 9898 /* If a program type was changed, we need to reset associated SEC() 9899 * handler, as it will be invalid now. The only exception is a generic 9900 * fallback handler, which by definition is program type-agnostic and 9901 * is a catch-all custom handler, optionally set by the application, 9902 * so should be able to handle any type of BPF program. 9903 */ 9904 if (prog->sec_def != &custom_fallback_def) 9905 prog->sec_def = NULL; 9906 return 0; 9907 } 9908 9909 __alias(bpf_program__expected_attach_type) 9910 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog); 9911 9912 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog) 9913 { 9914 return prog->expected_attach_type; 9915 } 9916 9917 int bpf_program__set_expected_attach_type(struct bpf_program *prog, 9918 enum bpf_attach_type type) 9919 { 9920 if (prog->obj->state >= OBJ_LOADED) 9921 return libbpf_err(-EBUSY); 9922 9923 prog->expected_attach_type = type; 9924 return 0; 9925 } 9926 9927 __u32 bpf_program__flags(const struct bpf_program *prog) 9928 { 9929 return prog->prog_flags; 9930 } 9931 9932 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags) 9933 { 9934 if (prog->obj->state >= OBJ_LOADED) 9935 return libbpf_err(-EBUSY); 9936 9937 prog->prog_flags = flags; 9938 return 0; 9939 } 9940 9941 __u32 bpf_program__log_level(const struct bpf_program *prog) 9942 { 9943 return prog->log_level; 9944 } 9945 9946 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level) 9947 { 9948 if (prog->obj->state >= OBJ_LOADED) 9949 return libbpf_err(-EBUSY); 9950 9951 prog->log_level = log_level; 9952 return 0; 9953 } 9954 9955 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size) 9956 { 9957 *log_size = prog->log_size; 9958 return prog->log_buf; 9959 } 9960 9961 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size) 9962 { 9963 if (log_size && !log_buf) 9964 return libbpf_err(-EINVAL); 9965 if (prog->log_size > UINT_MAX) 9966 return libbpf_err(-EINVAL); 9967 if (prog->obj->state >= OBJ_LOADED) 9968 return libbpf_err(-EBUSY); 9969 9970 prog->log_buf = log_buf; 9971 prog->log_size = log_size; 9972 return 0; 9973 } 9974 9975 struct bpf_func_info *bpf_program__func_info(const struct bpf_program *prog) 9976 { 9977 if (prog->func_info_rec_size != sizeof(struct bpf_func_info)) 9978 return libbpf_err_ptr(-EOPNOTSUPP); 9979 return prog->func_info; 9980 } 9981 9982 __u32 bpf_program__func_info_cnt(const struct bpf_program *prog) 9983 { 9984 return prog->func_info_cnt; 9985 } 9986 9987 struct bpf_line_info *bpf_program__line_info(const struct bpf_program *prog) 9988 { 9989 if (prog->line_info_rec_size != sizeof(struct bpf_line_info)) 9990 return libbpf_err_ptr(-EOPNOTSUPP); 9991 return prog->line_info; 9992 } 9993 9994 __u32 bpf_program__line_info_cnt(const struct bpf_program *prog) 9995 { 9996 return prog->line_info_cnt; 9997 } 9998 9999 int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts *opts) 10000 { 10001 LIBBPF_OPTS(bpf_prog_load_opts, attr); 10002 struct bpf_object *obj; 10003 const void *info; 10004 __u32 info_cnt, info_rec_size; 10005 int err, fd, prog_btf_fd; 10006 10007 if (!prog) 10008 return libbpf_err(-EINVAL); 10009 10010 if (!OPTS_VALID(opts, bpf_prog_load_opts)) 10011 return libbpf_err(-EINVAL); 10012 10013 obj = prog->obj; 10014 if (obj->state < OBJ_PREPARED) 10015 return libbpf_err(-EINVAL); 10016 10017 /* 10018 * Caller-provided opts take priority; fall back to 10019 * prog/object defaults when the caller leaves them zero. 10020 */ 10021 attr.attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0) ?: prog->attach_prog_fd; 10022 attr.prog_flags = OPTS_GET(opts, prog_flags, 0) ?: prog->prog_flags; 10023 attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0) ?: prog->prog_ifindex; 10024 attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version; 10025 attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array; 10026 attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0) ?: obj->fd_array_cnt; 10027 attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd; 10028 if (attr.token_fd) 10029 attr.prog_flags |= BPF_F_TOKEN_FD; 10030 10031 prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0); 10032 if (!prog_btf_fd && obj->btf) 10033 prog_btf_fd = btf__fd(obj->btf); 10034 10035 /* BTF func/line info: only pass if kernel supports it */ 10036 if (kernel_supports(obj, FEAT_BTF_FUNC) && prog_btf_fd > 0) { 10037 attr.prog_btf_fd = prog_btf_fd; 10038 10039 /* func_info/line_info triples: all-or-nothing from caller */ 10040 info = OPTS_GET(opts, func_info, NULL); 10041 info_cnt = OPTS_GET(opts, func_info_cnt, 0); 10042 info_rec_size = OPTS_GET(opts, func_info_rec_size, 0); 10043 if (!!info != !!info_cnt || !!info != !!info_rec_size) { 10044 pr_warn("prog '%s': func_info, func_info_cnt, and func_info_rec_size must all be specified or all omitted\n", 10045 prog->name); 10046 return libbpf_err(-EINVAL); 10047 } 10048 attr.func_info = info ?: prog->func_info; 10049 attr.func_info_cnt = info ? info_cnt : prog->func_info_cnt; 10050 attr.func_info_rec_size = info ? info_rec_size : prog->func_info_rec_size; 10051 10052 info = OPTS_GET(opts, line_info, NULL); 10053 info_cnt = OPTS_GET(opts, line_info_cnt, 0); 10054 info_rec_size = OPTS_GET(opts, line_info_rec_size, 0); 10055 if (!!info != !!info_cnt || !!info != !!info_rec_size) { 10056 pr_warn("prog '%s': line_info, line_info_cnt, and line_info_rec_size must all be specified or all omitted\n", 10057 prog->name); 10058 return libbpf_err(-EINVAL); 10059 } 10060 attr.line_info = info ?: prog->line_info; 10061 attr.line_info_cnt = info ? info_cnt : prog->line_info_cnt; 10062 attr.line_info_rec_size = info ? info_rec_size : prog->line_info_rec_size; 10063 } 10064 10065 /* Logging is caller-controlled; no fallback to prog/obj log settings */ 10066 attr.log_buf = OPTS_GET(opts, log_buf, NULL); 10067 attr.log_size = OPTS_GET(opts, log_size, 0); 10068 attr.log_level = OPTS_GET(opts, log_level, 0); 10069 10070 /* 10071 * Fields below may be mutated by prog_prepare_load_fn: 10072 * Seed them from prog/obj defaults here; 10073 * Later override with caller-provided opts. 10074 */ 10075 attr.expected_attach_type = prog->expected_attach_type; 10076 attr.attach_btf_id = prog->attach_btf_id; 10077 attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; 10078 10079 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) { 10080 err = prog->sec_def->prog_prepare_load_fn(prog, &attr, prog->sec_def->cookie); 10081 if (err) 10082 return libbpf_err(err); 10083 } 10084 10085 /* Re-apply caller overrides for output fields */ 10086 if (OPTS_GET(opts, expected_attach_type, 0)) 10087 attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0); 10088 if (OPTS_GET(opts, attach_btf_id, 0)) 10089 attr.attach_btf_id = OPTS_GET(opts, attach_btf_id, 0); 10090 if (OPTS_GET(opts, attach_btf_obj_fd, 0)) 10091 attr.attach_btf_obj_fd = OPTS_GET(opts, attach_btf_obj_fd, 0); 10092 10093 /* 10094 * Unlike bpf_object_load_prog(), we intentionally do not call bpf_prog_bind_map() 10095 * for RODATA maps here to avoid mutating the object's state. Callers can bind the 10096 * required maps themselves using bpf_prog_bind_map(). 10097 */ 10098 fd = bpf_prog_load(prog->type, prog->name, obj->license, prog->insns, prog->insns_cnt, 10099 &attr); 10100 10101 return libbpf_err(fd); 10102 } 10103 10104 #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \ 10105 .sec = (char *)sec_pfx, \ 10106 .prog_type = BPF_PROG_TYPE_##ptype, \ 10107 .expected_attach_type = atype, \ 10108 .cookie = (long)(flags), \ 10109 .prog_prepare_load_fn = libbpf_prepare_prog_load, \ 10110 __VA_ARGS__ \ 10111 } 10112 10113 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10114 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10115 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10116 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10117 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10118 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10119 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10120 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10121 static int attach_kprobe_session(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10122 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10123 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10124 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10125 static int attach_tracing_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); 10126 10127 static const struct bpf_sec_def section_defs[] = { 10128 SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE), 10129 SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE), 10130 SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE), 10131 SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), 10132 SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), 10133 SEC_DEF("uprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), 10134 SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), 10135 SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), 10136 SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), 10137 SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), 10138 SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), 10139 SEC_DEF("kprobe.session+", KPROBE, BPF_TRACE_KPROBE_SESSION, SEC_NONE, attach_kprobe_session), 10140 SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi), 10141 SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi), 10142 SEC_DEF("uprobe.session+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_NONE, attach_uprobe_multi), 10143 SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), 10144 SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), 10145 SEC_DEF("uprobe.session.s+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_SLEEPABLE, attach_uprobe_multi), 10146 SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), 10147 SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), 10148 SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt), 10149 SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT | SEC_SLEEPABLE, attach_usdt), 10150 SEC_DEF("tc/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */ 10151 SEC_DEF("tc/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), /* alias for tcx */ 10152 SEC_DEF("tcx/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), 10153 SEC_DEF("tcx/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), 10154 SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 10155 SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 10156 SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 10157 SEC_DEF("netkit/primary", SCHED_CLS, BPF_NETKIT_PRIMARY, SEC_NONE), 10158 SEC_DEF("netkit/peer", SCHED_CLS, BPF_NETKIT_PEER, SEC_NONE), 10159 SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp), 10160 SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp), 10161 SEC_DEF("tracepoint.s+", TRACEPOINT, 0, SEC_SLEEPABLE, attach_tp), 10162 SEC_DEF("tp.s+", TRACEPOINT, 0, SEC_SLEEPABLE, attach_tp), 10163 SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), 10164 SEC_DEF("raw_tp+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), 10165 SEC_DEF("raw_tracepoint.s+", RAW_TRACEPOINT, 0, SEC_SLEEPABLE, attach_raw_tp), 10166 SEC_DEF("raw_tp.s+", RAW_TRACEPOINT, 0, SEC_SLEEPABLE, attach_raw_tp), 10167 SEC_DEF("raw_tracepoint.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), 10168 SEC_DEF("raw_tp.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), 10169 SEC_DEF("tp_btf+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace), 10170 SEC_DEF("tp_btf.s+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 10171 SEC_DEF("fentry+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace), 10172 SEC_DEF("fmod_ret+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace), 10173 SEC_DEF("fexit+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace), 10174 SEC_DEF("fentry.s+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 10175 SEC_DEF("fmod_ret.s+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 10176 SEC_DEF("fexit.s+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 10177 SEC_DEF("fsession+", TRACING, BPF_TRACE_FSESSION, SEC_ATTACH_BTF, attach_trace), 10178 SEC_DEF("fsession.s+", TRACING, BPF_TRACE_FSESSION, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 10179 SEC_DEF("fsession.multi+", TRACING, BPF_TRACE_FSESSION_MULTI, 0, attach_tracing_multi), 10180 SEC_DEF("fsession.multi.s+", TRACING, BPF_TRACE_FSESSION_MULTI, SEC_SLEEPABLE, attach_tracing_multi), 10181 SEC_DEF("fentry.multi+", TRACING, BPF_TRACE_FENTRY_MULTI, 0, attach_tracing_multi), 10182 SEC_DEF("fexit.multi+", TRACING, BPF_TRACE_FEXIT_MULTI, 0, attach_tracing_multi), 10183 SEC_DEF("fentry.multi.s+", TRACING, BPF_TRACE_FENTRY_MULTI, SEC_SLEEPABLE, attach_tracing_multi), 10184 SEC_DEF("fexit.multi.s+", TRACING, BPF_TRACE_FEXIT_MULTI, SEC_SLEEPABLE, attach_tracing_multi), 10185 SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace), 10186 SEC_DEF("lsm+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm), 10187 SEC_DEF("lsm.s+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm), 10188 SEC_DEF("lsm_cgroup+", LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF), 10189 SEC_DEF("iter+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter), 10190 SEC_DEF("iter.s+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter), 10191 SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE), 10192 SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS), 10193 SEC_DEF("xdp/devmap", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE), 10194 SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS), 10195 SEC_DEF("xdp/cpumap", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE), 10196 SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS), 10197 SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT), 10198 SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE), 10199 SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE), 10200 SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE), 10201 SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE), 10202 SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE), 10203 SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT), 10204 SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT), 10205 SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT), 10206 SEC_DEF("sk_skb/verdict", SK_SKB, BPF_SK_SKB_VERDICT, SEC_ATTACHABLE_OPT), 10207 SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE), 10208 SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT), 10209 SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT), 10210 SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT), 10211 SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT), 10212 SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT), 10213 SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE), 10214 SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE), 10215 SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE), 10216 SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT), 10217 SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE), 10218 SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE), 10219 SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE), 10220 SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE), 10221 SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE), 10222 SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE), 10223 SEC_DEF("cgroup/connect_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_CONNECT, SEC_ATTACHABLE), 10224 SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE), 10225 SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE), 10226 SEC_DEF("cgroup/sendmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_SENDMSG, SEC_ATTACHABLE), 10227 SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE), 10228 SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE), 10229 SEC_DEF("cgroup/recvmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_RECVMSG, SEC_ATTACHABLE), 10230 SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE), 10231 SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE), 10232 SEC_DEF("cgroup/getpeername_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETPEERNAME, SEC_ATTACHABLE), 10233 SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE), 10234 SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE), 10235 SEC_DEF("cgroup/getsockname_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETSOCKNAME, SEC_ATTACHABLE), 10236 SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE), 10237 SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE), 10238 SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE), 10239 SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT), 10240 SEC_DEF("struct_ops+", STRUCT_OPS, 0, SEC_NONE), 10241 SEC_DEF("struct_ops.s+", STRUCT_OPS, 0, SEC_SLEEPABLE), 10242 SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE), 10243 SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE), 10244 }; 10245 10246 int libbpf_register_prog_handler(const char *sec, 10247 enum bpf_prog_type prog_type, 10248 enum bpf_attach_type exp_attach_type, 10249 const struct libbpf_prog_handler_opts *opts) 10250 { 10251 struct bpf_sec_def *sec_def; 10252 10253 if (!OPTS_VALID(opts, libbpf_prog_handler_opts)) 10254 return libbpf_err(-EINVAL); 10255 10256 if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */ 10257 return libbpf_err(-E2BIG); 10258 10259 if (sec) { 10260 sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1, 10261 sizeof(*sec_def)); 10262 if (!sec_def) 10263 return libbpf_err(-ENOMEM); 10264 10265 custom_sec_defs = sec_def; 10266 sec_def = &custom_sec_defs[custom_sec_def_cnt]; 10267 } else { 10268 if (has_custom_fallback_def) 10269 return libbpf_err(-EBUSY); 10270 10271 sec_def = &custom_fallback_def; 10272 } 10273 10274 sec_def->sec = sec ? strdup(sec) : NULL; 10275 if (sec && !sec_def->sec) 10276 return libbpf_err(-ENOMEM); 10277 10278 sec_def->prog_type = prog_type; 10279 sec_def->expected_attach_type = exp_attach_type; 10280 sec_def->cookie = OPTS_GET(opts, cookie, 0); 10281 10282 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL); 10283 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL); 10284 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL); 10285 10286 sec_def->handler_id = ++last_custom_sec_def_handler_id; 10287 10288 if (sec) 10289 custom_sec_def_cnt++; 10290 else 10291 has_custom_fallback_def = true; 10292 10293 return sec_def->handler_id; 10294 } 10295 10296 int libbpf_unregister_prog_handler(int handler_id) 10297 { 10298 struct bpf_sec_def *sec_defs; 10299 int i; 10300 10301 if (handler_id <= 0) 10302 return libbpf_err(-EINVAL); 10303 10304 if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) { 10305 memset(&custom_fallback_def, 0, sizeof(custom_fallback_def)); 10306 has_custom_fallback_def = false; 10307 return 0; 10308 } 10309 10310 for (i = 0; i < custom_sec_def_cnt; i++) { 10311 if (custom_sec_defs[i].handler_id == handler_id) 10312 break; 10313 } 10314 10315 if (i == custom_sec_def_cnt) 10316 return libbpf_err(-ENOENT); 10317 10318 free(custom_sec_defs[i].sec); 10319 for (i = i + 1; i < custom_sec_def_cnt; i++) 10320 custom_sec_defs[i - 1] = custom_sec_defs[i]; 10321 custom_sec_def_cnt--; 10322 10323 /* try to shrink the array, but it's ok if we couldn't */ 10324 sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs)); 10325 /* if new count is zero, reallocarray can return a valid NULL result; 10326 * in this case the previous pointer will be freed, so we *have to* 10327 * reassign old pointer to the new value (even if it's NULL) 10328 */ 10329 if (sec_defs || custom_sec_def_cnt == 0) 10330 custom_sec_defs = sec_defs; 10331 10332 return 0; 10333 } 10334 10335 static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name) 10336 { 10337 size_t len = strlen(sec_def->sec); 10338 10339 /* "type/" always has to have proper SEC("type/extras") form */ 10340 if (sec_def->sec[len - 1] == '/') { 10341 if (str_has_pfx(sec_name, sec_def->sec)) 10342 return true; 10343 return false; 10344 } 10345 10346 /* "type+" means it can be either exact SEC("type") or 10347 * well-formed SEC("type/extras") with proper '/' separator 10348 */ 10349 if (sec_def->sec[len - 1] == '+') { 10350 len--; 10351 /* not even a prefix */ 10352 if (strncmp(sec_name, sec_def->sec, len) != 0) 10353 return false; 10354 /* exact match or has '/' separator */ 10355 if (sec_name[len] == '\0' || sec_name[len] == '/') 10356 return true; 10357 return false; 10358 } 10359 10360 return strcmp(sec_name, sec_def->sec) == 0; 10361 } 10362 10363 static const struct bpf_sec_def *find_sec_def(const char *sec_name) 10364 { 10365 const struct bpf_sec_def *sec_def; 10366 int i, n; 10367 10368 n = custom_sec_def_cnt; 10369 for (i = 0; i < n; i++) { 10370 sec_def = &custom_sec_defs[i]; 10371 if (sec_def_matches(sec_def, sec_name)) 10372 return sec_def; 10373 } 10374 10375 n = ARRAY_SIZE(section_defs); 10376 for (i = 0; i < n; i++) { 10377 sec_def = §ion_defs[i]; 10378 if (sec_def_matches(sec_def, sec_name)) 10379 return sec_def; 10380 } 10381 10382 if (has_custom_fallback_def) 10383 return &custom_fallback_def; 10384 10385 return NULL; 10386 } 10387 10388 #define MAX_TYPE_NAME_SIZE 32 10389 10390 static char *libbpf_get_type_names(bool attach_type) 10391 { 10392 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; 10393 char *buf; 10394 10395 buf = malloc(len); 10396 if (!buf) 10397 return NULL; 10398 10399 buf[0] = '\0'; 10400 /* Forge string buf with all available names */ 10401 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 10402 const struct bpf_sec_def *sec_def = §ion_defs[i]; 10403 10404 if (attach_type) { 10405 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) 10406 continue; 10407 10408 if (!(sec_def->cookie & SEC_ATTACHABLE)) 10409 continue; 10410 } 10411 10412 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { 10413 free(buf); 10414 return NULL; 10415 } 10416 strcat(buf, " "); 10417 strcat(buf, section_defs[i].sec); 10418 } 10419 10420 return buf; 10421 } 10422 10423 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 10424 enum bpf_attach_type *expected_attach_type) 10425 { 10426 const struct bpf_sec_def *sec_def; 10427 char *type_names; 10428 10429 if (!name) 10430 return libbpf_err(-EINVAL); 10431 10432 sec_def = find_sec_def(name); 10433 if (sec_def) { 10434 *prog_type = sec_def->prog_type; 10435 *expected_attach_type = sec_def->expected_attach_type; 10436 return 0; 10437 } 10438 10439 pr_debug("failed to guess program type from ELF section '%s'\n", name); 10440 type_names = libbpf_get_type_names(false); 10441 if (type_names != NULL) { 10442 pr_debug("supported section(type) names are:%s\n", type_names); 10443 free(type_names); 10444 } 10445 10446 return libbpf_err(-ESRCH); 10447 } 10448 10449 const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t) 10450 { 10451 if (t < 0 || t >= ARRAY_SIZE(attach_type_name)) 10452 return NULL; 10453 10454 return attach_type_name[t]; 10455 } 10456 10457 const char *libbpf_bpf_link_type_str(enum bpf_link_type t) 10458 { 10459 if (t < 0 || t >= ARRAY_SIZE(link_type_name)) 10460 return NULL; 10461 10462 return link_type_name[t]; 10463 } 10464 10465 const char *libbpf_bpf_map_type_str(enum bpf_map_type t) 10466 { 10467 if (t < 0 || t >= ARRAY_SIZE(map_type_name)) 10468 return NULL; 10469 10470 return map_type_name[t]; 10471 } 10472 10473 const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t) 10474 { 10475 if (t < 0 || t >= ARRAY_SIZE(prog_type_name)) 10476 return NULL; 10477 10478 return prog_type_name[t]; 10479 } 10480 10481 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, 10482 int sec_idx, 10483 size_t offset) 10484 { 10485 struct bpf_map *map; 10486 size_t i; 10487 10488 for (i = 0; i < obj->nr_maps; i++) { 10489 map = &obj->maps[i]; 10490 if (!bpf_map__is_struct_ops(map)) 10491 continue; 10492 if (map->sec_idx == sec_idx && 10493 map->sec_offset <= offset && 10494 offset - map->sec_offset < map->def.value_size) 10495 return map; 10496 } 10497 10498 return NULL; 10499 } 10500 10501 /* Collect the reloc from ELF, populate the st_ops->progs[], and update 10502 * st_ops->data for shadow type. 10503 */ 10504 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 10505 Elf64_Shdr *shdr, Elf_Data *data) 10506 { 10507 const struct btf_type *type; 10508 const struct btf_member *member; 10509 struct bpf_struct_ops *st_ops; 10510 struct bpf_program *prog; 10511 unsigned int shdr_idx; 10512 const struct btf *btf; 10513 struct bpf_map *map; 10514 unsigned int moff, insn_idx; 10515 const char *name; 10516 __u32 member_idx; 10517 Elf64_Sym *sym; 10518 Elf64_Rel *rel; 10519 int i, nrels; 10520 10521 btf = obj->btf; 10522 nrels = shdr->sh_size / shdr->sh_entsize; 10523 for (i = 0; i < nrels; i++) { 10524 rel = elf_rel_by_idx(data, i); 10525 if (!rel) { 10526 pr_warn("struct_ops reloc: failed to get %d reloc\n", i); 10527 return -LIBBPF_ERRNO__FORMAT; 10528 } 10529 10530 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); 10531 if (!sym) { 10532 pr_warn("struct_ops reloc: symbol %zx not found\n", 10533 (size_t)ELF64_R_SYM(rel->r_info)); 10534 return -LIBBPF_ERRNO__FORMAT; 10535 } 10536 10537 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; 10538 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset); 10539 if (!map) { 10540 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n", 10541 (size_t)rel->r_offset); 10542 return -EINVAL; 10543 } 10544 10545 moff = rel->r_offset - map->sec_offset; 10546 shdr_idx = sym->st_shndx; 10547 st_ops = map->st_ops; 10548 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %u (\'%s\')\n", 10549 map->name, 10550 (long long)(rel->r_info >> 32), 10551 (long long)sym->st_value, 10552 shdr_idx, (size_t)rel->r_offset, 10553 map->sec_offset, sym->st_name, name); 10554 10555 if (shdr_idx >= SHN_LORESERVE) { 10556 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n", 10557 map->name, (size_t)rel->r_offset, shdr_idx); 10558 return -LIBBPF_ERRNO__RELOC; 10559 } 10560 if (sym->st_value % BPF_INSN_SZ) { 10561 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n", 10562 map->name, (unsigned long long)sym->st_value); 10563 return -LIBBPF_ERRNO__FORMAT; 10564 } 10565 insn_idx = sym->st_value / BPF_INSN_SZ; 10566 10567 type = btf__type_by_id(btf, st_ops->type_id); 10568 member = find_member_by_offset(type, moff * 8); 10569 if (!member) { 10570 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", 10571 map->name, moff); 10572 return -EINVAL; 10573 } 10574 member_idx = member - btf_members(type); 10575 name = btf__name_by_offset(btf, member->name_off); 10576 10577 if (!resolve_func_ptr(btf, member->type, NULL)) { 10578 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", 10579 map->name, name); 10580 return -EINVAL; 10581 } 10582 10583 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); 10584 if (!prog) { 10585 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", 10586 map->name, shdr_idx, name); 10587 return -EINVAL; 10588 } 10589 10590 /* prevent the use of BPF prog with invalid type */ 10591 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) { 10592 pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n", 10593 map->name, prog->name); 10594 return -EINVAL; 10595 } 10596 10597 st_ops->progs[member_idx] = prog; 10598 10599 /* st_ops->data will be exposed to users, being returned by 10600 * bpf_map__initial_value() as a pointer to the shadow 10601 * type. All function pointers in the original struct type 10602 * should be converted to a pointer to struct bpf_program 10603 * in the shadow type. 10604 */ 10605 *((struct bpf_program **)(st_ops->data + moff)) = prog; 10606 } 10607 10608 return 0; 10609 } 10610 10611 #define BTF_TRACE_PREFIX "btf_trace_" 10612 #define BTF_LSM_PREFIX "bpf_lsm_" 10613 #define BTF_ITER_PREFIX "bpf_iter_" 10614 #define BTF_MAX_NAME_SIZE 128 10615 10616 void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, 10617 const char **prefix, int *kind) 10618 { 10619 switch (attach_type) { 10620 case BPF_TRACE_RAW_TP: 10621 *prefix = BTF_TRACE_PREFIX; 10622 *kind = BTF_KIND_TYPEDEF; 10623 break; 10624 case BPF_LSM_MAC: 10625 case BPF_LSM_CGROUP: 10626 *prefix = BTF_LSM_PREFIX; 10627 *kind = BTF_KIND_FUNC; 10628 break; 10629 case BPF_TRACE_ITER: 10630 *prefix = BTF_ITER_PREFIX; 10631 *kind = BTF_KIND_FUNC; 10632 break; 10633 default: 10634 *prefix = ""; 10635 *kind = BTF_KIND_FUNC; 10636 } 10637 } 10638 10639 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 10640 const char *name, __u32 kind) 10641 { 10642 char btf_type_name[BTF_MAX_NAME_SIZE]; 10643 int ret; 10644 10645 ret = snprintf(btf_type_name, sizeof(btf_type_name), 10646 "%s%s", prefix, name); 10647 /* snprintf returns the number of characters written excluding the 10648 * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it 10649 * indicates truncation. 10650 */ 10651 if (ret < 0 || ret >= sizeof(btf_type_name)) 10652 return -ENAMETOOLONG; 10653 return btf__find_by_name_kind(btf, btf_type_name, kind); 10654 } 10655 10656 static inline int find_attach_btf_id(struct btf *btf, const char *name, 10657 enum bpf_attach_type attach_type) 10658 { 10659 const char *prefix; 10660 int kind; 10661 10662 btf_get_kernel_prefix_kind(attach_type, &prefix, &kind); 10663 return find_btf_by_prefix_kind(btf, prefix, name, kind); 10664 } 10665 10666 int libbpf_find_vmlinux_btf_id(const char *name, 10667 enum bpf_attach_type attach_type) 10668 { 10669 struct btf *btf; 10670 int err; 10671 10672 btf = btf__load_vmlinux_btf(); 10673 err = libbpf_get_error(btf); 10674 if (err) { 10675 pr_warn("vmlinux BTF is not found\n"); 10676 return libbpf_err(err); 10677 } 10678 10679 err = find_attach_btf_id(btf, name, attach_type); 10680 if (err <= 0) 10681 pr_warn("%s is not found in vmlinux BTF\n", name); 10682 10683 btf__free(btf); 10684 return libbpf_err(err); 10685 } 10686 10687 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int token_fd) 10688 { 10689 struct bpf_prog_info info; 10690 __u32 info_len = sizeof(info); 10691 struct btf *btf; 10692 int err; 10693 10694 memset(&info, 0, info_len); 10695 err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len); 10696 if (err) { 10697 pr_warn("failed bpf_prog_get_info_by_fd for FD %u: %s\n", 10698 attach_prog_fd, errstr(err)); 10699 return err; 10700 } 10701 10702 err = -EINVAL; 10703 if (!info.btf_id) { 10704 pr_warn("The target program doesn't have BTF\n"); 10705 goto out; 10706 } 10707 btf = btf_load_from_kernel(info.btf_id, NULL, token_fd); 10708 err = libbpf_get_error(btf); 10709 if (err) { 10710 pr_warn("Failed to get BTF %u of the program: %s\n", info.btf_id, errstr(err)); 10711 goto out; 10712 } 10713 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 10714 btf__free(btf); 10715 if (err <= 0) { 10716 pr_warn("%s is not found in prog's BTF\n", name); 10717 goto out; 10718 } 10719 out: 10720 return err; 10721 } 10722 10723 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name, 10724 enum bpf_attach_type attach_type, 10725 int *btf_obj_fd, int *btf_type_id) 10726 { 10727 int ret, i, mod_len = 0; 10728 const char *fn_name, *mod_name = NULL; 10729 10730 fn_name = strchr(attach_name, ':'); 10731 if (fn_name) { 10732 mod_name = attach_name; 10733 mod_len = fn_name - mod_name; 10734 fn_name++; 10735 } 10736 10737 if (!mod_name || strncmp(mod_name, "vmlinux", mod_len) == 0) { 10738 ret = find_attach_btf_id(obj->btf_vmlinux, 10739 mod_name ? fn_name : attach_name, 10740 attach_type); 10741 if (ret > 0) { 10742 *btf_obj_fd = 0; /* vmlinux BTF */ 10743 *btf_type_id = ret; 10744 return 0; 10745 } 10746 if (ret != -ENOENT) 10747 return ret; 10748 } 10749 10750 ret = load_module_btfs(obj); 10751 if (ret) 10752 return ret; 10753 10754 for (i = 0; i < obj->btf_module_cnt; i++) { 10755 const struct module_btf *mod = &obj->btf_modules[i]; 10756 10757 if (mod_name && strncmp(mod->name, mod_name, mod_len) != 0) 10758 continue; 10759 10760 ret = find_attach_btf_id(mod->btf, 10761 mod_name ? fn_name : attach_name, 10762 attach_type); 10763 if (ret > 0) { 10764 *btf_obj_fd = mod->fd; 10765 *btf_type_id = ret; 10766 return 0; 10767 } 10768 if (ret == -ENOENT) 10769 continue; 10770 10771 return ret; 10772 } 10773 10774 return -ESRCH; 10775 } 10776 10777 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, 10778 int *btf_obj_fd, int *btf_type_id) 10779 { 10780 enum bpf_attach_type attach_type = prog->expected_attach_type; 10781 __u32 attach_prog_fd = prog->attach_prog_fd; 10782 int err = 0; 10783 10784 /* BPF program's BTF ID */ 10785 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) { 10786 if (!attach_prog_fd) { 10787 pr_warn("prog '%s': attach program FD is not set\n", prog->name); 10788 return -EINVAL; 10789 } 10790 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd); 10791 if (err < 0) { 10792 pr_warn("prog '%s': failed to find BPF program (FD %u) BTF ID for '%s': %s\n", 10793 prog->name, attach_prog_fd, attach_name, errstr(err)); 10794 return err; 10795 } 10796 *btf_obj_fd = 0; 10797 *btf_type_id = err; 10798 return 0; 10799 } 10800 10801 /* kernel/module BTF ID */ 10802 if (prog->obj->gen_loader) { 10803 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); 10804 *btf_obj_fd = 0; 10805 *btf_type_id = 1; 10806 } else { 10807 err = find_kernel_btf_id(prog->obj, attach_name, 10808 attach_type, btf_obj_fd, 10809 btf_type_id); 10810 } 10811 if (err) { 10812 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %s\n", 10813 prog->name, attach_name, errstr(err)); 10814 return err; 10815 } 10816 return 0; 10817 } 10818 10819 int libbpf_attach_type_by_name(const char *name, 10820 enum bpf_attach_type *attach_type) 10821 { 10822 char *type_names; 10823 const struct bpf_sec_def *sec_def; 10824 10825 if (!name) 10826 return libbpf_err(-EINVAL); 10827 10828 sec_def = find_sec_def(name); 10829 if (!sec_def) { 10830 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); 10831 type_names = libbpf_get_type_names(true); 10832 if (type_names != NULL) { 10833 pr_debug("attachable section(type) names are:%s\n", type_names); 10834 free(type_names); 10835 } 10836 10837 return libbpf_err(-EINVAL); 10838 } 10839 10840 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) 10841 return libbpf_err(-EINVAL); 10842 if (!(sec_def->cookie & SEC_ATTACHABLE)) 10843 return libbpf_err(-EINVAL); 10844 10845 *attach_type = sec_def->expected_attach_type; 10846 return 0; 10847 } 10848 10849 int bpf_map__fd(const struct bpf_map *map) 10850 { 10851 if (!map) 10852 return libbpf_err(-EINVAL); 10853 if (!map_is_created(map)) 10854 return -1; 10855 return map->fd; 10856 } 10857 10858 static bool map_uses_real_name(const struct bpf_map *map) 10859 { 10860 /* Since libbpf started to support custom .data.* and .rodata.* maps, 10861 * their user-visible name differs from kernel-visible name. Users see 10862 * such map's corresponding ELF section name as a map name. 10863 * This check distinguishes .data/.rodata from .data.* and .rodata.* 10864 * maps to know which name has to be returned to the user. 10865 * Map name of the custom .percpu.* maps might be truncated to 10866 * BPF_OBJ_NAME_LEN-1 chars in internal_map_name(). Hence, percpu data 10867 * maps must use real name for their user-visible name. 10868 */ 10869 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0) 10870 return true; 10871 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0) 10872 return true; 10873 if (map->libbpf_type == LIBBPF_MAP_PERCPU) 10874 return true; 10875 return false; 10876 } 10877 10878 const char *bpf_map__name(const struct bpf_map *map) 10879 { 10880 if (!map) 10881 return NULL; 10882 10883 if (map_uses_real_name(map)) 10884 return map->real_name; 10885 10886 return map->name; 10887 } 10888 10889 enum bpf_map_type bpf_map__type(const struct bpf_map *map) 10890 { 10891 return map->def.type; 10892 } 10893 10894 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type) 10895 { 10896 if (map_is_created(map)) 10897 return libbpf_err(-EBUSY); 10898 map->def.type = type; 10899 return 0; 10900 } 10901 10902 __u32 bpf_map__map_flags(const struct bpf_map *map) 10903 { 10904 return map->def.map_flags; 10905 } 10906 10907 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags) 10908 { 10909 if (map_is_created(map)) 10910 return libbpf_err(-EBUSY); 10911 map->def.map_flags = flags; 10912 return 0; 10913 } 10914 10915 __u64 bpf_map__map_extra(const struct bpf_map *map) 10916 { 10917 return map->map_extra; 10918 } 10919 10920 int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra) 10921 { 10922 if (map_is_created(map)) 10923 return libbpf_err(-EBUSY); 10924 map->map_extra = map_extra; 10925 return 0; 10926 } 10927 10928 __u32 bpf_map__numa_node(const struct bpf_map *map) 10929 { 10930 return map->numa_node; 10931 } 10932 10933 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node) 10934 { 10935 if (map_is_created(map)) 10936 return libbpf_err(-EBUSY); 10937 map->numa_node = numa_node; 10938 return 0; 10939 } 10940 10941 __u32 bpf_map__key_size(const struct bpf_map *map) 10942 { 10943 return map->def.key_size; 10944 } 10945 10946 int bpf_map__set_key_size(struct bpf_map *map, __u32 size) 10947 { 10948 if (map_is_created(map)) 10949 return libbpf_err(-EBUSY); 10950 map->def.key_size = size; 10951 return 0; 10952 } 10953 10954 __u32 bpf_map__value_size(const struct bpf_map *map) 10955 { 10956 return map->def.value_size; 10957 } 10958 10959 static int map_btf_datasec_resize(struct bpf_map *map, __u32 size) 10960 { 10961 struct btf *btf; 10962 struct btf_type *datasec_type, *var_type; 10963 struct btf_var_secinfo *var; 10964 const struct btf_type *array_type; 10965 const struct btf_array *array; 10966 int vlen, element_sz, new_array_id; 10967 __u32 nr_elements; 10968 10969 /* check btf existence */ 10970 btf = bpf_object__btf(map->obj); 10971 if (!btf) 10972 return -ENOENT; 10973 10974 /* verify map is datasec */ 10975 datasec_type = btf_type_by_id(btf, bpf_map__btf_value_type_id(map)); 10976 if (!btf_is_datasec(datasec_type)) { 10977 pr_warn("map '%s': cannot be resized, map value type is not a datasec\n", 10978 bpf_map__name(map)); 10979 return -EINVAL; 10980 } 10981 10982 /* verify datasec has at least one var */ 10983 vlen = btf_vlen(datasec_type); 10984 if (vlen == 0) { 10985 pr_warn("map '%s': cannot be resized, map value datasec is empty\n", 10986 bpf_map__name(map)); 10987 return -EINVAL; 10988 } 10989 10990 /* verify last var in the datasec is an array */ 10991 var = &btf_var_secinfos(datasec_type)[vlen - 1]; 10992 var_type = btf_type_by_id(btf, var->type); 10993 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL); 10994 if (!btf_is_array(array_type)) { 10995 pr_warn("map '%s': cannot be resized, last var must be an array\n", 10996 bpf_map__name(map)); 10997 return -EINVAL; 10998 } 10999 11000 /* verify request size aligns with array */ 11001 array = btf_array(array_type); 11002 element_sz = btf__resolve_size(btf, array->type); 11003 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) { 11004 pr_warn("map '%s': cannot be resized, element size (%d) doesn't align with new total size (%u)\n", 11005 bpf_map__name(map), element_sz, size); 11006 return -EINVAL; 11007 } 11008 11009 /* create a new array based on the existing array, but with new length */ 11010 nr_elements = (size - var->offset) / element_sz; 11011 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements); 11012 if (new_array_id < 0) 11013 return new_array_id; 11014 11015 /* adding a new btf type invalidates existing pointers to btf objects, 11016 * so refresh pointers before proceeding 11017 */ 11018 datasec_type = btf_type_by_id(btf, map->btf_value_type_id); 11019 var = &btf_var_secinfos(datasec_type)[vlen - 1]; 11020 var_type = btf_type_by_id(btf, var->type); 11021 11022 /* finally update btf info */ 11023 datasec_type->size = size; 11024 var->size = size - var->offset; 11025 var_type->type = new_array_id; 11026 11027 return 0; 11028 } 11029 11030 int bpf_map__set_value_size(struct bpf_map *map, __u32 size) 11031 { 11032 if (map_is_created(map)) 11033 return libbpf_err(-EBUSY); 11034 11035 if (map->mmaped) { 11036 size_t mmap_old_sz, mmap_new_sz; 11037 int err; 11038 11039 if (map->def.type != BPF_MAP_TYPE_ARRAY && 11040 map->def.type != BPF_MAP_TYPE_PERCPU_ARRAY) 11041 return libbpf_err(-EOPNOTSUPP); 11042 11043 mmap_old_sz = bpf_map_mmap_sz(map); 11044 mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries); 11045 err = bpf_map_mmap_resize(map, mmap_old_sz, mmap_new_sz); 11046 if (err) { 11047 pr_warn("map '%s': failed to resize memory-mapped region: %s\n", 11048 bpf_map__name(map), errstr(err)); 11049 return libbpf_err(err); 11050 } 11051 err = map_btf_datasec_resize(map, size); 11052 if (err && err != -ENOENT) { 11053 pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %s\n", 11054 bpf_map__name(map), errstr(err)); 11055 map->btf_value_type_id = 0; 11056 map->btf_key_type_id = 0; 11057 } 11058 } 11059 11060 map->def.value_size = size; 11061 return 0; 11062 } 11063 11064 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 11065 { 11066 return map ? map->btf_key_type_id : 0; 11067 } 11068 11069 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 11070 { 11071 return map ? map->btf_value_type_id : 0; 11072 } 11073 11074 int bpf_map__set_initial_value(struct bpf_map *map, 11075 const void *data, size_t size) 11076 { 11077 size_t actual_sz; 11078 11079 if (map_is_created(map)) 11080 return libbpf_err(-EBUSY); 11081 11082 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG) 11083 return libbpf_err(-EINVAL); 11084 11085 if (map->def.type == BPF_MAP_TYPE_ARENA) 11086 actual_sz = map->obj->arena_data_sz; 11087 else 11088 actual_sz = map->def.value_size; 11089 if (size != actual_sz) 11090 return libbpf_err(-EINVAL); 11091 11092 memcpy(map->mmaped, data, size); 11093 return 0; 11094 } 11095 11096 void *bpf_map__initial_value(const struct bpf_map *map, size_t *psize) 11097 { 11098 if (bpf_map__is_struct_ops(map)) { 11099 if (psize) 11100 *psize = map->def.value_size; 11101 return map->st_ops->data; 11102 } 11103 11104 if (!map->mmaped) 11105 return NULL; 11106 11107 if (map->def.type == BPF_MAP_TYPE_ARENA) 11108 *psize = map->obj->arena_data_sz; 11109 else 11110 *psize = map->def.value_size; 11111 11112 return map->mmaped; 11113 } 11114 11115 bool bpf_map__is_internal(const struct bpf_map *map) 11116 { 11117 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 11118 } 11119 11120 __u32 bpf_map__ifindex(const struct bpf_map *map) 11121 { 11122 return map->map_ifindex; 11123 } 11124 11125 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 11126 { 11127 if (map_is_created(map)) 11128 return libbpf_err(-EBUSY); 11129 map->map_ifindex = ifindex; 11130 return 0; 11131 } 11132 11133 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 11134 { 11135 if (!bpf_map_type__is_map_in_map(map->def.type)) { 11136 pr_warn("error: unsupported map type\n"); 11137 return libbpf_err(-EINVAL); 11138 } 11139 if (map->inner_map_fd != -1) { 11140 pr_warn("error: inner_map_fd already specified\n"); 11141 return libbpf_err(-EINVAL); 11142 } 11143 if (map->inner_map) { 11144 bpf_map__destroy(map->inner_map); 11145 zfree(&map->inner_map); 11146 } 11147 map->inner_map_fd = fd; 11148 return 0; 11149 } 11150 11151 int bpf_map__set_exclusive_program(struct bpf_map *map, struct bpf_program *prog) 11152 { 11153 if (map_is_created(map)) { 11154 pr_warn("exclusive programs must be set before map creation\n"); 11155 return libbpf_err(-EINVAL); 11156 } 11157 11158 if (map->obj != prog->obj) { 11159 pr_warn("excl_prog and map must be from the same bpf object\n"); 11160 return libbpf_err(-EINVAL); 11161 } 11162 11163 map->excl_prog = prog; 11164 return 0; 11165 } 11166 11167 struct bpf_program *bpf_map__exclusive_program(struct bpf_map *map) 11168 { 11169 return map->excl_prog; 11170 } 11171 11172 static struct bpf_map * 11173 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 11174 { 11175 ssize_t idx; 11176 struct bpf_map *s, *e; 11177 11178 if (!obj || !obj->maps) 11179 return errno = EINVAL, NULL; 11180 11181 s = obj->maps; 11182 e = obj->maps + obj->nr_maps; 11183 11184 if ((m < s) || (m >= e)) { 11185 pr_warn("error in %s: map handler doesn't belong to object\n", 11186 __func__); 11187 return errno = EINVAL, NULL; 11188 } 11189 11190 idx = (m - obj->maps) + i; 11191 if (idx >= obj->nr_maps || idx < 0) 11192 return NULL; 11193 return &obj->maps[idx]; 11194 } 11195 11196 struct bpf_map * 11197 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev) 11198 { 11199 if (prev == NULL && obj != NULL) 11200 return obj->maps; 11201 11202 return __bpf_map__iter(prev, obj, 1); 11203 } 11204 11205 struct bpf_map * 11206 bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next) 11207 { 11208 if (next == NULL && obj != NULL) { 11209 if (!obj->nr_maps) 11210 return NULL; 11211 return obj->maps + obj->nr_maps - 1; 11212 } 11213 11214 return __bpf_map__iter(next, obj, -1); 11215 } 11216 11217 struct bpf_map * 11218 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 11219 { 11220 struct bpf_map *pos; 11221 11222 bpf_object__for_each_map(pos, obj) { 11223 /* if it's a special internal map name (which always starts 11224 * with dot) then check if that special name matches the 11225 * real map name (ELF section name) 11226 */ 11227 if (name[0] == '.') { 11228 if (pos->real_name && strcmp(pos->real_name, name) == 0) 11229 return pos; 11230 continue; 11231 } 11232 /* otherwise map name has to be an exact match */ 11233 if (map_uses_real_name(pos)) { 11234 if (strcmp(pos->real_name, name) == 0) 11235 return pos; 11236 continue; 11237 } 11238 if (strcmp(pos->name, name) == 0) 11239 return pos; 11240 } 11241 return errno = ENOENT, NULL; 11242 } 11243 11244 int 11245 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 11246 { 11247 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 11248 } 11249 11250 static int validate_map_op(const struct bpf_map *map, size_t key_sz, 11251 size_t value_sz, bool check_value_sz, __u64 flags) 11252 { 11253 if (!map_is_created(map)) /* map is not yet created */ 11254 return -ENOENT; 11255 11256 if (map->def.key_size != key_sz) { 11257 pr_warn("map '%s': unexpected key size %zu provided, expected %u\n", 11258 map->name, key_sz, map->def.key_size); 11259 return -EINVAL; 11260 } 11261 11262 if (map->fd < 0) { 11263 pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name); 11264 return -EINVAL; 11265 } 11266 11267 if (!check_value_sz) 11268 return 0; 11269 11270 switch (map->def.type) { 11271 case BPF_MAP_TYPE_PERCPU_ARRAY: 11272 case BPF_MAP_TYPE_PERCPU_HASH: 11273 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 11274 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: { 11275 int num_cpu = libbpf_num_possible_cpus(); 11276 size_t elem_sz = roundup(map->def.value_size, 8); 11277 11278 if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) { 11279 if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS)) { 11280 pr_warn("map '%s': BPF_F_CPU and BPF_F_ALL_CPUS are mutually exclusive\n", 11281 map->name); 11282 return -EINVAL; 11283 } 11284 if (map->def.value_size != value_sz) { 11285 pr_warn("map '%s': unexpected value size %zu provided for either BPF_F_CPU or BPF_F_ALL_CPUS, expected %u\n", 11286 map->name, value_sz, map->def.value_size); 11287 return -EINVAL; 11288 } 11289 break; 11290 } 11291 11292 if (value_sz != num_cpu * elem_sz) { 11293 pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zu\n", 11294 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); 11295 return -EINVAL; 11296 } 11297 break; 11298 } 11299 default: 11300 if (map->def.value_size != value_sz) { 11301 pr_warn("map '%s': unexpected value size %zu provided, expected %u\n", 11302 map->name, value_sz, map->def.value_size); 11303 return -EINVAL; 11304 } 11305 break; 11306 } 11307 return 0; 11308 } 11309 11310 int bpf_map__lookup_elem(const struct bpf_map *map, 11311 const void *key, size_t key_sz, 11312 void *value, size_t value_sz, __u64 flags) 11313 { 11314 int err; 11315 11316 err = validate_map_op(map, key_sz, value_sz, true, flags); 11317 if (err) 11318 return libbpf_err(err); 11319 11320 return bpf_map_lookup_elem_flags(map->fd, key, value, flags); 11321 } 11322 11323 int bpf_map__update_elem(const struct bpf_map *map, 11324 const void *key, size_t key_sz, 11325 const void *value, size_t value_sz, __u64 flags) 11326 { 11327 int err; 11328 11329 err = validate_map_op(map, key_sz, value_sz, true, flags); 11330 if (err) 11331 return libbpf_err(err); 11332 11333 return bpf_map_update_elem(map->fd, key, value, flags); 11334 } 11335 11336 int bpf_map__delete_elem(const struct bpf_map *map, 11337 const void *key, size_t key_sz, __u64 flags) 11338 { 11339 int err; 11340 11341 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */, flags); 11342 if (err) 11343 return libbpf_err(err); 11344 11345 return bpf_map_delete_elem_flags(map->fd, key, flags); 11346 } 11347 11348 int bpf_map__lookup_and_delete_elem(const struct bpf_map *map, 11349 const void *key, size_t key_sz, 11350 void *value, size_t value_sz, __u64 flags) 11351 { 11352 int err; 11353 11354 err = validate_map_op(map, key_sz, value_sz, true, flags); 11355 if (err) 11356 return libbpf_err(err); 11357 11358 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags); 11359 } 11360 11361 int bpf_map__get_next_key(const struct bpf_map *map, 11362 const void *cur_key, void *next_key, size_t key_sz) 11363 { 11364 int err; 11365 11366 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */, 0); 11367 if (err) 11368 return libbpf_err(err); 11369 11370 return bpf_map_get_next_key(map->fd, cur_key, next_key); 11371 } 11372 11373 long libbpf_get_error(const void *ptr) 11374 { 11375 if (!IS_ERR_OR_NULL(ptr)) 11376 return 0; 11377 11378 if (IS_ERR(ptr)) 11379 errno = -PTR_ERR(ptr); 11380 11381 /* If ptr == NULL, then errno should be already set by the failing 11382 * API, because libbpf never returns NULL on success and it now always 11383 * sets errno on error. So no extra errno handling for ptr == NULL 11384 * case. 11385 */ 11386 return -errno; 11387 } 11388 11389 /* Replace link's underlying BPF program with the new one */ 11390 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) 11391 { 11392 int ret; 11393 int prog_fd = bpf_program__fd(prog); 11394 11395 if (prog_fd < 0) { 11396 pr_warn("prog '%s': can't use BPF program without FD (was it loaded?)\n", 11397 prog->name); 11398 return libbpf_err(-EINVAL); 11399 } 11400 11401 ret = bpf_link_update(bpf_link__fd(link), prog_fd, NULL); 11402 return libbpf_err_errno(ret); 11403 } 11404 11405 /* Release "ownership" of underlying BPF resource (typically, BPF program 11406 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected 11407 * link, when destructed through bpf_link__destroy() call won't attempt to 11408 * detach/unregisted that BPF resource. This is useful in situations where, 11409 * say, attached BPF program has to outlive userspace program that attached it 11410 * in the system. Depending on type of BPF program, though, there might be 11411 * additional steps (like pinning BPF program in BPF FS) necessary to ensure 11412 * exit of userspace program doesn't trigger automatic detachment and clean up 11413 * inside the kernel. 11414 */ 11415 void bpf_link__disconnect(struct bpf_link *link) 11416 { 11417 link->disconnected = true; 11418 } 11419 11420 int bpf_link__destroy(struct bpf_link *link) 11421 { 11422 int err = 0; 11423 11424 if (IS_ERR_OR_NULL(link)) 11425 return 0; 11426 11427 if (!link->disconnected && link->detach) 11428 err = link->detach(link); 11429 if (link->pin_path) 11430 free(link->pin_path); 11431 if (link->dealloc) 11432 link->dealloc(link); 11433 else 11434 free(link); 11435 11436 return libbpf_err(err); 11437 } 11438 11439 int bpf_link__fd(const struct bpf_link *link) 11440 { 11441 return link->fd; 11442 } 11443 11444 const char *bpf_link__pin_path(const struct bpf_link *link) 11445 { 11446 return link->pin_path; 11447 } 11448 11449 static int bpf_link__detach_fd(struct bpf_link *link) 11450 { 11451 return libbpf_err_errno(close(link->fd)); 11452 } 11453 11454 struct bpf_link *bpf_link__open(const char *path) 11455 { 11456 struct bpf_link *link; 11457 int fd; 11458 11459 fd = bpf_obj_get(path); 11460 if (fd < 0) { 11461 fd = -errno; 11462 pr_warn("failed to open link at %s: %d\n", path, fd); 11463 return libbpf_err_ptr(fd); 11464 } 11465 11466 link = calloc(1, sizeof(*link)); 11467 if (!link) { 11468 close(fd); 11469 return libbpf_err_ptr(-ENOMEM); 11470 } 11471 link->detach = &bpf_link__detach_fd; 11472 link->fd = fd; 11473 11474 link->pin_path = strdup(path); 11475 if (!link->pin_path) { 11476 bpf_link__destroy(link); 11477 return libbpf_err_ptr(-ENOMEM); 11478 } 11479 11480 return link; 11481 } 11482 11483 int bpf_link__detach(struct bpf_link *link) 11484 { 11485 return bpf_link_detach(link->fd) ? -errno : 0; 11486 } 11487 11488 int bpf_link__pin(struct bpf_link *link, const char *path) 11489 { 11490 int err; 11491 11492 if (link->pin_path) 11493 return libbpf_err(-EBUSY); 11494 err = make_parent_dir(path); 11495 if (err) 11496 return libbpf_err(err); 11497 err = check_path(path); 11498 if (err) 11499 return libbpf_err(err); 11500 11501 link->pin_path = strdup(path); 11502 if (!link->pin_path) 11503 return libbpf_err(-ENOMEM); 11504 11505 if (bpf_obj_pin(link->fd, link->pin_path)) { 11506 err = -errno; 11507 zfree(&link->pin_path); 11508 return libbpf_err(err); 11509 } 11510 11511 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); 11512 return 0; 11513 } 11514 11515 int bpf_link__unpin(struct bpf_link *link) 11516 { 11517 int err; 11518 11519 if (!link->pin_path) 11520 return libbpf_err(-EINVAL); 11521 11522 err = unlink(link->pin_path); 11523 if (err != 0) 11524 return -errno; 11525 11526 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); 11527 zfree(&link->pin_path); 11528 return 0; 11529 } 11530 11531 struct bpf_link_perf { 11532 struct bpf_link link; 11533 int perf_event_fd; 11534 /* legacy kprobe support: keep track of probe identifier and type */ 11535 char *legacy_probe_name; 11536 bool legacy_is_kprobe; 11537 bool legacy_is_retprobe; 11538 }; 11539 11540 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe); 11541 static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe); 11542 11543 static int bpf_link_perf_detach(struct bpf_link *link) 11544 { 11545 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 11546 int err = 0; 11547 11548 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0) 11549 err = -errno; 11550 11551 if (perf_link->perf_event_fd != link->fd) 11552 close(perf_link->perf_event_fd); 11553 close(link->fd); 11554 11555 /* legacy uprobe/kprobe needs to be removed after perf event fd closure */ 11556 if (perf_link->legacy_probe_name) { 11557 if (perf_link->legacy_is_kprobe) { 11558 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name, 11559 perf_link->legacy_is_retprobe); 11560 } else { 11561 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name, 11562 perf_link->legacy_is_retprobe); 11563 } 11564 } 11565 11566 return err; 11567 } 11568 11569 static void bpf_link_perf_dealloc(struct bpf_link *link) 11570 { 11571 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 11572 11573 free(perf_link->legacy_probe_name); 11574 free(perf_link); 11575 } 11576 11577 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd, 11578 const struct bpf_perf_event_opts *opts) 11579 { 11580 struct bpf_link_perf *link; 11581 int prog_fd, link_fd = -1, err; 11582 bool force_ioctl_attach; 11583 11584 if (!OPTS_VALID(opts, bpf_perf_event_opts)) 11585 return libbpf_err_ptr(-EINVAL); 11586 11587 if (pfd < 0) { 11588 pr_warn("prog '%s': invalid perf event FD %d\n", 11589 prog->name, pfd); 11590 return libbpf_err_ptr(-EINVAL); 11591 } 11592 prog_fd = bpf_program__fd(prog); 11593 if (prog_fd < 0) { 11594 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", 11595 prog->name); 11596 return libbpf_err_ptr(-EINVAL); 11597 } 11598 11599 link = calloc(1, sizeof(*link)); 11600 if (!link) 11601 return libbpf_err_ptr(-ENOMEM); 11602 link->link.detach = &bpf_link_perf_detach; 11603 link->link.dealloc = &bpf_link_perf_dealloc; 11604 link->perf_event_fd = pfd; 11605 11606 force_ioctl_attach = OPTS_GET(opts, force_ioctl_attach, false); 11607 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) { 11608 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts, 11609 .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0)); 11610 11611 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts); 11612 if (link_fd < 0) { 11613 err = -errno; 11614 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %s\n", 11615 prog->name, pfd, errstr(err)); 11616 goto err_out; 11617 } 11618 link->link.fd = link_fd; 11619 } else { 11620 if (OPTS_GET(opts, bpf_cookie, 0)) { 11621 pr_warn("prog '%s': user context value is not supported\n", prog->name); 11622 err = -EOPNOTSUPP; 11623 goto err_out; 11624 } 11625 11626 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 11627 err = -errno; 11628 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n", 11629 prog->name, pfd, errstr(err)); 11630 if (err == -EPROTO) 11631 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n", 11632 prog->name, pfd); 11633 goto err_out; 11634 } 11635 link->link.fd = pfd; 11636 } 11637 11638 if (!OPTS_GET(opts, dont_enable, false)) { 11639 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 11640 err = -errno; 11641 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n", 11642 prog->name, pfd, errstr(err)); 11643 goto err_out; 11644 } 11645 } 11646 11647 return &link->link; 11648 err_out: 11649 if (link_fd >= 0) 11650 close(link_fd); 11651 free(link); 11652 return libbpf_err_ptr(err); 11653 } 11654 11655 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd) 11656 { 11657 return bpf_program__attach_perf_event_opts(prog, pfd, NULL); 11658 } 11659 11660 /* 11661 * this function is expected to parse integer in the range of [0, 2^31-1] from 11662 * given file using scanf format string fmt. If actual parsed value is 11663 * negative, the result might be indistinguishable from error 11664 */ 11665 static int parse_uint_from_file(const char *file, const char *fmt) 11666 { 11667 int err, ret; 11668 FILE *f; 11669 11670 f = fopen(file, "re"); 11671 if (!f) { 11672 err = -errno; 11673 pr_debug("failed to open '%s': %s\n", file, errstr(err)); 11674 return err; 11675 } 11676 err = fscanf(f, fmt, &ret); 11677 if (err != 1) { 11678 err = err == EOF ? -EIO : -errno; 11679 pr_debug("failed to parse '%s': %s\n", file, errstr(err)); 11680 fclose(f); 11681 return err; 11682 } 11683 fclose(f); 11684 return ret; 11685 } 11686 11687 static int determine_kprobe_perf_type(void) 11688 { 11689 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 11690 11691 return parse_uint_from_file(file, "%d\n"); 11692 } 11693 11694 static int determine_uprobe_perf_type(void) 11695 { 11696 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 11697 11698 return parse_uint_from_file(file, "%d\n"); 11699 } 11700 11701 static int determine_kprobe_retprobe_bit(void) 11702 { 11703 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 11704 11705 return parse_uint_from_file(file, "config:%d\n"); 11706 } 11707 11708 static int determine_uprobe_retprobe_bit(void) 11709 { 11710 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 11711 11712 return parse_uint_from_file(file, "config:%d\n"); 11713 } 11714 11715 #define PERF_UPROBE_REF_CTR_OFFSET_BITS 32 11716 #define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32 11717 11718 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 11719 uint64_t offset, int pid, size_t ref_ctr_off) 11720 { 11721 const size_t attr_sz = sizeof(struct perf_event_attr); 11722 struct perf_event_attr attr; 11723 int type, pfd; 11724 11725 if ((__u64)ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS)) 11726 return -EINVAL; 11727 11728 memset(&attr, 0, attr_sz); 11729 11730 type = uprobe ? determine_uprobe_perf_type() 11731 : determine_kprobe_perf_type(); 11732 if (type < 0) { 11733 pr_warn("failed to determine %s perf type: %s\n", 11734 uprobe ? "uprobe" : "kprobe", 11735 errstr(type)); 11736 return type; 11737 } 11738 if (retprobe) { 11739 int bit = uprobe ? determine_uprobe_retprobe_bit() 11740 : determine_kprobe_retprobe_bit(); 11741 11742 if (bit < 0) { 11743 pr_warn("failed to determine %s retprobe bit: %s\n", 11744 uprobe ? "uprobe" : "kprobe", 11745 errstr(bit)); 11746 return bit; 11747 } 11748 attr.config |= 1 << bit; 11749 } 11750 attr.size = attr_sz; 11751 attr.type = type; 11752 attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT; 11753 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 11754 attr.config2 = offset; /* kprobe_addr or probe_offset */ 11755 11756 /* pid filter is meaningful only for uprobes */ 11757 pfd = syscall(__NR_perf_event_open, &attr, 11758 pid < 0 ? -1 : pid /* pid */, 11759 pid == -1 ? 0 : -1 /* cpu */, 11760 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 11761 return pfd >= 0 ? pfd : -errno; 11762 } 11763 11764 static int append_to_file(const char *file, const char *fmt, ...) 11765 { 11766 int fd, n, err = 0; 11767 va_list ap; 11768 char buf[1024]; 11769 11770 va_start(ap, fmt); 11771 n = vsnprintf(buf, sizeof(buf), fmt, ap); 11772 va_end(ap); 11773 11774 if (n < 0 || n >= sizeof(buf)) 11775 return -EINVAL; 11776 11777 fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0); 11778 if (fd < 0) 11779 return -errno; 11780 11781 if (write(fd, buf, n) < 0) 11782 err = -errno; 11783 11784 close(fd); 11785 return err; 11786 } 11787 11788 #define DEBUGFS "/sys/kernel/debug/tracing" 11789 #define TRACEFS "/sys/kernel/tracing" 11790 11791 static bool use_debugfs(void) 11792 { 11793 static int has_debugfs = -1; 11794 11795 if (has_debugfs < 0) 11796 has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0; 11797 11798 return has_debugfs == 1; 11799 } 11800 11801 static const char *tracefs_path(void) 11802 { 11803 return use_debugfs() ? DEBUGFS : TRACEFS; 11804 } 11805 11806 static const char *tracefs_kprobe_events(void) 11807 { 11808 return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events"; 11809 } 11810 11811 static const char *tracefs_uprobe_events(void) 11812 { 11813 return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events"; 11814 } 11815 11816 static const char *tracefs_available_filter_functions(void) 11817 { 11818 return use_debugfs() ? DEBUGFS"/available_filter_functions" 11819 : TRACEFS"/available_filter_functions"; 11820 } 11821 11822 static const char *tracefs_available_filter_functions_addrs(void) 11823 { 11824 return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" 11825 : TRACEFS"/available_filter_functions_addrs"; 11826 } 11827 11828 static void gen_probe_legacy_event_name(char *buf, size_t buf_sz, 11829 const char *name, size_t offset) 11830 { 11831 static int index = 0; 11832 int i; 11833 11834 snprintf(buf, buf_sz, "libbpf_%d_%d_%s_0x%zx", getpid(), 11835 __sync_fetch_and_add(&index, 1), name, offset); 11836 11837 /* sanitize name in the probe name */ 11838 for (i = 0; buf[i]; i++) { 11839 if (!isalnum(buf[i])) 11840 buf[i] = '_'; 11841 } 11842 } 11843 11844 static int add_kprobe_event_legacy(const char *probe_name, bool retprobe, 11845 const char *kfunc_name, size_t offset) 11846 { 11847 return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx", 11848 retprobe ? 'r' : 'p', 11849 retprobe ? "kretprobes" : "kprobes", 11850 probe_name, kfunc_name, offset); 11851 } 11852 11853 static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe) 11854 { 11855 return append_to_file(tracefs_kprobe_events(), "-:%s/%s", 11856 retprobe ? "kretprobes" : "kprobes", probe_name); 11857 } 11858 11859 static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe) 11860 { 11861 char file[256]; 11862 11863 snprintf(file, sizeof(file), "%s/events/%s/%s/id", 11864 tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name); 11865 11866 return parse_uint_from_file(file, "%d\n"); 11867 } 11868 11869 static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, 11870 const char *kfunc_name, size_t offset, int pid) 11871 { 11872 const size_t attr_sz = sizeof(struct perf_event_attr); 11873 struct perf_event_attr attr; 11874 int type, pfd, err; 11875 11876 err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset); 11877 if (err < 0) { 11878 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n", 11879 kfunc_name, offset, 11880 errstr(err)); 11881 return err; 11882 } 11883 type = determine_kprobe_perf_type_legacy(probe_name, retprobe); 11884 if (type < 0) { 11885 err = type; 11886 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n", 11887 kfunc_name, offset, 11888 errstr(err)); 11889 goto err_clean_legacy; 11890 } 11891 11892 memset(&attr, 0, attr_sz); 11893 attr.size = attr_sz; 11894 attr.config = type; 11895 attr.type = PERF_TYPE_TRACEPOINT; 11896 11897 pfd = syscall(__NR_perf_event_open, &attr, 11898 pid < 0 ? -1 : pid, /* pid */ 11899 pid == -1 ? 0 : -1, /* cpu */ 11900 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 11901 if (pfd < 0) { 11902 err = -errno; 11903 pr_warn("legacy kprobe perf_event_open() failed: %s\n", 11904 errstr(err)); 11905 goto err_clean_legacy; 11906 } 11907 return pfd; 11908 11909 err_clean_legacy: 11910 /* Clear the newly added legacy kprobe_event */ 11911 remove_kprobe_event_legacy(probe_name, retprobe); 11912 return err; 11913 } 11914 11915 static const char *arch_specific_syscall_pfx(void) 11916 { 11917 #if defined(__x86_64__) 11918 return "x64"; 11919 #elif defined(__i386__) 11920 return "ia32"; 11921 #elif defined(__s390x__) 11922 return "s390x"; 11923 #elif defined(__arm__) 11924 return "arm"; 11925 #elif defined(__aarch64__) 11926 return "arm64"; 11927 #elif defined(__mips__) 11928 return "mips"; 11929 #elif defined(__riscv) 11930 return "riscv"; 11931 #elif defined(__powerpc__) 11932 return "powerpc"; 11933 #elif defined(__powerpc64__) 11934 return "powerpc64"; 11935 #else 11936 return NULL; 11937 #endif 11938 } 11939 11940 int probe_kern_syscall_wrapper(int token_fd) 11941 { 11942 char syscall_name[64]; 11943 const char *ksys_pfx; 11944 11945 ksys_pfx = arch_specific_syscall_pfx(); 11946 if (!ksys_pfx) 11947 return 0; 11948 11949 snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx); 11950 11951 if (determine_kprobe_perf_type() >= 0) { 11952 int pfd; 11953 11954 pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0); 11955 if (pfd >= 0) 11956 close(pfd); 11957 11958 return pfd >= 0 ? 1 : 0; 11959 } else { /* legacy mode */ 11960 char probe_name[MAX_EVENT_NAME_LEN]; 11961 11962 gen_probe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0); 11963 if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0) 11964 return 0; 11965 11966 (void)remove_kprobe_event_legacy(probe_name, false); 11967 return 1; 11968 } 11969 } 11970 11971 struct bpf_link * 11972 bpf_program__attach_kprobe_opts(const struct bpf_program *prog, 11973 const char *func_name, 11974 const struct bpf_kprobe_opts *opts) 11975 { 11976 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 11977 enum probe_attach_mode attach_mode; 11978 char *legacy_probe = NULL; 11979 struct bpf_link *link; 11980 size_t offset; 11981 bool retprobe, legacy; 11982 int pfd, err; 11983 11984 if (!OPTS_VALID(opts, bpf_kprobe_opts)) 11985 return libbpf_err_ptr(-EINVAL); 11986 11987 attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT); 11988 retprobe = OPTS_GET(opts, retprobe, false); 11989 offset = OPTS_GET(opts, offset, 0); 11990 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 11991 11992 legacy = determine_kprobe_perf_type() < 0; 11993 switch (attach_mode) { 11994 case PROBE_ATTACH_MODE_LEGACY: 11995 legacy = true; 11996 pe_opts.force_ioctl_attach = true; 11997 break; 11998 case PROBE_ATTACH_MODE_PERF: 11999 if (legacy) 12000 return libbpf_err_ptr(-ENOTSUP); 12001 pe_opts.force_ioctl_attach = true; 12002 break; 12003 case PROBE_ATTACH_MODE_LINK: 12004 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) 12005 return libbpf_err_ptr(-ENOTSUP); 12006 break; 12007 case PROBE_ATTACH_MODE_DEFAULT: 12008 break; 12009 default: 12010 return libbpf_err_ptr(-EINVAL); 12011 } 12012 if (!func_name && legacy) 12013 return libbpf_err_ptr(-EOPNOTSUPP); 12014 12015 if (!legacy) { 12016 pfd = perf_event_open_probe(false /* uprobe */, retprobe, 12017 func_name, offset, 12018 -1 /* pid */, 0 /* ref_ctr_off */); 12019 } else { 12020 char probe_name[MAX_EVENT_NAME_LEN]; 12021 12022 gen_probe_legacy_event_name(probe_name, sizeof(probe_name), 12023 func_name, offset); 12024 12025 legacy_probe = strdup(probe_name); 12026 if (!legacy_probe) 12027 return libbpf_err_ptr(-ENOMEM); 12028 12029 pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name, 12030 offset, -1 /* pid */); 12031 } 12032 if (pfd < 0) { 12033 err = pfd; 12034 pr_warn("prog '%s': failed to create %s '%s%s0x%zx' perf event: %s\n", 12035 prog->name, retprobe ? "kretprobe" : "kprobe", 12036 func_name ?: "", func_name ? "+" : "", 12037 offset, errstr(err)); 12038 goto err_out; 12039 } 12040 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 12041 err = libbpf_get_error(link); 12042 if (err) { 12043 close(pfd); 12044 pr_warn("prog '%s': failed to attach to %s '%s%s0x%zx': %s\n", 12045 prog->name, retprobe ? "kretprobe" : "kprobe", 12046 func_name ?: "", func_name ? "+" : "", 12047 offset, errstr(err)); 12048 goto err_clean_legacy; 12049 } 12050 if (legacy) { 12051 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 12052 12053 perf_link->legacy_probe_name = legacy_probe; 12054 perf_link->legacy_is_kprobe = true; 12055 perf_link->legacy_is_retprobe = retprobe; 12056 } 12057 12058 return link; 12059 12060 err_clean_legacy: 12061 if (legacy) 12062 remove_kprobe_event_legacy(legacy_probe, retprobe); 12063 err_out: 12064 free(legacy_probe); 12065 return libbpf_err_ptr(err); 12066 } 12067 12068 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog, 12069 bool retprobe, 12070 const char *func_name) 12071 { 12072 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts, 12073 .retprobe = retprobe, 12074 ); 12075 12076 return bpf_program__attach_kprobe_opts(prog, func_name, &opts); 12077 } 12078 12079 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog, 12080 const char *syscall_name, 12081 const struct bpf_ksyscall_opts *opts) 12082 { 12083 LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); 12084 char func_name[128]; 12085 12086 if (!OPTS_VALID(opts, bpf_ksyscall_opts)) 12087 return libbpf_err_ptr(-EINVAL); 12088 12089 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { 12090 /* arch_specific_syscall_pfx() should never return NULL here 12091 * because it is guarded by kernel_supports(). However, since 12092 * compiler does not know that we have an explicit conditional 12093 * as well. 12094 */ 12095 snprintf(func_name, sizeof(func_name), "__%s_sys_%s", 12096 arch_specific_syscall_pfx() ? : "", syscall_name); 12097 } else { 12098 snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name); 12099 } 12100 12101 kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false); 12102 kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 12103 12104 return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts); 12105 } 12106 12107 /* Adapted from perf/util/string.c */ 12108 bool glob_match(const char *str, const char *pat) 12109 { 12110 while (*str && *pat && *pat != '*') { 12111 if (*pat == '?') { /* Matches any single character */ 12112 str++; 12113 pat++; 12114 continue; 12115 } 12116 if (*str != *pat) 12117 return false; 12118 str++; 12119 pat++; 12120 } 12121 /* Check wild card */ 12122 if (*pat == '*') { 12123 while (*pat == '*') 12124 pat++; 12125 if (!*pat) /* Tail wild card matches all */ 12126 return true; 12127 while (*str) 12128 if (glob_match(str++, pat)) 12129 return true; 12130 } 12131 return !*str && !*pat; 12132 } 12133 12134 struct kprobe_multi_resolve { 12135 const char *pattern; 12136 unsigned long *addrs; 12137 size_t cap; 12138 size_t cnt; 12139 }; 12140 12141 struct avail_kallsyms_data { 12142 char **syms; 12143 size_t cnt; 12144 struct kprobe_multi_resolve *res; 12145 }; 12146 12147 static int avail_func_cmp(const void *a, const void *b) 12148 { 12149 return strcmp(*(const char **)a, *(const char **)b); 12150 } 12151 12152 static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type, 12153 const char *sym_name, void *ctx) 12154 { 12155 struct avail_kallsyms_data *data = ctx; 12156 struct kprobe_multi_resolve *res = data->res; 12157 int err; 12158 12159 if (!glob_match(sym_name, res->pattern)) 12160 return 0; 12161 12162 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) { 12163 /* Some versions of kernel strip out .llvm.<hash> suffix from 12164 * function names reported in available_filter_functions, but 12165 * don't do so for kallsyms. While this is clearly a kernel 12166 * bug (fixed by [0]) we try to accommodate that in libbpf to 12167 * make multi-kprobe usability a bit better: if no match is 12168 * found, we will strip .llvm. suffix and try one more time. 12169 * 12170 * [0] fb6a421fb615 ("kallsyms: Match symbols exactly with CONFIG_LTO_CLANG") 12171 */ 12172 char sym_trim[256], *psym_trim = sym_trim; 12173 const char *sym_sfx; 12174 12175 if (!(sym_sfx = strstr(sym_name, ".llvm."))) 12176 return 0; 12177 12178 /* psym_trim vs sym_trim dance is done to avoid pointer vs array 12179 * coercion differences and get proper `const char **` pointer 12180 * which avail_func_cmp() expects 12181 */ 12182 snprintf(sym_trim, sizeof(sym_trim), "%.*s", (int)(sym_sfx - sym_name), sym_name); 12183 if (!bsearch(&psym_trim, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) 12184 return 0; 12185 } 12186 12187 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1); 12188 if (err) 12189 return err; 12190 12191 res->addrs[res->cnt++] = (unsigned long)sym_addr; 12192 return 0; 12193 } 12194 12195 static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res) 12196 { 12197 const char *available_functions_file = tracefs_available_filter_functions(); 12198 struct avail_kallsyms_data data; 12199 char sym_name[500]; 12200 FILE *f; 12201 int err = 0, ret, i; 12202 char **syms = NULL; 12203 size_t cap = 0, cnt = 0; 12204 12205 f = fopen(available_functions_file, "re"); 12206 if (!f) { 12207 err = -errno; 12208 pr_warn("failed to open %s: %s\n", available_functions_file, errstr(err)); 12209 return err; 12210 } 12211 12212 while (true) { 12213 char *name; 12214 12215 ret = fscanf(f, "%499s%*[^\n]\n", sym_name); 12216 if (ret == EOF && feof(f)) 12217 break; 12218 12219 if (ret != 1) { 12220 pr_warn("failed to parse available_filter_functions entry: %d\n", ret); 12221 err = -EINVAL; 12222 goto cleanup; 12223 } 12224 12225 if (!glob_match(sym_name, res->pattern)) 12226 continue; 12227 12228 err = libbpf_ensure_mem((void **)&syms, &cap, sizeof(*syms), cnt + 1); 12229 if (err) 12230 goto cleanup; 12231 12232 name = strdup(sym_name); 12233 if (!name) { 12234 err = -errno; 12235 goto cleanup; 12236 } 12237 12238 syms[cnt++] = name; 12239 } 12240 12241 /* no entries found, bail out */ 12242 if (cnt == 0) { 12243 err = -ENOENT; 12244 goto cleanup; 12245 } 12246 12247 /* sort available functions */ 12248 qsort(syms, cnt, sizeof(*syms), avail_func_cmp); 12249 12250 data.syms = syms; 12251 data.res = res; 12252 data.cnt = cnt; 12253 libbpf_kallsyms_parse(avail_kallsyms_cb, &data); 12254 12255 if (res->cnt == 0) 12256 err = -ENOENT; 12257 12258 cleanup: 12259 for (i = 0; i < cnt; i++) 12260 free((char *)syms[i]); 12261 free(syms); 12262 12263 fclose(f); 12264 return err; 12265 } 12266 12267 static bool has_available_filter_functions_addrs(void) 12268 { 12269 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1; 12270 } 12271 12272 static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res) 12273 { 12274 const char *available_path = tracefs_available_filter_functions_addrs(); 12275 char sym_name[500]; 12276 FILE *f; 12277 int ret, err = 0; 12278 unsigned long long sym_addr; 12279 12280 f = fopen(available_path, "re"); 12281 if (!f) { 12282 err = -errno; 12283 pr_warn("failed to open %s: %s\n", available_path, errstr(err)); 12284 return err; 12285 } 12286 12287 while (true) { 12288 ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name); 12289 if (ret == EOF && feof(f)) 12290 break; 12291 12292 if (ret != 2) { 12293 pr_warn("failed to parse available_filter_functions_addrs entry: %d\n", 12294 ret); 12295 err = -EINVAL; 12296 goto cleanup; 12297 } 12298 12299 if (!glob_match(sym_name, res->pattern)) 12300 continue; 12301 12302 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, 12303 sizeof(*res->addrs), res->cnt + 1); 12304 if (err) 12305 goto cleanup; 12306 12307 res->addrs[res->cnt++] = (unsigned long)sym_addr; 12308 } 12309 12310 if (res->cnt == 0) 12311 err = -ENOENT; 12312 12313 cleanup: 12314 fclose(f); 12315 return err; 12316 } 12317 12318 struct bpf_link * 12319 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, 12320 const char *pattern, 12321 const struct bpf_kprobe_multi_opts *opts) 12322 { 12323 LIBBPF_OPTS(bpf_link_create_opts, lopts); 12324 struct kprobe_multi_resolve res = { 12325 .pattern = pattern, 12326 }; 12327 enum bpf_attach_type attach_type; 12328 struct bpf_link *link = NULL; 12329 const unsigned long *addrs; 12330 int err, link_fd, prog_fd; 12331 bool retprobe, session, unique_match; 12332 const __u64 *cookies; 12333 const char **syms; 12334 size_t cnt; 12335 12336 if (!OPTS_VALID(opts, bpf_kprobe_multi_opts)) 12337 return libbpf_err_ptr(-EINVAL); 12338 12339 prog_fd = bpf_program__fd(prog); 12340 if (prog_fd < 0) { 12341 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", 12342 prog->name); 12343 return libbpf_err_ptr(-EINVAL); 12344 } 12345 12346 syms = OPTS_GET(opts, syms, false); 12347 addrs = OPTS_GET(opts, addrs, false); 12348 cnt = OPTS_GET(opts, cnt, false); 12349 cookies = OPTS_GET(opts, cookies, false); 12350 unique_match = OPTS_GET(opts, unique_match, false); 12351 12352 if (!pattern && !addrs && !syms) 12353 return libbpf_err_ptr(-EINVAL); 12354 if (pattern && (addrs || syms || cookies || cnt)) 12355 return libbpf_err_ptr(-EINVAL); 12356 if (!pattern && !cnt) 12357 return libbpf_err_ptr(-EINVAL); 12358 if (!pattern && unique_match) 12359 return libbpf_err_ptr(-EINVAL); 12360 if (addrs && syms) 12361 return libbpf_err_ptr(-EINVAL); 12362 12363 /* 12364 * Exact function name (no wildcards) without unique_match: 12365 * bypass kallsyms parsing and pass the symbol directly to the 12366 * kernel via syms[] array. When unique_match is set, fall 12367 * through to the slow path which detects duplicate symbols. 12368 */ 12369 if (pattern && !strpbrk(pattern, "*?") && !unique_match) { 12370 syms = &pattern; 12371 cnt = 1; 12372 } else if (pattern) { 12373 if (has_available_filter_functions_addrs()) 12374 err = libbpf_available_kprobes_parse(&res); 12375 else 12376 err = libbpf_available_kallsyms_parse(&res); 12377 if (err) 12378 goto error; 12379 12380 if (unique_match && res.cnt != 1) { 12381 pr_warn("prog '%s': failed to find a unique match for '%s' (%zu matches)\n", 12382 prog->name, pattern, res.cnt); 12383 err = -EINVAL; 12384 goto error; 12385 } 12386 12387 addrs = res.addrs; 12388 cnt = res.cnt; 12389 } 12390 12391 retprobe = OPTS_GET(opts, retprobe, false); 12392 session = OPTS_GET(opts, session, false); 12393 12394 if (retprobe && session) 12395 return libbpf_err_ptr(-EINVAL); 12396 12397 attach_type = session ? BPF_TRACE_KPROBE_SESSION : BPF_TRACE_KPROBE_MULTI; 12398 12399 lopts.kprobe_multi.syms = syms; 12400 lopts.kprobe_multi.addrs = addrs; 12401 lopts.kprobe_multi.cookies = cookies; 12402 lopts.kprobe_multi.cnt = cnt; 12403 lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0; 12404 12405 link = calloc(1, sizeof(*link)); 12406 if (!link) { 12407 err = -ENOMEM; 12408 goto error; 12409 } 12410 link->detach = &bpf_link__detach_fd; 12411 12412 link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts); 12413 if (link_fd < 0) { 12414 err = -errno; 12415 /* 12416 * Normalize error code: when exact name bypasses kallsyms 12417 * parsing, kernel returns ESRCH from ftrace_lookup_symbols(). 12418 * Convert to ENOENT for API consistency with the pattern 12419 * matching path which returns ENOENT from userspace. 12420 */ 12421 if (err == -ESRCH) 12422 err = -ENOENT; 12423 pr_warn("prog '%s': failed to attach: %s\n", 12424 prog->name, errstr(err)); 12425 goto error; 12426 } 12427 link->fd = link_fd; 12428 free(res.addrs); 12429 return link; 12430 12431 error: 12432 free(link); 12433 free(res.addrs); 12434 return libbpf_err_ptr(err); 12435 } 12436 12437 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12438 { 12439 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts); 12440 long offset = 0; 12441 const char *func_name; 12442 char *func; 12443 int n; 12444 12445 *link = NULL; 12446 12447 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */ 12448 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0) 12449 return 0; 12450 12451 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/"); 12452 if (opts.retprobe) 12453 func_name = prog->sec_name + sizeof("kretprobe/") - 1; 12454 else 12455 func_name = prog->sec_name + sizeof("kprobe/") - 1; 12456 12457 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset); 12458 if (n < 1) { 12459 pr_warn("kprobe name is invalid: %s\n", func_name); 12460 return -EINVAL; 12461 } 12462 12463 if (offset < 0) { 12464 free(func); 12465 pr_warn("kprobe offset must be a non-negative integer: %li\n", offset); 12466 return -EINVAL; 12467 } 12468 12469 if (opts.retprobe && offset != 0) { 12470 free(func); 12471 pr_warn("kretprobes do not support offset specification\n"); 12472 return -EINVAL; 12473 } 12474 12475 opts.offset = offset; 12476 *link = bpf_program__attach_kprobe_opts(prog, func, &opts); 12477 free(func); 12478 return libbpf_get_error(*link); 12479 } 12480 12481 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12482 { 12483 LIBBPF_OPTS(bpf_ksyscall_opts, opts); 12484 const char *syscall_name; 12485 12486 *link = NULL; 12487 12488 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */ 12489 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0) 12490 return 0; 12491 12492 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/"); 12493 if (opts.retprobe) 12494 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1; 12495 else 12496 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1; 12497 12498 *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts); 12499 return *link ? 0 : -errno; 12500 } 12501 12502 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12503 { 12504 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); 12505 const char *spec; 12506 char *pattern; 12507 int n; 12508 12509 *link = NULL; 12510 12511 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */ 12512 if (strcmp(prog->sec_name, "kprobe.multi") == 0 || 12513 strcmp(prog->sec_name, "kretprobe.multi") == 0) 12514 return 0; 12515 12516 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/"); 12517 if (opts.retprobe) 12518 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1; 12519 else 12520 spec = prog->sec_name + sizeof("kprobe.multi/") - 1; 12521 12522 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern); 12523 if (n < 1) { 12524 pr_warn("kprobe multi pattern is invalid: %s\n", spec); 12525 return -EINVAL; 12526 } 12527 12528 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts); 12529 free(pattern); 12530 return libbpf_get_error(*link); 12531 } 12532 12533 static int attach_kprobe_session(const struct bpf_program *prog, long cookie, 12534 struct bpf_link **link) 12535 { 12536 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts, .session = true); 12537 const char *spec; 12538 char *pattern; 12539 int n; 12540 12541 *link = NULL; 12542 12543 /* no auto-attach for SEC("kprobe.session") */ 12544 if (strcmp(prog->sec_name, "kprobe.session") == 0) 12545 return 0; 12546 12547 spec = prog->sec_name + sizeof("kprobe.session/") - 1; 12548 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern); 12549 if (n < 1) { 12550 pr_warn("kprobe session pattern is invalid: %s\n", spec); 12551 return -EINVAL; 12552 } 12553 12554 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts); 12555 free(pattern); 12556 return *link ? 0 : -errno; 12557 } 12558 12559 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12560 { 12561 char *probe_type = NULL, *binary_path = NULL, *func_name = NULL; 12562 LIBBPF_OPTS(bpf_uprobe_multi_opts, opts); 12563 int n, ret = -EINVAL; 12564 12565 *link = NULL; 12566 12567 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", 12568 &probe_type, &binary_path, &func_name); 12569 switch (n) { 12570 case 1: 12571 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ 12572 ret = 0; 12573 break; 12574 case 3: 12575 opts.session = str_has_pfx(probe_type, "uprobe.session"); 12576 opts.retprobe = str_has_pfx(probe_type, "uretprobe.multi"); 12577 12578 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts); 12579 ret = libbpf_get_error(*link); 12580 break; 12581 default: 12582 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, 12583 prog->sec_name); 12584 break; 12585 } 12586 free(probe_type); 12587 free(binary_path); 12588 free(func_name); 12589 return ret; 12590 } 12591 12592 #define MAX_BPF_FUNC_ARGS 12 12593 12594 static bool btf_type_is_modifier(const struct btf_type *t) 12595 { 12596 switch (BTF_INFO_KIND(t->info)) { 12597 case BTF_KIND_TYPEDEF: 12598 case BTF_KIND_VOLATILE: 12599 case BTF_KIND_CONST: 12600 case BTF_KIND_RESTRICT: 12601 case BTF_KIND_TYPE_TAG: 12602 return true; 12603 default: 12604 return false; 12605 } 12606 } 12607 12608 #define MAX_RESOLVE_DEPTH 32 12609 12610 static int btf_get_type_size(const struct btf *btf, __u32 type_id, 12611 const struct btf_type **ret_type) 12612 { 12613 const struct btf_type *t; 12614 int i; 12615 12616 *ret_type = btf__type_by_id(btf, 0); 12617 if (!type_id) 12618 return 0; 12619 t = btf__type_by_id(btf, type_id); 12620 for (i = 0; i < MAX_RESOLVE_DEPTH && t && btf_type_is_modifier(t); i++) 12621 t = btf__type_by_id(btf, t->type); 12622 if (!t || i == MAX_RESOLVE_DEPTH) 12623 return -EINVAL; 12624 *ret_type = t; 12625 if (btf_is_ptr(t)) 12626 return btf__pointer_size(btf); 12627 if (btf_is_int(t) || btf_is_any_enum(t) || btf_is_struct(t) || btf_is_union(t)) 12628 return t->size; 12629 return -EINVAL; 12630 } 12631 12632 bool btf_type_is_traceable_func(const struct btf *btf, const struct btf_type *t) 12633 { 12634 const struct btf_param *args; 12635 const struct btf_type *proto; 12636 __u32 i, nargs; 12637 int ret; 12638 12639 if (!btf_is_func(t)) 12640 return false; 12641 proto = btf__type_by_id(btf, t->type); 12642 if (!proto || !btf_is_func_proto(proto)) 12643 return false; 12644 12645 args = (const struct btf_param *)(proto + 1); 12646 nargs = btf_vlen(proto); 12647 if (nargs > MAX_BPF_FUNC_ARGS) 12648 return false; 12649 12650 /* No support for struct return type. */ 12651 ret = btf_get_type_size(btf, proto->type, &t); 12652 if (ret < 0 || btf_is_struct(t) || btf_is_union(t)) 12653 return false; 12654 12655 for (i = 0; i < nargs; i++) { 12656 /* No support for variable args. */ 12657 if (i == nargs - 1 && args[i].type == 0) 12658 return false; 12659 ret = btf_get_type_size(btf, args[i].type, &t); 12660 /* No support of struct argument size greater than 16 bytes. */ 12661 if (ret < 0 || ret > 16) 12662 return false; 12663 /* No support for void argument. */ 12664 if (ret == 0) 12665 return false; 12666 } 12667 12668 return true; 12669 } 12670 12671 static int 12672 collect_btf_func_ids_by_glob(const struct btf *btf, const char *pattern, __u32 **ids) 12673 { 12674 __u32 type_id, nr_types = btf__type_cnt(btf); 12675 size_t cap = 0, cnt = 0; 12676 12677 if (!pattern) 12678 return -EINVAL; 12679 12680 for (type_id = 1; type_id < nr_types; type_id++) { 12681 const struct btf_type *t = btf__type_by_id(btf, type_id); 12682 const char *name; 12683 int err; 12684 12685 if (btf_kind(t) != BTF_KIND_FUNC) 12686 continue; 12687 name = btf__name_by_offset(btf, t->name_off); 12688 if (!name) 12689 continue; 12690 12691 if (!glob_match(name, pattern)) 12692 continue; 12693 if (!btf_type_is_traceable_func(btf, t)) 12694 continue; 12695 12696 err = libbpf_ensure_mem((void **) ids, &cap, sizeof(**ids), cnt + 1); 12697 if (err) { 12698 free(*ids); 12699 return -ENOMEM; 12700 } 12701 (*ids)[cnt++] = type_id; 12702 } 12703 12704 return cnt; 12705 } 12706 12707 static int collect_func_ids_by_glob(const struct bpf_program *prog, const char *pattern, __u32 **ids) 12708 { 12709 struct bpf_object *obj = prog->obj; 12710 const struct module_btf *mod; 12711 struct btf *btf = NULL; 12712 const char *sep; 12713 int err; 12714 12715 err = bpf_object__load_vmlinux_btf(obj, true); 12716 if (err) 12717 return err; 12718 12719 /* In case we have module specified, we will find its btf and use that. */ 12720 sep = strchr(pattern, ':'); 12721 if (sep) { 12722 mod = find_attach_module(obj, pattern); 12723 if (!mod) { 12724 err = -EINVAL; 12725 goto cleanup; 12726 } 12727 btf = mod->btf; 12728 pattern = sep + 1; 12729 } else { 12730 /* Program is loaded for kernel module. */ 12731 if (prog->attach_btf_obj_fd) { 12732 err = -EINVAL; 12733 goto cleanup; 12734 } 12735 btf = obj->btf_vmlinux; 12736 } 12737 12738 err = collect_btf_func_ids_by_glob(btf, pattern, ids); 12739 12740 cleanup: 12741 bpf_object_cleanup_btf(obj); 12742 return err; 12743 } 12744 12745 struct bpf_link * 12746 bpf_program__attach_tracing_multi(const struct bpf_program *prog, const char *pattern, 12747 const struct bpf_tracing_multi_opts *opts) 12748 { 12749 LIBBPF_OPTS(bpf_link_create_opts, lopts); 12750 int prog_fd, link_fd, err, cnt; 12751 __u32 *free_ids = NULL; 12752 struct bpf_link *link; 12753 const __u64 *cookies; 12754 const __u32 *ids; 12755 12756 if (!OPTS_VALID(opts, bpf_tracing_multi_opts)) 12757 return libbpf_err_ptr(-EINVAL); 12758 12759 prog_fd = bpf_program__fd(prog); 12760 if (prog_fd < 0) { 12761 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", 12762 prog->name); 12763 return libbpf_err_ptr(-EINVAL); 12764 } 12765 12766 cnt = OPTS_GET(opts, cnt, 0); 12767 ids = OPTS_GET(opts, ids, NULL); 12768 cookies = OPTS_GET(opts, cookies, NULL); 12769 12770 if (!!ids != !!cnt) 12771 return libbpf_err_ptr(-EINVAL); 12772 if (pattern && (ids || cookies)) 12773 return libbpf_err_ptr(-EINVAL); 12774 if (!pattern && !ids) 12775 return libbpf_err_ptr(-EINVAL); 12776 12777 if (pattern) { 12778 cnt = collect_func_ids_by_glob(prog, pattern, &free_ids); 12779 if (cnt < 0) 12780 return libbpf_err_ptr(cnt); 12781 if (cnt == 0) 12782 return libbpf_err_ptr(-EINVAL); 12783 ids = (const __u32 *) free_ids; 12784 } 12785 12786 lopts.tracing_multi.ids = ids; 12787 lopts.tracing_multi.cookies = cookies; 12788 lopts.tracing_multi.cnt = cnt; 12789 12790 link = calloc(1, sizeof(*link)); 12791 if (!link) { 12792 err = -ENOMEM; 12793 goto error; 12794 } 12795 link->detach = &bpf_link__detach_fd; 12796 12797 link_fd = bpf_link_create(prog_fd, 0, prog->expected_attach_type, &lopts); 12798 if (link_fd < 0) { 12799 err = -errno; 12800 pr_warn("prog '%s': failed to attach: %s\n", prog->name, errstr(err)); 12801 goto error; 12802 } 12803 link->fd = link_fd; 12804 free(free_ids); 12805 return link; 12806 12807 error: 12808 free(link); 12809 free(free_ids); 12810 return libbpf_err_ptr(err); 12811 } 12812 12813 static int attach_tracing_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12814 { 12815 static const char *const prefixes[] = { 12816 "fentry.multi", 12817 "fexit.multi", 12818 "fsession.multi", 12819 "fentry.multi.s", 12820 "fexit.multi.s", 12821 "fsession.multi.s", 12822 }; 12823 const char *spec = NULL; 12824 char *pattern; 12825 size_t i; 12826 int n; 12827 12828 *link = NULL; 12829 12830 for (i = 0; i < ARRAY_SIZE(prefixes); i++) { 12831 size_t pfx_len; 12832 12833 if (!str_has_pfx(prog->sec_name, prefixes[i])) 12834 continue; 12835 12836 pfx_len = strlen(prefixes[i]); 12837 /* no auto-attach case of, e.g., SEC("fentry.multi") */ 12838 if (prog->sec_name[pfx_len] == '\0') 12839 return 0; 12840 12841 if (prog->sec_name[pfx_len] != '/') 12842 continue; 12843 12844 spec = prog->sec_name + pfx_len + 1; 12845 break; 12846 } 12847 12848 if (!spec) { 12849 pr_warn("prog '%s': invalid section name '%s'\n", 12850 prog->name, prog->sec_name); 12851 return -EINVAL; 12852 } 12853 12854 n = sscanf(spec, "%m[a-zA-Z0-9_.*?:]", &pattern); 12855 if (n < 1) { 12856 pr_warn("tracing multi pattern is invalid: %s\n", spec); 12857 return -EINVAL; 12858 } 12859 12860 *link = bpf_program__attach_tracing_multi(prog, pattern, NULL); 12861 free(pattern); 12862 return libbpf_get_error(*link); 12863 } 12864 12865 static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe, 12866 const char *binary_path, size_t offset) 12867 { 12868 return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx", 12869 retprobe ? 'r' : 'p', 12870 retprobe ? "uretprobes" : "uprobes", 12871 probe_name, binary_path, offset); 12872 } 12873 12874 static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe) 12875 { 12876 return append_to_file(tracefs_uprobe_events(), "-:%s/%s", 12877 retprobe ? "uretprobes" : "uprobes", probe_name); 12878 } 12879 12880 static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe) 12881 { 12882 char file[512]; 12883 12884 snprintf(file, sizeof(file), "%s/events/%s/%s/id", 12885 tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name); 12886 12887 return parse_uint_from_file(file, "%d\n"); 12888 } 12889 12890 static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, 12891 const char *binary_path, size_t offset, int pid) 12892 { 12893 const size_t attr_sz = sizeof(struct perf_event_attr); 12894 struct perf_event_attr attr; 12895 int type, pfd, err; 12896 12897 err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset); 12898 if (err < 0) { 12899 pr_warn("failed to add legacy uprobe event for %s:0x%zx: %s\n", 12900 binary_path, (size_t)offset, errstr(err)); 12901 return err; 12902 } 12903 type = determine_uprobe_perf_type_legacy(probe_name, retprobe); 12904 if (type < 0) { 12905 err = type; 12906 pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %s\n", 12907 binary_path, offset, errstr(err)); 12908 goto err_clean_legacy; 12909 } 12910 12911 memset(&attr, 0, attr_sz); 12912 attr.size = attr_sz; 12913 attr.config = type; 12914 attr.type = PERF_TYPE_TRACEPOINT; 12915 12916 pfd = syscall(__NR_perf_event_open, &attr, 12917 pid < 0 ? -1 : pid, /* pid */ 12918 pid == -1 ? 0 : -1, /* cpu */ 12919 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 12920 if (pfd < 0) { 12921 err = -errno; 12922 pr_warn("legacy uprobe perf_event_open() failed: %s\n", errstr(err)); 12923 goto err_clean_legacy; 12924 } 12925 return pfd; 12926 12927 err_clean_legacy: 12928 /* Clear the newly added legacy uprobe_event */ 12929 remove_uprobe_event_legacy(probe_name, retprobe); 12930 return err; 12931 } 12932 12933 /* Find offset of function name in archive specified by path. Currently 12934 * supported are .zip files that do not compress their contents, as used on 12935 * Android in the form of APKs, for example. "file_name" is the name of the ELF 12936 * file inside the archive. "func_name" matches symbol name or name@@LIB for 12937 * library functions. 12938 * 12939 * An overview of the APK format specifically provided here: 12940 * https://en.wikipedia.org/w/index.php?title=Apk_(file_format)&oldid=1139099120#Package_contents 12941 */ 12942 static long elf_find_func_offset_from_archive(const char *archive_path, const char *file_name, 12943 const char *func_name) 12944 { 12945 struct zip_archive *archive; 12946 struct zip_entry entry; 12947 long ret; 12948 Elf *elf; 12949 12950 archive = zip_archive_open(archive_path); 12951 if (IS_ERR(archive)) { 12952 ret = PTR_ERR(archive); 12953 pr_warn("zip: failed to open %s: %ld\n", archive_path, ret); 12954 return ret; 12955 } 12956 12957 ret = zip_archive_find_entry(archive, file_name, &entry); 12958 if (ret) { 12959 pr_warn("zip: could not find archive member %s in %s: %ld\n", file_name, 12960 archive_path, ret); 12961 goto out; 12962 } 12963 pr_debug("zip: found entry for %s in %s at 0x%lx\n", file_name, archive_path, 12964 (unsigned long)entry.data_offset); 12965 12966 if (entry.compression) { 12967 pr_warn("zip: entry %s of %s is compressed and cannot be handled\n", file_name, 12968 archive_path); 12969 ret = -LIBBPF_ERRNO__FORMAT; 12970 goto out; 12971 } 12972 12973 elf = elf_memory((void *)entry.data, entry.data_length); 12974 if (!elf) { 12975 pr_warn("elf: could not read elf file %s from %s: %s\n", file_name, archive_path, 12976 elf_errmsg(-1)); 12977 ret = -LIBBPF_ERRNO__LIBELF; 12978 goto out; 12979 } 12980 12981 ret = elf_find_func_offset(elf, file_name, func_name); 12982 if (ret > 0) { 12983 pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n", 12984 func_name, file_name, archive_path, entry.data_offset, (unsigned long)ret, 12985 (unsigned long)(ret + entry.data_offset)); 12986 ret += entry.data_offset; 12987 } 12988 elf_end(elf); 12989 12990 out: 12991 zip_archive_close(archive); 12992 return ret; 12993 } 12994 12995 static const char *arch_specific_lib_paths(void) 12996 { 12997 /* 12998 * Based on https://packages.debian.org/sid/libc6. 12999 * 13000 * Assume that the traced program is built for the same architecture 13001 * as libbpf, which should cover the vast majority of cases. 13002 */ 13003 #if defined(__x86_64__) 13004 return "/lib/x86_64-linux-gnu"; 13005 #elif defined(__i386__) 13006 return "/lib/i386-linux-gnu"; 13007 #elif defined(__s390x__) 13008 return "/lib/s390x-linux-gnu"; 13009 #elif defined(__arm__) && defined(__SOFTFP__) 13010 return "/lib/arm-linux-gnueabi"; 13011 #elif defined(__arm__) && !defined(__SOFTFP__) 13012 return "/lib/arm-linux-gnueabihf"; 13013 #elif defined(__aarch64__) 13014 return "/lib/aarch64-linux-gnu"; 13015 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64 13016 return "/lib/mips64el-linux-gnuabi64"; 13017 #elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32 13018 return "/lib/mipsel-linux-gnu"; 13019 #elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 13020 return "/lib/powerpc64le-linux-gnu"; 13021 #elif defined(__sparc__) && defined(__arch64__) 13022 return "/lib/sparc64-linux-gnu"; 13023 #elif defined(__riscv) && __riscv_xlen == 64 13024 return "/lib/riscv64-linux-gnu"; 13025 #else 13026 return NULL; 13027 #endif 13028 } 13029 13030 /* Get full path to program/shared library. */ 13031 static int resolve_full_path(const char *file, char *result, size_t result_sz) 13032 { 13033 const char *search_paths[4] = {}; 13034 int i, perm; 13035 13036 if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { 13037 search_paths[0] = getenv("LD_LIBRARY_PATH"); 13038 search_paths[1] = "/usr/lib64:/usr/lib"; 13039 search_paths[2] = arch_specific_lib_paths(); 13040 search_paths[3] = "/lib64:/lib"; 13041 perm = R_OK; 13042 } else { 13043 search_paths[0] = getenv("PATH"); 13044 search_paths[1] = "/usr/bin:/usr/sbin"; 13045 perm = R_OK | X_OK; 13046 } 13047 13048 for (i = 0; i < ARRAY_SIZE(search_paths); i++) { 13049 const char *s; 13050 13051 if (!search_paths[i]) 13052 continue; 13053 for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) { 13054 const char *next_path; 13055 int seg_len; 13056 13057 if (s[0] == ':') 13058 s++; 13059 next_path = strchr(s, ':'); 13060 seg_len = next_path ? next_path - s : strlen(s); 13061 if (!seg_len) 13062 continue; 13063 snprintf(result, result_sz, "%.*s/%s", seg_len, s, file); 13064 /* ensure it has required permissions */ 13065 if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0) 13066 continue; 13067 pr_debug("resolved '%s' to '%s'\n", file, result); 13068 return 0; 13069 } 13070 } 13071 return -ENOENT; 13072 } 13073 13074 struct bpf_link * 13075 bpf_program__attach_uprobe_multi(const struct bpf_program *prog, 13076 pid_t pid, 13077 const char *path, 13078 const char *func_pattern, 13079 const struct bpf_uprobe_multi_opts *opts) 13080 { 13081 const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL; 13082 LIBBPF_OPTS(bpf_link_create_opts, lopts); 13083 unsigned long *resolved_offsets = NULL; 13084 enum bpf_attach_type attach_type; 13085 int err = 0, link_fd, prog_fd; 13086 struct bpf_link *link = NULL; 13087 char full_path[PATH_MAX]; 13088 bool retprobe, session; 13089 const __u64 *cookies; 13090 const char **syms; 13091 size_t cnt; 13092 13093 if (!OPTS_VALID(opts, bpf_uprobe_multi_opts)) 13094 return libbpf_err_ptr(-EINVAL); 13095 13096 prog_fd = bpf_program__fd(prog); 13097 if (prog_fd < 0) { 13098 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", 13099 prog->name); 13100 return libbpf_err_ptr(-EINVAL); 13101 } 13102 13103 syms = OPTS_GET(opts, syms, NULL); 13104 offsets = OPTS_GET(opts, offsets, NULL); 13105 ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL); 13106 cookies = OPTS_GET(opts, cookies, NULL); 13107 cnt = OPTS_GET(opts, cnt, 0); 13108 retprobe = OPTS_GET(opts, retprobe, false); 13109 session = OPTS_GET(opts, session, false); 13110 13111 /* 13112 * User can specify 2 mutually exclusive set of inputs: 13113 * 13114 * 1) use only path/func_pattern/pid arguments 13115 * 13116 * 2) use path/pid with allowed combinations of: 13117 * syms/offsets/ref_ctr_offsets/cookies/cnt 13118 * 13119 * - syms and offsets are mutually exclusive 13120 * - ref_ctr_offsets and cookies are optional 13121 * 13122 * Any other usage results in error. 13123 */ 13124 13125 if (!path) 13126 return libbpf_err_ptr(-EINVAL); 13127 if (!func_pattern && cnt == 0) 13128 return libbpf_err_ptr(-EINVAL); 13129 13130 if (func_pattern) { 13131 if (syms || offsets || ref_ctr_offsets || cookies || cnt) 13132 return libbpf_err_ptr(-EINVAL); 13133 } else { 13134 if (!!syms == !!offsets) 13135 return libbpf_err_ptr(-EINVAL); 13136 } 13137 13138 if (retprobe && session) 13139 return libbpf_err_ptr(-EINVAL); 13140 13141 if (func_pattern) { 13142 if (!strchr(path, '/')) { 13143 err = resolve_full_path(path, full_path, sizeof(full_path)); 13144 if (err) { 13145 pr_warn("prog '%s': failed to resolve full path for '%s': %s\n", 13146 prog->name, path, errstr(err)); 13147 return libbpf_err_ptr(err); 13148 } 13149 path = full_path; 13150 } 13151 13152 err = elf_resolve_pattern_offsets(path, func_pattern, 13153 &resolved_offsets, &cnt); 13154 if (err < 0) 13155 return libbpf_err_ptr(err); 13156 offsets = resolved_offsets; 13157 } else if (syms) { 13158 err = elf_resolve_syms_offsets(path, cnt, syms, &resolved_offsets, STT_FUNC); 13159 if (err < 0) 13160 return libbpf_err_ptr(err); 13161 offsets = resolved_offsets; 13162 } 13163 13164 attach_type = session ? BPF_TRACE_UPROBE_SESSION : BPF_TRACE_UPROBE_MULTI; 13165 13166 lopts.uprobe_multi.path = path; 13167 lopts.uprobe_multi.offsets = offsets; 13168 lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets; 13169 lopts.uprobe_multi.cookies = cookies; 13170 lopts.uprobe_multi.cnt = cnt; 13171 lopts.uprobe_multi.flags = retprobe ? BPF_F_UPROBE_MULTI_RETURN : 0; 13172 13173 if (pid == 0) 13174 pid = getpid(); 13175 if (pid > 0) 13176 lopts.uprobe_multi.pid = pid; 13177 13178 link = calloc(1, sizeof(*link)); 13179 if (!link) { 13180 err = -ENOMEM; 13181 goto error; 13182 } 13183 link->detach = &bpf_link__detach_fd; 13184 13185 link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts); 13186 if (link_fd < 0) { 13187 err = -errno; 13188 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n", 13189 prog->name, errstr(err)); 13190 goto error; 13191 } 13192 link->fd = link_fd; 13193 free(resolved_offsets); 13194 return link; 13195 13196 error: 13197 free(resolved_offsets); 13198 free(link); 13199 return libbpf_err_ptr(err); 13200 } 13201 13202 LIBBPF_API struct bpf_link * 13203 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, 13204 const char *binary_path, size_t func_offset, 13205 const struct bpf_uprobe_opts *opts) 13206 { 13207 const char *archive_path = NULL, *archive_sep = NULL; 13208 char *legacy_probe = NULL; 13209 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 13210 enum probe_attach_mode attach_mode; 13211 char full_path[PATH_MAX]; 13212 struct bpf_link *link; 13213 size_t ref_ctr_off; 13214 int pfd, err; 13215 bool retprobe, legacy; 13216 const char *func_name; 13217 13218 if (!OPTS_VALID(opts, bpf_uprobe_opts)) 13219 return libbpf_err_ptr(-EINVAL); 13220 13221 attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT); 13222 retprobe = OPTS_GET(opts, retprobe, false); 13223 ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0); 13224 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 13225 13226 if (!binary_path) 13227 return libbpf_err_ptr(-EINVAL); 13228 13229 /* Check if "binary_path" refers to an archive. */ 13230 archive_sep = strstr(binary_path, "!/"); 13231 if (archive_sep) { 13232 full_path[0] = '\0'; 13233 libbpf_strlcpy(full_path, binary_path, 13234 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1))); 13235 archive_path = full_path; 13236 binary_path = archive_sep + 2; 13237 } else if (!strchr(binary_path, '/')) { 13238 err = resolve_full_path(binary_path, full_path, sizeof(full_path)); 13239 if (err) { 13240 pr_warn("prog '%s': failed to resolve full path for '%s': %s\n", 13241 prog->name, binary_path, errstr(err)); 13242 return libbpf_err_ptr(err); 13243 } 13244 binary_path = full_path; 13245 } 13246 func_name = OPTS_GET(opts, func_name, NULL); 13247 if (func_name) { 13248 long sym_off; 13249 13250 if (archive_path) { 13251 sym_off = elf_find_func_offset_from_archive(archive_path, binary_path, 13252 func_name); 13253 binary_path = archive_path; 13254 } else { 13255 sym_off = elf_find_func_offset_from_file(binary_path, func_name); 13256 } 13257 if (sym_off < 0) 13258 return libbpf_err_ptr(sym_off); 13259 func_offset += sym_off; 13260 } 13261 13262 legacy = determine_uprobe_perf_type() < 0; 13263 switch (attach_mode) { 13264 case PROBE_ATTACH_MODE_LEGACY: 13265 legacy = true; 13266 pe_opts.force_ioctl_attach = true; 13267 break; 13268 case PROBE_ATTACH_MODE_PERF: 13269 if (legacy) 13270 return libbpf_err_ptr(-ENOTSUP); 13271 pe_opts.force_ioctl_attach = true; 13272 break; 13273 case PROBE_ATTACH_MODE_LINK: 13274 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) 13275 return libbpf_err_ptr(-ENOTSUP); 13276 break; 13277 case PROBE_ATTACH_MODE_DEFAULT: 13278 break; 13279 default: 13280 return libbpf_err_ptr(-EINVAL); 13281 } 13282 13283 if (!legacy) { 13284 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path, 13285 func_offset, pid, ref_ctr_off); 13286 } else { 13287 char probe_name[MAX_EVENT_NAME_LEN]; 13288 13289 if (ref_ctr_off) 13290 return libbpf_err_ptr(-EINVAL); 13291 13292 gen_probe_legacy_event_name(probe_name, sizeof(probe_name), 13293 strrchr(binary_path, '/') ? : binary_path, 13294 func_offset); 13295 13296 legacy_probe = strdup(probe_name); 13297 if (!legacy_probe) 13298 return libbpf_err_ptr(-ENOMEM); 13299 13300 pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe, 13301 binary_path, func_offset, pid); 13302 } 13303 if (pfd < 0) { 13304 err = pfd; 13305 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 13306 prog->name, retprobe ? "uretprobe" : "uprobe", 13307 binary_path, func_offset, 13308 errstr(err)); 13309 goto err_out; 13310 } 13311 13312 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 13313 err = libbpf_get_error(link); 13314 if (err) { 13315 close(pfd); 13316 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n", 13317 prog->name, retprobe ? "uretprobe" : "uprobe", 13318 binary_path, func_offset, 13319 errstr(err)); 13320 goto err_clean_legacy; 13321 } 13322 if (legacy) { 13323 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 13324 13325 perf_link->legacy_probe_name = legacy_probe; 13326 perf_link->legacy_is_kprobe = false; 13327 perf_link->legacy_is_retprobe = retprobe; 13328 } 13329 return link; 13330 13331 err_clean_legacy: 13332 if (legacy) 13333 remove_uprobe_event_legacy(legacy_probe, retprobe); 13334 err_out: 13335 free(legacy_probe); 13336 return libbpf_err_ptr(err); 13337 } 13338 13339 /* Format of u[ret]probe section definition supporting auto-attach: 13340 * u[ret]probe/binary:function[+offset] 13341 * 13342 * binary can be an absolute/relative path or a filename; the latter is resolved to a 13343 * full binary path via bpf_program__attach_uprobe_opts. 13344 * 13345 * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be 13346 * specified (and auto-attach is not possible) or the above format is specified for 13347 * auto-attach. 13348 */ 13349 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) 13350 { 13351 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts); 13352 char *probe_type = NULL, *binary_path = NULL, *func_name = NULL, *func_off; 13353 int n, c, ret = -EINVAL; 13354 long offset = 0; 13355 13356 *link = NULL; 13357 13358 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", 13359 &probe_type, &binary_path, &func_name); 13360 switch (n) { 13361 case 1: 13362 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ 13363 ret = 0; 13364 break; 13365 case 2: 13366 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n", 13367 prog->name, prog->sec_name); 13368 break; 13369 case 3: 13370 /* check if user specifies `+offset`, if yes, this should be 13371 * the last part of the string, make sure sscanf read to EOL 13372 */ 13373 func_off = strrchr(func_name, '+'); 13374 if (func_off) { 13375 n = sscanf(func_off, "+%li%n", &offset, &c); 13376 if (n == 1 && *(func_off + c) == '\0') 13377 func_off[0] = '\0'; 13378 else 13379 offset = 0; 13380 } 13381 opts.retprobe = strcmp(probe_type, "uretprobe") == 0 || 13382 strcmp(probe_type, "uretprobe.s") == 0; 13383 if (opts.retprobe && offset != 0) { 13384 pr_warn("prog '%s': uretprobes do not support offset specification\n", 13385 prog->name); 13386 break; 13387 } 13388 opts.func_name = func_name; 13389 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts); 13390 ret = libbpf_get_error(*link); 13391 break; 13392 default: 13393 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, 13394 prog->sec_name); 13395 break; 13396 } 13397 free(probe_type); 13398 free(binary_path); 13399 free(func_name); 13400 13401 return ret; 13402 } 13403 13404 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog, 13405 bool retprobe, pid_t pid, 13406 const char *binary_path, 13407 size_t func_offset) 13408 { 13409 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe); 13410 13411 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts); 13412 } 13413 13414 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog, 13415 pid_t pid, const char *binary_path, 13416 const char *usdt_provider, const char *usdt_name, 13417 const struct bpf_usdt_opts *opts) 13418 { 13419 char resolved_path[512]; 13420 struct bpf_object *obj = prog->obj; 13421 struct bpf_link *link; 13422 __u64 usdt_cookie; 13423 int err; 13424 13425 if (!OPTS_VALID(opts, bpf_uprobe_opts)) 13426 return libbpf_err_ptr(-EINVAL); 13427 13428 if (bpf_program__fd(prog) < 0) { 13429 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", 13430 prog->name); 13431 return libbpf_err_ptr(-EINVAL); 13432 } 13433 13434 if (!binary_path) 13435 return libbpf_err_ptr(-EINVAL); 13436 13437 if (!strchr(binary_path, '/')) { 13438 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path)); 13439 if (err) { 13440 pr_warn("prog '%s': failed to resolve full path for '%s': %s\n", 13441 prog->name, binary_path, errstr(err)); 13442 return libbpf_err_ptr(err); 13443 } 13444 binary_path = resolved_path; 13445 } 13446 13447 /* USDT manager is instantiated lazily on first USDT attach. It will 13448 * be destroyed together with BPF object in bpf_object__close(). 13449 */ 13450 if (IS_ERR(obj->usdt_man)) 13451 return libbpf_ptr(obj->usdt_man); 13452 if (!obj->usdt_man) { 13453 obj->usdt_man = usdt_manager_new(obj); 13454 if (IS_ERR(obj->usdt_man)) 13455 return libbpf_ptr(obj->usdt_man); 13456 } 13457 13458 usdt_cookie = OPTS_GET(opts, usdt_cookie, 0); 13459 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path, 13460 usdt_provider, usdt_name, usdt_cookie); 13461 err = libbpf_get_error(link); 13462 if (err) 13463 return libbpf_err_ptr(err); 13464 return link; 13465 } 13466 13467 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link) 13468 { 13469 char *path = NULL, *provider = NULL, *name = NULL; 13470 const char *sec_name; 13471 int n, err; 13472 13473 sec_name = bpf_program__section_name(prog); 13474 if (strcmp(sec_name, "usdt") == 0) { 13475 /* no auto-attach for just SEC("usdt") */ 13476 *link = NULL; 13477 return 0; 13478 } 13479 13480 n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name); 13481 if (n != 3) { 13482 pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n", 13483 sec_name); 13484 err = -EINVAL; 13485 } else { 13486 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path, 13487 provider, name, NULL); 13488 err = libbpf_get_error(*link); 13489 } 13490 free(path); 13491 free(provider); 13492 free(name); 13493 return err; 13494 } 13495 13496 static int determine_tracepoint_id(const char *tp_category, 13497 const char *tp_name) 13498 { 13499 char file[PATH_MAX]; 13500 int ret; 13501 13502 ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id", 13503 tracefs_path(), tp_category, tp_name); 13504 if (ret < 0) 13505 return -errno; 13506 if (ret >= sizeof(file)) { 13507 pr_debug("tracepoint %s/%s path is too long\n", 13508 tp_category, tp_name); 13509 return -E2BIG; 13510 } 13511 return parse_uint_from_file(file, "%d\n"); 13512 } 13513 13514 static int perf_event_open_tracepoint(const char *tp_category, 13515 const char *tp_name) 13516 { 13517 const size_t attr_sz = sizeof(struct perf_event_attr); 13518 struct perf_event_attr attr; 13519 int tp_id, pfd, err; 13520 13521 tp_id = determine_tracepoint_id(tp_category, tp_name); 13522 if (tp_id < 0) { 13523 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 13524 tp_category, tp_name, 13525 errstr(tp_id)); 13526 return tp_id; 13527 } 13528 13529 memset(&attr, 0, attr_sz); 13530 attr.type = PERF_TYPE_TRACEPOINT; 13531 attr.size = attr_sz; 13532 attr.config = tp_id; 13533 13534 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 13535 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 13536 if (pfd < 0) { 13537 err = -errno; 13538 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", 13539 tp_category, tp_name, 13540 errstr(err)); 13541 return err; 13542 } 13543 return pfd; 13544 } 13545 13546 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, 13547 const char *tp_category, 13548 const char *tp_name, 13549 const struct bpf_tracepoint_opts *opts) 13550 { 13551 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 13552 struct bpf_link *link; 13553 int pfd, err; 13554 13555 if (!OPTS_VALID(opts, bpf_tracepoint_opts)) 13556 return libbpf_err_ptr(-EINVAL); 13557 13558 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 13559 13560 pfd = perf_event_open_tracepoint(tp_category, tp_name); 13561 if (pfd < 0) { 13562 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 13563 prog->name, tp_category, tp_name, 13564 errstr(pfd)); 13565 return libbpf_err_ptr(pfd); 13566 } 13567 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 13568 err = libbpf_get_error(link); 13569 if (err) { 13570 close(pfd); 13571 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n", 13572 prog->name, tp_category, tp_name, 13573 errstr(err)); 13574 return libbpf_err_ptr(err); 13575 } 13576 return link; 13577 } 13578 13579 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog, 13580 const char *tp_category, 13581 const char *tp_name) 13582 { 13583 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL); 13584 } 13585 13586 /* 13587 * Match section name against a prefix array. Returns pointer past 13588 * "prefix/" on match, empty string for bare sections (exact prefix 13589 * match), or NULL if no prefix matches. 13590 */ 13591 static const char *sec_name_match_prefix(const char *sec_name, 13592 const char *const *prefixes, 13593 size_t n) 13594 { 13595 size_t i; 13596 13597 for (i = 0; i < n; i++) { 13598 size_t pfx_len; 13599 13600 if (!str_has_pfx(sec_name, prefixes[i])) 13601 continue; 13602 13603 pfx_len = strlen(prefixes[i]); 13604 if (sec_name[pfx_len] == '\0') 13605 return sec_name + pfx_len; 13606 13607 if (sec_name[pfx_len] != '/' || sec_name[pfx_len + 1] == '\0') 13608 continue; 13609 13610 return sec_name + pfx_len + 1; 13611 } 13612 return NULL; 13613 } 13614 13615 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) 13616 { 13617 static const char *const prefixes[] = { 13618 "tp.s", 13619 "tp", 13620 "tracepoint.s", 13621 "tracepoint", 13622 }; 13623 char *sec_name, *tp_cat, *tp_name; 13624 const char *match; 13625 13626 *link = NULL; 13627 13628 match = sec_name_match_prefix(prog->sec_name, prefixes, ARRAY_SIZE(prefixes)); 13629 if (!match) { 13630 pr_warn("prog '%s': invalid section name '%s'\n", prog->name, prog->sec_name); 13631 return -EINVAL; 13632 } 13633 if (!match[0]) /* bare section name no autoattach */ 13634 return 0; 13635 13636 sec_name = strdup(prog->sec_name); 13637 if (!sec_name) 13638 return -ENOMEM; 13639 13640 tp_cat = sec_name + (match - prog->sec_name); 13641 tp_name = strchr(tp_cat, '/'); 13642 if (!tp_name) { 13643 free(sec_name); 13644 return -EINVAL; 13645 } 13646 *tp_name = '\0'; 13647 tp_name++; 13648 13649 *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); 13650 free(sec_name); 13651 return libbpf_get_error(*link); 13652 } 13653 13654 struct bpf_link * 13655 bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog, 13656 const char *tp_name, 13657 struct bpf_raw_tracepoint_opts *opts) 13658 { 13659 LIBBPF_OPTS(bpf_raw_tp_opts, raw_opts); 13660 struct bpf_link *link; 13661 int prog_fd, pfd; 13662 13663 if (!OPTS_VALID(opts, bpf_raw_tracepoint_opts)) 13664 return libbpf_err_ptr(-EINVAL); 13665 13666 prog_fd = bpf_program__fd(prog); 13667 if (prog_fd < 0) { 13668 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 13669 return libbpf_err_ptr(-EINVAL); 13670 } 13671 13672 link = calloc(1, sizeof(*link)); 13673 if (!link) 13674 return libbpf_err_ptr(-ENOMEM); 13675 link->detach = &bpf_link__detach_fd; 13676 13677 raw_opts.tp_name = tp_name; 13678 raw_opts.cookie = OPTS_GET(opts, cookie, 0); 13679 pfd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_opts); 13680 if (pfd < 0) { 13681 pfd = -errno; 13682 free(link); 13683 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n", 13684 prog->name, tp_name, errstr(pfd)); 13685 return libbpf_err_ptr(pfd); 13686 } 13687 link->fd = pfd; 13688 return link; 13689 } 13690 13691 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog, 13692 const char *tp_name) 13693 { 13694 return bpf_program__attach_raw_tracepoint_opts(prog, tp_name, NULL); 13695 } 13696 13697 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) 13698 { 13699 static const char *const prefixes[] = { 13700 "raw_tp", 13701 "raw_tracepoint", 13702 "raw_tp.w", 13703 "raw_tracepoint.w", 13704 "raw_tp.s", 13705 "raw_tracepoint.s", 13706 }; 13707 const char *match; 13708 13709 *link = NULL; 13710 13711 match = sec_name_match_prefix(prog->sec_name, prefixes, ARRAY_SIZE(prefixes)); 13712 if (!match) { 13713 pr_warn("prog '%s': invalid section name '%s'\n", prog->name, prog->sec_name); 13714 return -EINVAL; 13715 } 13716 if (!match[0]) 13717 return 0; 13718 13719 *link = bpf_program__attach_raw_tracepoint(prog, match); 13720 return libbpf_get_error(*link); 13721 } 13722 13723 /* Common logic for all BPF program types that attach to a btf_id */ 13724 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog, 13725 const struct bpf_trace_opts *opts) 13726 { 13727 LIBBPF_OPTS(bpf_link_create_opts, link_opts); 13728 struct bpf_link *link; 13729 int prog_fd, pfd; 13730 13731 if (!OPTS_VALID(opts, bpf_trace_opts)) 13732 return libbpf_err_ptr(-EINVAL); 13733 13734 prog_fd = bpf_program__fd(prog); 13735 if (prog_fd < 0) { 13736 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 13737 return libbpf_err_ptr(-EINVAL); 13738 } 13739 13740 link = calloc(1, sizeof(*link)); 13741 if (!link) 13742 return libbpf_err_ptr(-ENOMEM); 13743 link->detach = &bpf_link__detach_fd; 13744 13745 /* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */ 13746 link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0); 13747 pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts); 13748 if (pfd < 0) { 13749 pfd = -errno; 13750 free(link); 13751 pr_warn("prog '%s': failed to attach: %s\n", 13752 prog->name, errstr(pfd)); 13753 return libbpf_err_ptr(pfd); 13754 } 13755 link->fd = pfd; 13756 return link; 13757 } 13758 13759 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) 13760 { 13761 return bpf_program__attach_btf_id(prog, NULL); 13762 } 13763 13764 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog, 13765 const struct bpf_trace_opts *opts) 13766 { 13767 return bpf_program__attach_btf_id(prog, opts); 13768 } 13769 13770 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog) 13771 { 13772 return bpf_program__attach_btf_id(prog, NULL); 13773 } 13774 13775 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link) 13776 { 13777 *link = bpf_program__attach_trace(prog); 13778 return libbpf_get_error(*link); 13779 } 13780 13781 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link) 13782 { 13783 *link = bpf_program__attach_lsm(prog); 13784 return libbpf_get_error(*link); 13785 } 13786 13787 static struct bpf_link * 13788 bpf_program_attach_fd(const struct bpf_program *prog, 13789 int target_fd, const char *target_name, 13790 const struct bpf_link_create_opts *opts) 13791 { 13792 enum bpf_attach_type attach_type; 13793 struct bpf_link *link; 13794 int prog_fd, link_fd; 13795 13796 prog_fd = bpf_program__fd(prog); 13797 if (prog_fd < 0) { 13798 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 13799 return libbpf_err_ptr(-EINVAL); 13800 } 13801 13802 link = calloc(1, sizeof(*link)); 13803 if (!link) 13804 return libbpf_err_ptr(-ENOMEM); 13805 link->detach = &bpf_link__detach_fd; 13806 13807 attach_type = bpf_program__expected_attach_type(prog); 13808 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, opts); 13809 if (link_fd < 0) { 13810 link_fd = -errno; 13811 free(link); 13812 pr_warn("prog '%s': failed to attach to %s: %s\n", 13813 prog->name, target_name, 13814 errstr(link_fd)); 13815 return libbpf_err_ptr(link_fd); 13816 } 13817 link->fd = link_fd; 13818 return link; 13819 } 13820 13821 struct bpf_link * 13822 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd) 13823 { 13824 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL); 13825 } 13826 13827 struct bpf_link * 13828 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd) 13829 { 13830 return bpf_program_attach_fd(prog, netns_fd, "netns", NULL); 13831 } 13832 13833 struct bpf_link * 13834 bpf_program__attach_sockmap(const struct bpf_program *prog, int map_fd) 13835 { 13836 return bpf_program_attach_fd(prog, map_fd, "sockmap", NULL); 13837 } 13838 13839 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex) 13840 { 13841 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 13842 return bpf_program_attach_fd(prog, ifindex, "xdp", NULL); 13843 } 13844 13845 struct bpf_link * 13846 bpf_program__attach_cgroup_opts(const struct bpf_program *prog, int cgroup_fd, 13847 const struct bpf_cgroup_opts *opts) 13848 { 13849 LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 13850 __u32 relative_id; 13851 int relative_fd; 13852 13853 if (!OPTS_VALID(opts, bpf_cgroup_opts)) 13854 return libbpf_err_ptr(-EINVAL); 13855 13856 relative_id = OPTS_GET(opts, relative_id, 0); 13857 relative_fd = OPTS_GET(opts, relative_fd, 0); 13858 13859 if (relative_fd && relative_id) { 13860 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n", 13861 prog->name); 13862 return libbpf_err_ptr(-EINVAL); 13863 } 13864 13865 link_create_opts.cgroup.expected_revision = OPTS_GET(opts, expected_revision, 0); 13866 link_create_opts.cgroup.relative_fd = relative_fd; 13867 link_create_opts.cgroup.relative_id = relative_id; 13868 link_create_opts.flags = OPTS_GET(opts, flags, 0); 13869 13870 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", &link_create_opts); 13871 } 13872 13873 struct bpf_link * 13874 bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex, 13875 const struct bpf_tcx_opts *opts) 13876 { 13877 LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 13878 __u32 relative_id; 13879 int relative_fd; 13880 13881 if (!OPTS_VALID(opts, bpf_tcx_opts)) 13882 return libbpf_err_ptr(-EINVAL); 13883 13884 relative_id = OPTS_GET(opts, relative_id, 0); 13885 relative_fd = OPTS_GET(opts, relative_fd, 0); 13886 13887 /* validate we don't have unexpected combinations of non-zero fields */ 13888 if (!ifindex) { 13889 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n", 13890 prog->name); 13891 return libbpf_err_ptr(-EINVAL); 13892 } 13893 if (relative_fd && relative_id) { 13894 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n", 13895 prog->name); 13896 return libbpf_err_ptr(-EINVAL); 13897 } 13898 13899 link_create_opts.tcx.expected_revision = OPTS_GET(opts, expected_revision, 0); 13900 link_create_opts.tcx.relative_fd = relative_fd; 13901 link_create_opts.tcx.relative_id = relative_id; 13902 link_create_opts.flags = OPTS_GET(opts, flags, 0); 13903 13904 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 13905 return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts); 13906 } 13907 13908 struct bpf_link * 13909 bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex, 13910 const struct bpf_netkit_opts *opts) 13911 { 13912 LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 13913 __u32 relative_id; 13914 int relative_fd; 13915 13916 if (!OPTS_VALID(opts, bpf_netkit_opts)) 13917 return libbpf_err_ptr(-EINVAL); 13918 13919 relative_id = OPTS_GET(opts, relative_id, 0); 13920 relative_fd = OPTS_GET(opts, relative_fd, 0); 13921 13922 /* validate we don't have unexpected combinations of non-zero fields */ 13923 if (!ifindex) { 13924 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n", 13925 prog->name); 13926 return libbpf_err_ptr(-EINVAL); 13927 } 13928 if (relative_fd && relative_id) { 13929 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n", 13930 prog->name); 13931 return libbpf_err_ptr(-EINVAL); 13932 } 13933 13934 link_create_opts.netkit.expected_revision = OPTS_GET(opts, expected_revision, 0); 13935 link_create_opts.netkit.relative_fd = relative_fd; 13936 link_create_opts.netkit.relative_id = relative_id; 13937 link_create_opts.flags = OPTS_GET(opts, flags, 0); 13938 13939 return bpf_program_attach_fd(prog, ifindex, "netkit", &link_create_opts); 13940 } 13941 13942 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, 13943 int target_fd, 13944 const char *attach_func_name) 13945 { 13946 int btf_id; 13947 13948 if (!!target_fd != !!attach_func_name) { 13949 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n", 13950 prog->name); 13951 return libbpf_err_ptr(-EINVAL); 13952 } 13953 13954 if (prog->type != BPF_PROG_TYPE_EXT) { 13955 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace\n", 13956 prog->name); 13957 return libbpf_err_ptr(-EINVAL); 13958 } 13959 13960 if (target_fd) { 13961 LIBBPF_OPTS(bpf_link_create_opts, target_opts); 13962 13963 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd, prog->obj->token_fd); 13964 if (btf_id < 0) 13965 return libbpf_err_ptr(btf_id); 13966 13967 target_opts.target_btf_id = btf_id; 13968 13969 return bpf_program_attach_fd(prog, target_fd, "freplace", 13970 &target_opts); 13971 } else { 13972 /* no target, so use raw_tracepoint_open for compatibility 13973 * with old kernels 13974 */ 13975 return bpf_program__attach_trace(prog); 13976 } 13977 } 13978 13979 struct bpf_link * 13980 bpf_program__attach_iter(const struct bpf_program *prog, 13981 const struct bpf_iter_attach_opts *opts) 13982 { 13983 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 13984 struct bpf_link *link; 13985 int prog_fd, link_fd; 13986 __u32 target_fd = 0; 13987 13988 if (!OPTS_VALID(opts, bpf_iter_attach_opts)) 13989 return libbpf_err_ptr(-EINVAL); 13990 13991 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0); 13992 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0); 13993 13994 prog_fd = bpf_program__fd(prog); 13995 if (prog_fd < 0) { 13996 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 13997 return libbpf_err_ptr(-EINVAL); 13998 } 13999 14000 link = calloc(1, sizeof(*link)); 14001 if (!link) 14002 return libbpf_err_ptr(-ENOMEM); 14003 link->detach = &bpf_link__detach_fd; 14004 14005 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER, 14006 &link_create_opts); 14007 if (link_fd < 0) { 14008 link_fd = -errno; 14009 free(link); 14010 pr_warn("prog '%s': failed to attach to iterator: %s\n", 14011 prog->name, errstr(link_fd)); 14012 return libbpf_err_ptr(link_fd); 14013 } 14014 link->fd = link_fd; 14015 return link; 14016 } 14017 14018 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link) 14019 { 14020 *link = bpf_program__attach_iter(prog, NULL); 14021 return libbpf_get_error(*link); 14022 } 14023 14024 struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog, 14025 const struct bpf_netfilter_opts *opts) 14026 { 14027 LIBBPF_OPTS(bpf_link_create_opts, lopts); 14028 struct bpf_link *link; 14029 int prog_fd, link_fd; 14030 14031 if (!OPTS_VALID(opts, bpf_netfilter_opts)) 14032 return libbpf_err_ptr(-EINVAL); 14033 14034 prog_fd = bpf_program__fd(prog); 14035 if (prog_fd < 0) { 14036 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 14037 return libbpf_err_ptr(-EINVAL); 14038 } 14039 14040 link = calloc(1, sizeof(*link)); 14041 if (!link) 14042 return libbpf_err_ptr(-ENOMEM); 14043 14044 link->detach = &bpf_link__detach_fd; 14045 14046 lopts.netfilter.pf = OPTS_GET(opts, pf, 0); 14047 lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0); 14048 lopts.netfilter.priority = OPTS_GET(opts, priority, 0); 14049 lopts.netfilter.flags = OPTS_GET(opts, flags, 0); 14050 14051 link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts); 14052 if (link_fd < 0) { 14053 link_fd = -errno; 14054 free(link); 14055 pr_warn("prog '%s': failed to attach to netfilter: %s\n", 14056 prog->name, errstr(link_fd)); 14057 return libbpf_err_ptr(link_fd); 14058 } 14059 link->fd = link_fd; 14060 14061 return link; 14062 } 14063 14064 struct bpf_link *bpf_program__attach(const struct bpf_program *prog) 14065 { 14066 struct bpf_link *link = NULL; 14067 int err; 14068 14069 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) 14070 return libbpf_err_ptr(-EOPNOTSUPP); 14071 14072 if (bpf_program__fd(prog) < 0) { 14073 pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n", 14074 prog->name); 14075 return libbpf_err_ptr(-EINVAL); 14076 } 14077 14078 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link); 14079 if (err) 14080 return libbpf_err_ptr(err); 14081 14082 /* When calling bpf_program__attach() explicitly, auto-attach support 14083 * is expected to work, so NULL returned link is considered an error. 14084 * This is different for skeleton's attach, see comment in 14085 * bpf_object__attach_skeleton(). 14086 */ 14087 if (!link) 14088 return libbpf_err_ptr(-EOPNOTSUPP); 14089 14090 return link; 14091 } 14092 14093 struct bpf_link_struct_ops { 14094 struct bpf_link link; 14095 int map_fd; 14096 }; 14097 14098 static int bpf_link__detach_struct_ops(struct bpf_link *link) 14099 { 14100 struct bpf_link_struct_ops *st_link; 14101 __u32 zero = 0; 14102 14103 st_link = container_of(link, struct bpf_link_struct_ops, link); 14104 14105 if (st_link->map_fd < 0) 14106 /* w/o a real link */ 14107 return bpf_map_delete_elem(link->fd, &zero); 14108 14109 return close(link->fd); 14110 } 14111 14112 struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map) 14113 { 14114 struct bpf_link_struct_ops *link; 14115 __u32 zero = 0; 14116 int err, fd; 14117 14118 if (!bpf_map__is_struct_ops(map)) { 14119 pr_warn("map '%s': can't attach non-struct_ops map\n", map->name); 14120 return libbpf_err_ptr(-EINVAL); 14121 } 14122 14123 if (map->fd < 0) { 14124 pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name); 14125 return libbpf_err_ptr(-EINVAL); 14126 } 14127 14128 link = calloc(1, sizeof(*link)); 14129 if (!link) 14130 return libbpf_err_ptr(-EINVAL); 14131 14132 /* kern_vdata should be prepared during the loading phase. */ 14133 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); 14134 /* It can be EBUSY if the map has been used to create or 14135 * update a link before. We don't allow updating the value of 14136 * a struct_ops once it is set. That ensures that the value 14137 * never changed. So, it is safe to skip EBUSY. 14138 */ 14139 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) { 14140 free(link); 14141 return libbpf_err_ptr(err); 14142 } 14143 14144 link->link.detach = bpf_link__detach_struct_ops; 14145 14146 if (!(map->def.map_flags & BPF_F_LINK)) { 14147 /* w/o a real link */ 14148 link->link.fd = map->fd; 14149 link->map_fd = -1; 14150 return &link->link; 14151 } 14152 14153 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL); 14154 if (fd < 0) { 14155 free(link); 14156 return libbpf_err_ptr(fd); 14157 } 14158 14159 link->link.fd = fd; 14160 link->map_fd = map->fd; 14161 14162 return &link->link; 14163 } 14164 14165 /* 14166 * Swap the back struct_ops of a link with a new struct_ops map. 14167 */ 14168 int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map) 14169 { 14170 struct bpf_link_struct_ops *st_ops_link; 14171 __u32 zero = 0; 14172 int err; 14173 14174 if (!bpf_map__is_struct_ops(map)) 14175 return libbpf_err(-EINVAL); 14176 14177 if (map->fd < 0) { 14178 pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name); 14179 return libbpf_err(-EINVAL); 14180 } 14181 14182 st_ops_link = container_of(link, struct bpf_link_struct_ops, link); 14183 /* Ensure the type of a link is correct */ 14184 if (st_ops_link->map_fd < 0) 14185 return libbpf_err(-EINVAL); 14186 14187 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); 14188 /* It can be EBUSY if the map has been used to create or 14189 * update a link before. We don't allow updating the value of 14190 * a struct_ops once it is set. That ensures that the value 14191 * never changed. So, it is safe to skip EBUSY. 14192 */ 14193 if (err && err != -EBUSY) 14194 return err; 14195 14196 err = bpf_link_update(link->fd, map->fd, NULL); 14197 if (err < 0) 14198 return err; 14199 14200 st_ops_link->map_fd = map->fd; 14201 14202 return 0; 14203 } 14204 14205 typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr, 14206 void *private_data); 14207 14208 static enum bpf_perf_event_ret 14209 perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 14210 void **copy_mem, size_t *copy_size, 14211 bpf_perf_event_print_t fn, void *private_data) 14212 { 14213 struct perf_event_mmap_page *header = mmap_mem; 14214 __u64 data_head = ring_buffer_read_head(header); 14215 __u64 data_tail = header->data_tail; 14216 void *base = ((__u8 *)header) + page_size; 14217 int ret = LIBBPF_PERF_EVENT_CONT; 14218 struct perf_event_header *ehdr; 14219 size_t ehdr_size; 14220 14221 while (data_head != data_tail) { 14222 ehdr = base + (data_tail & (mmap_size - 1)); 14223 ehdr_size = ehdr->size; 14224 14225 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 14226 void *copy_start = ehdr; 14227 size_t len_first = base + mmap_size - copy_start; 14228 size_t len_second = ehdr_size - len_first; 14229 14230 if (*copy_size < ehdr_size) { 14231 free(*copy_mem); 14232 *copy_mem = malloc(ehdr_size); 14233 if (!*copy_mem) { 14234 *copy_size = 0; 14235 ret = LIBBPF_PERF_EVENT_ERROR; 14236 break; 14237 } 14238 *copy_size = ehdr_size; 14239 } 14240 14241 memcpy(*copy_mem, copy_start, len_first); 14242 memcpy(*copy_mem + len_first, base, len_second); 14243 ehdr = *copy_mem; 14244 } 14245 14246 ret = fn(ehdr, private_data); 14247 data_tail += ehdr_size; 14248 if (ret != LIBBPF_PERF_EVENT_CONT) 14249 break; 14250 } 14251 14252 ring_buffer_write_tail(header, data_tail); 14253 return libbpf_err(ret); 14254 } 14255 14256 struct perf_buffer; 14257 14258 struct perf_buffer_params { 14259 struct perf_event_attr *attr; 14260 /* if event_cb is specified, it takes precedence */ 14261 perf_buffer_event_fn event_cb; 14262 /* sample_cb and lost_cb are higher-level common-case callbacks */ 14263 perf_buffer_sample_fn sample_cb; 14264 perf_buffer_lost_fn lost_cb; 14265 void *ctx; 14266 int cpu_cnt; 14267 int *cpus; 14268 int *map_keys; 14269 }; 14270 14271 struct perf_cpu_buf { 14272 struct perf_buffer *pb; 14273 void *base; /* mmap()'ed memory */ 14274 void *buf; /* for reconstructing segmented data */ 14275 size_t buf_size; 14276 int fd; 14277 int cpu; 14278 int map_key; 14279 }; 14280 14281 struct perf_buffer { 14282 perf_buffer_event_fn event_cb; 14283 perf_buffer_sample_fn sample_cb; 14284 perf_buffer_lost_fn lost_cb; 14285 void *ctx; /* passed into callbacks */ 14286 14287 size_t page_size; 14288 size_t mmap_size; 14289 struct perf_cpu_buf **cpu_bufs; 14290 struct epoll_event *events; 14291 int cpu_cnt; /* number of allocated CPU buffers */ 14292 int epoll_fd; /* perf event FD */ 14293 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 14294 }; 14295 14296 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 14297 struct perf_cpu_buf *cpu_buf) 14298 { 14299 if (!cpu_buf) 14300 return; 14301 if (cpu_buf->base && 14302 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 14303 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 14304 if (cpu_buf->fd >= 0) { 14305 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 14306 close(cpu_buf->fd); 14307 } 14308 free(cpu_buf->buf); 14309 free(cpu_buf); 14310 } 14311 14312 void perf_buffer__free(struct perf_buffer *pb) 14313 { 14314 int i; 14315 14316 if (IS_ERR_OR_NULL(pb)) 14317 return; 14318 if (pb->cpu_bufs) { 14319 for (i = 0; i < pb->cpu_cnt; i++) { 14320 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 14321 14322 if (!cpu_buf) 14323 continue; 14324 14325 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 14326 perf_buffer__free_cpu_buf(pb, cpu_buf); 14327 } 14328 free(pb->cpu_bufs); 14329 } 14330 if (pb->epoll_fd >= 0) 14331 close(pb->epoll_fd); 14332 free(pb->events); 14333 free(pb); 14334 } 14335 14336 static struct perf_cpu_buf * 14337 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 14338 int cpu, int map_key) 14339 { 14340 struct perf_cpu_buf *cpu_buf; 14341 int err; 14342 14343 cpu_buf = calloc(1, sizeof(*cpu_buf)); 14344 if (!cpu_buf) 14345 return ERR_PTR(-ENOMEM); 14346 14347 cpu_buf->pb = pb; 14348 cpu_buf->cpu = cpu; 14349 cpu_buf->map_key = map_key; 14350 14351 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 14352 -1, PERF_FLAG_FD_CLOEXEC); 14353 if (cpu_buf->fd < 0) { 14354 err = -errno; 14355 pr_warn("failed to open perf buffer event on cpu #%d: %s\n", 14356 cpu, errstr(err)); 14357 goto error; 14358 } 14359 14360 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 14361 PROT_READ | PROT_WRITE, MAP_SHARED, 14362 cpu_buf->fd, 0); 14363 if (cpu_buf->base == MAP_FAILED) { 14364 cpu_buf->base = NULL; 14365 err = -errno; 14366 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", 14367 cpu, errstr(err)); 14368 goto error; 14369 } 14370 14371 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 14372 err = -errno; 14373 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", 14374 cpu, errstr(err)); 14375 goto error; 14376 } 14377 14378 return cpu_buf; 14379 14380 error: 14381 perf_buffer__free_cpu_buf(pb, cpu_buf); 14382 return (struct perf_cpu_buf *)ERR_PTR(err); 14383 } 14384 14385 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 14386 struct perf_buffer_params *p); 14387 14388 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 14389 perf_buffer_sample_fn sample_cb, 14390 perf_buffer_lost_fn lost_cb, 14391 void *ctx, 14392 const struct perf_buffer_opts *opts) 14393 { 14394 const size_t attr_sz = sizeof(struct perf_event_attr); 14395 struct perf_buffer_params p = {}; 14396 struct perf_event_attr attr; 14397 __u32 sample_period; 14398 14399 if (!OPTS_VALID(opts, perf_buffer_opts)) 14400 return libbpf_err_ptr(-EINVAL); 14401 14402 sample_period = OPTS_GET(opts, sample_period, 1); 14403 if (!sample_period) 14404 sample_period = 1; 14405 14406 memset(&attr, 0, attr_sz); 14407 attr.size = attr_sz; 14408 attr.config = PERF_COUNT_SW_BPF_OUTPUT; 14409 attr.type = PERF_TYPE_SOFTWARE; 14410 attr.sample_type = PERF_SAMPLE_RAW; 14411 attr.wakeup_events = sample_period; 14412 14413 p.attr = &attr; 14414 p.sample_cb = sample_cb; 14415 p.lost_cb = lost_cb; 14416 p.ctx = ctx; 14417 14418 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 14419 } 14420 14421 struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt, 14422 struct perf_event_attr *attr, 14423 perf_buffer_event_fn event_cb, void *ctx, 14424 const struct perf_buffer_raw_opts *opts) 14425 { 14426 struct perf_buffer_params p = {}; 14427 14428 if (!attr) 14429 return libbpf_err_ptr(-EINVAL); 14430 14431 if (!OPTS_VALID(opts, perf_buffer_raw_opts)) 14432 return libbpf_err_ptr(-EINVAL); 14433 14434 p.attr = attr; 14435 p.event_cb = event_cb; 14436 p.ctx = ctx; 14437 p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0); 14438 p.cpus = OPTS_GET(opts, cpus, NULL); 14439 p.map_keys = OPTS_GET(opts, map_keys, NULL); 14440 14441 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 14442 } 14443 14444 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 14445 struct perf_buffer_params *p) 14446 { 14447 const char *online_cpus_file = "/sys/devices/system/cpu/online"; 14448 struct bpf_map_info map; 14449 struct perf_buffer *pb; 14450 bool *online = NULL; 14451 __u32 map_info_len; 14452 int err, i, j, n; 14453 14454 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) { 14455 pr_warn("page count should be power of two, but is %zu\n", 14456 page_cnt); 14457 return ERR_PTR(-EINVAL); 14458 } 14459 14460 /* best-effort sanity checks */ 14461 memset(&map, 0, sizeof(map)); 14462 map_info_len = sizeof(map); 14463 err = bpf_map_get_info_by_fd(map_fd, &map, &map_info_len); 14464 if (err) { 14465 err = -errno; 14466 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return 14467 * -EBADFD, -EFAULT, or -E2BIG on real error 14468 */ 14469 if (err != -EINVAL) { 14470 pr_warn("failed to get map info for map FD %d: %s\n", 14471 map_fd, errstr(err)); 14472 return ERR_PTR(err); 14473 } 14474 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n", 14475 map_fd); 14476 } else { 14477 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 14478 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 14479 map.name); 14480 return ERR_PTR(-EINVAL); 14481 } 14482 } 14483 14484 pb = calloc(1, sizeof(*pb)); 14485 if (!pb) 14486 return ERR_PTR(-ENOMEM); 14487 14488 pb->event_cb = p->event_cb; 14489 pb->sample_cb = p->sample_cb; 14490 pb->lost_cb = p->lost_cb; 14491 pb->ctx = p->ctx; 14492 14493 pb->page_size = getpagesize(); 14494 pb->mmap_size = pb->page_size * page_cnt; 14495 pb->map_fd = map_fd; 14496 14497 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 14498 if (pb->epoll_fd < 0) { 14499 err = -errno; 14500 pr_warn("failed to create epoll instance: %s\n", 14501 errstr(err)); 14502 goto error; 14503 } 14504 14505 if (p->cpu_cnt > 0) { 14506 pb->cpu_cnt = p->cpu_cnt; 14507 } else { 14508 pb->cpu_cnt = libbpf_num_possible_cpus(); 14509 if (pb->cpu_cnt < 0) { 14510 err = pb->cpu_cnt; 14511 goto error; 14512 } 14513 if (map.max_entries && map.max_entries < pb->cpu_cnt) 14514 pb->cpu_cnt = map.max_entries; 14515 } 14516 14517 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 14518 if (!pb->events) { 14519 err = -ENOMEM; 14520 pr_warn("failed to allocate events: out of memory\n"); 14521 goto error; 14522 } 14523 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 14524 if (!pb->cpu_bufs) { 14525 err = -ENOMEM; 14526 pr_warn("failed to allocate buffers: out of memory\n"); 14527 goto error; 14528 } 14529 14530 err = parse_cpu_mask_file(online_cpus_file, &online, &n); 14531 if (err) { 14532 pr_warn("failed to get online CPU mask: %s\n", errstr(err)); 14533 goto error; 14534 } 14535 14536 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { 14537 struct perf_cpu_buf *cpu_buf; 14538 int cpu, map_key; 14539 14540 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 14541 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 14542 14543 /* in case user didn't explicitly requested particular CPUs to 14544 * be attached to, skip offline/not present CPUs 14545 */ 14546 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) 14547 continue; 14548 14549 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 14550 if (IS_ERR(cpu_buf)) { 14551 err = PTR_ERR(cpu_buf); 14552 goto error; 14553 } 14554 14555 pb->cpu_bufs[j] = cpu_buf; 14556 14557 err = bpf_map_update_elem(pb->map_fd, &map_key, 14558 &cpu_buf->fd, 0); 14559 if (err) { 14560 err = -errno; 14561 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 14562 cpu, map_key, cpu_buf->fd, 14563 errstr(err)); 14564 goto error; 14565 } 14566 14567 pb->events[j].events = EPOLLIN; 14568 pb->events[j].data.ptr = cpu_buf; 14569 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 14570 &pb->events[j]) < 0) { 14571 err = -errno; 14572 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 14573 cpu, cpu_buf->fd, 14574 errstr(err)); 14575 goto error; 14576 } 14577 j++; 14578 } 14579 pb->cpu_cnt = j; 14580 free(online); 14581 14582 return pb; 14583 14584 error: 14585 free(online); 14586 if (pb) 14587 perf_buffer__free(pb); 14588 return ERR_PTR(err); 14589 } 14590 14591 struct perf_sample_raw { 14592 struct perf_event_header header; 14593 uint32_t size; 14594 char data[]; 14595 }; 14596 14597 struct perf_sample_lost { 14598 struct perf_event_header header; 14599 uint64_t id; 14600 uint64_t lost; 14601 uint64_t sample_id; 14602 }; 14603 14604 static enum bpf_perf_event_ret 14605 perf_buffer__process_record(struct perf_event_header *e, void *ctx) 14606 { 14607 struct perf_cpu_buf *cpu_buf = ctx; 14608 struct perf_buffer *pb = cpu_buf->pb; 14609 void *data = e; 14610 14611 /* user wants full control over parsing perf event */ 14612 if (pb->event_cb) 14613 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 14614 14615 switch (e->type) { 14616 case PERF_RECORD_SAMPLE: { 14617 struct perf_sample_raw *s = data; 14618 14619 if (pb->sample_cb) 14620 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 14621 break; 14622 } 14623 case PERF_RECORD_LOST: { 14624 struct perf_sample_lost *s = data; 14625 14626 if (pb->lost_cb) 14627 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 14628 break; 14629 } 14630 default: 14631 pr_warn("unknown perf sample type %u\n", e->type); 14632 return LIBBPF_PERF_EVENT_ERROR; 14633 } 14634 return LIBBPF_PERF_EVENT_CONT; 14635 } 14636 14637 static int perf_buffer__process_records(struct perf_buffer *pb, 14638 struct perf_cpu_buf *cpu_buf) 14639 { 14640 enum bpf_perf_event_ret ret; 14641 14642 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size, 14643 pb->page_size, &cpu_buf->buf, 14644 &cpu_buf->buf_size, 14645 perf_buffer__process_record, cpu_buf); 14646 if (ret != LIBBPF_PERF_EVENT_CONT) 14647 return ret; 14648 return 0; 14649 } 14650 14651 int perf_buffer__epoll_fd(const struct perf_buffer *pb) 14652 { 14653 return pb->epoll_fd; 14654 } 14655 14656 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 14657 { 14658 int i, cnt, err; 14659 14660 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 14661 if (cnt < 0) 14662 return -errno; 14663 14664 for (i = 0; i < cnt; i++) { 14665 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 14666 14667 err = perf_buffer__process_records(pb, cpu_buf); 14668 if (err) { 14669 pr_warn("error while processing records: %s\n", errstr(err)); 14670 return libbpf_err(err); 14671 } 14672 } 14673 return cnt; 14674 } 14675 14676 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer 14677 * manager. 14678 */ 14679 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb) 14680 { 14681 return pb->cpu_cnt; 14682 } 14683 14684 /* 14685 * Return perf_event FD of a ring buffer in *buf_idx* slot of 14686 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using 14687 * select()/poll()/epoll() Linux syscalls. 14688 */ 14689 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx) 14690 { 14691 struct perf_cpu_buf *cpu_buf; 14692 14693 if (buf_idx >= pb->cpu_cnt) 14694 return libbpf_err(-EINVAL); 14695 14696 cpu_buf = pb->cpu_bufs[buf_idx]; 14697 if (!cpu_buf) 14698 return libbpf_err(-ENOENT); 14699 14700 return cpu_buf->fd; 14701 } 14702 14703 int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size) 14704 { 14705 struct perf_cpu_buf *cpu_buf; 14706 14707 if (buf_idx >= pb->cpu_cnt) 14708 return libbpf_err(-EINVAL); 14709 14710 cpu_buf = pb->cpu_bufs[buf_idx]; 14711 if (!cpu_buf) 14712 return libbpf_err(-ENOENT); 14713 14714 *buf = cpu_buf->base; 14715 *buf_size = pb->mmap_size; 14716 return 0; 14717 } 14718 14719 /* 14720 * Consume data from perf ring buffer corresponding to slot *buf_idx* in 14721 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to 14722 * consume, do nothing and return success. 14723 * Returns: 14724 * - 0 on success; 14725 * - <0 on failure. 14726 */ 14727 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx) 14728 { 14729 struct perf_cpu_buf *cpu_buf; 14730 14731 if (buf_idx >= pb->cpu_cnt) 14732 return libbpf_err(-EINVAL); 14733 14734 cpu_buf = pb->cpu_bufs[buf_idx]; 14735 if (!cpu_buf) 14736 return libbpf_err(-ENOENT); 14737 14738 return perf_buffer__process_records(pb, cpu_buf); 14739 } 14740 14741 int perf_buffer__consume(struct perf_buffer *pb) 14742 { 14743 int i, err; 14744 14745 for (i = 0; i < pb->cpu_cnt; i++) { 14746 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 14747 14748 if (!cpu_buf) 14749 continue; 14750 14751 err = perf_buffer__process_records(pb, cpu_buf); 14752 if (err) { 14753 pr_warn("perf_buffer: failed to process records in buffer #%d: %s\n", 14754 i, errstr(err)); 14755 return libbpf_err(err); 14756 } 14757 } 14758 return 0; 14759 } 14760 14761 int bpf_program__set_attach_target(struct bpf_program *prog, 14762 int attach_prog_fd, 14763 const char *attach_func_name) 14764 { 14765 int btf_obj_fd = 0, btf_id = 0, err; 14766 14767 if (!prog || attach_prog_fd < 0) 14768 return libbpf_err(-EINVAL); 14769 14770 if (prog->obj->state >= OBJ_LOADED) 14771 return libbpf_err(-EINVAL); 14772 14773 if (attach_prog_fd && !attach_func_name) { 14774 /* Store attach_prog_fd. The BTF ID will be resolved later during 14775 * the normal object/program load phase. 14776 */ 14777 prog->attach_prog_fd = attach_prog_fd; 14778 return 0; 14779 } 14780 14781 if (attach_prog_fd) { 14782 btf_id = libbpf_find_prog_btf_id(attach_func_name, 14783 attach_prog_fd, prog->obj->token_fd); 14784 if (btf_id < 0) 14785 return libbpf_err(btf_id); 14786 } else { 14787 if (!attach_func_name) 14788 return libbpf_err(-EINVAL); 14789 14790 /* load btf_vmlinux, if not yet */ 14791 err = bpf_object__load_vmlinux_btf(prog->obj, true); 14792 if (err) 14793 return libbpf_err(err); 14794 err = find_kernel_btf_id(prog->obj, attach_func_name, 14795 prog->expected_attach_type, 14796 &btf_obj_fd, &btf_id); 14797 if (err) 14798 return libbpf_err(err); 14799 } 14800 14801 prog->attach_btf_id = btf_id; 14802 prog->attach_btf_obj_fd = btf_obj_fd; 14803 prog->attach_prog_fd = attach_prog_fd; 14804 return 0; 14805 } 14806 14807 int bpf_program__assoc_struct_ops(struct bpf_program *prog, struct bpf_map *map, 14808 struct bpf_prog_assoc_struct_ops_opts *opts) 14809 { 14810 int prog_fd, map_fd; 14811 14812 prog_fd = bpf_program__fd(prog); 14813 if (prog_fd < 0) { 14814 pr_warn("prog '%s': can't associate BPF program without FD (was it loaded?)\n", 14815 prog->name); 14816 return libbpf_err(-EINVAL); 14817 } 14818 14819 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) { 14820 pr_warn("prog '%s': can't associate struct_ops program\n", prog->name); 14821 return libbpf_err(-EINVAL); 14822 } 14823 14824 map_fd = bpf_map__fd(map); 14825 if (map_fd < 0) { 14826 pr_warn("map '%s': can't associate BPF map without FD (was it created?)\n", map->name); 14827 return libbpf_err(-EINVAL); 14828 } 14829 14830 if (!bpf_map__is_struct_ops(map)) { 14831 pr_warn("map '%s': can't associate non-struct_ops map\n", map->name); 14832 return libbpf_err(-EINVAL); 14833 } 14834 14835 return bpf_prog_assoc_struct_ops(prog_fd, map_fd, opts); 14836 } 14837 14838 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) 14839 { 14840 int err = 0, n, len, start, end = -1; 14841 bool *tmp; 14842 14843 *mask = NULL; 14844 *mask_sz = 0; 14845 14846 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 14847 while (*s) { 14848 if (*s == ',' || *s == '\n') { 14849 s++; 14850 continue; 14851 } 14852 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); 14853 if (n <= 0 || n > 2) { 14854 pr_warn("Failed to get CPU range %s: %d\n", s, n); 14855 err = -EINVAL; 14856 goto cleanup; 14857 } else if (n == 1) { 14858 end = start; 14859 } 14860 if (start < 0 || start > end) { 14861 pr_warn("Invalid CPU range [%d,%d] in %s\n", 14862 start, end, s); 14863 err = -EINVAL; 14864 goto cleanup; 14865 } 14866 tmp = realloc(*mask, end + 1); 14867 if (!tmp) { 14868 err = -ENOMEM; 14869 goto cleanup; 14870 } 14871 *mask = tmp; 14872 memset(tmp + *mask_sz, 0, start - *mask_sz); 14873 memset(tmp + start, 1, end - start + 1); 14874 *mask_sz = end + 1; 14875 s += len; 14876 } 14877 if (!*mask_sz) { 14878 pr_warn("Empty CPU range\n"); 14879 return -EINVAL; 14880 } 14881 return 0; 14882 cleanup: 14883 free(*mask); 14884 *mask = NULL; 14885 return err; 14886 } 14887 14888 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) 14889 { 14890 int fd, err = 0, len; 14891 char buf[128]; 14892 14893 fd = open(fcpu, O_RDONLY | O_CLOEXEC); 14894 if (fd < 0) { 14895 err = -errno; 14896 pr_warn("Failed to open cpu mask file %s: %s\n", fcpu, errstr(err)); 14897 return err; 14898 } 14899 len = read(fd, buf, sizeof(buf)); 14900 close(fd); 14901 if (len <= 0) { 14902 err = len ? -errno : -EINVAL; 14903 pr_warn("Failed to read cpu mask from %s: %s\n", fcpu, errstr(err)); 14904 return err; 14905 } 14906 if (len >= sizeof(buf)) { 14907 pr_warn("CPU mask is too big in file %s\n", fcpu); 14908 return -E2BIG; 14909 } 14910 buf[len] = '\0'; 14911 14912 return parse_cpu_mask_str(buf, mask, mask_sz); 14913 } 14914 14915 int libbpf_num_possible_cpus(void) 14916 { 14917 static const char *fcpu = "/sys/devices/system/cpu/possible"; 14918 static int cpus; 14919 int err, n, i, tmp_cpus; 14920 bool *mask; 14921 14922 tmp_cpus = READ_ONCE(cpus); 14923 if (tmp_cpus > 0) 14924 return tmp_cpus; 14925 14926 err = parse_cpu_mask_file(fcpu, &mask, &n); 14927 if (err) 14928 return libbpf_err(err); 14929 14930 tmp_cpus = 0; 14931 for (i = 0; i < n; i++) { 14932 if (mask[i]) 14933 tmp_cpus++; 14934 } 14935 free(mask); 14936 14937 WRITE_ONCE(cpus, tmp_cpus); 14938 return tmp_cpus; 14939 } 14940 14941 static int populate_skeleton_maps(const struct bpf_object *obj, 14942 struct bpf_map_skeleton *maps, 14943 size_t map_cnt, size_t map_skel_sz) 14944 { 14945 int i; 14946 14947 for (i = 0; i < map_cnt; i++) { 14948 struct bpf_map_skeleton *map_skel = (void *)maps + i * map_skel_sz; 14949 struct bpf_map **map = map_skel->map; 14950 const char *name = map_skel->name; 14951 void **mmaped = map_skel->mmaped; 14952 14953 *map = bpf_object__find_map_by_name(obj, name); 14954 if (!*map) { 14955 pr_warn("failed to find skeleton map '%s'\n", name); 14956 return -ESRCH; 14957 } 14958 14959 /* externs shouldn't be pre-setup from user code */ 14960 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) 14961 *mmaped = (*map)->mmaped; 14962 } 14963 return 0; 14964 } 14965 14966 static int populate_skeleton_progs(const struct bpf_object *obj, 14967 struct bpf_prog_skeleton *progs, 14968 size_t prog_cnt, size_t prog_skel_sz) 14969 { 14970 int i; 14971 14972 for (i = 0; i < prog_cnt; i++) { 14973 struct bpf_prog_skeleton *prog_skel = (void *)progs + i * prog_skel_sz; 14974 struct bpf_program **prog = prog_skel->prog; 14975 const char *name = prog_skel->name; 14976 14977 *prog = bpf_object__find_program_by_name(obj, name); 14978 if (!*prog) { 14979 pr_warn("failed to find skeleton program '%s'\n", name); 14980 return -ESRCH; 14981 } 14982 } 14983 return 0; 14984 } 14985 14986 int bpf_object__open_skeleton(struct bpf_object_skeleton *s, 14987 const struct bpf_object_open_opts *opts) 14988 { 14989 struct bpf_object *obj; 14990 int err; 14991 14992 obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts); 14993 if (IS_ERR(obj)) { 14994 err = PTR_ERR(obj); 14995 pr_warn("failed to initialize skeleton BPF object '%s': %s\n", 14996 s->name, errstr(err)); 14997 return libbpf_err(err); 14998 } 14999 15000 *s->obj = obj; 15001 err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz); 15002 if (err) { 15003 pr_warn("failed to populate skeleton maps for '%s': %s\n", s->name, errstr(err)); 15004 return libbpf_err(err); 15005 } 15006 15007 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz); 15008 if (err) { 15009 pr_warn("failed to populate skeleton progs for '%s': %s\n", s->name, errstr(err)); 15010 return libbpf_err(err); 15011 } 15012 15013 return 0; 15014 } 15015 15016 int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s) 15017 { 15018 int err, len, var_idx, i; 15019 const char *var_name; 15020 const struct bpf_map *map; 15021 struct btf *btf; 15022 __u32 map_type_id; 15023 const struct btf_type *map_type, *var_type; 15024 const struct bpf_var_skeleton *var_skel; 15025 struct btf_var_secinfo *var; 15026 15027 if (!s->obj) 15028 return libbpf_err(-EINVAL); 15029 15030 btf = bpf_object__btf(s->obj); 15031 if (!btf) { 15032 pr_warn("subskeletons require BTF at runtime (object %s)\n", 15033 bpf_object__name(s->obj)); 15034 return libbpf_err(-errno); 15035 } 15036 15037 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz); 15038 if (err) { 15039 pr_warn("failed to populate subskeleton maps: %s\n", errstr(err)); 15040 return libbpf_err(err); 15041 } 15042 15043 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz); 15044 if (err) { 15045 pr_warn("failed to populate subskeleton maps: %s\n", errstr(err)); 15046 return libbpf_err(err); 15047 } 15048 15049 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) { 15050 var_skel = (void *)s->vars + var_idx * s->var_skel_sz; 15051 map = *var_skel->map; 15052 map_type_id = bpf_map__btf_value_type_id(map); 15053 map_type = btf__type_by_id(btf, map_type_id); 15054 15055 if (!btf_is_datasec(map_type)) { 15056 pr_warn("type for map '%1$s' is not a datasec: %2$s\n", 15057 bpf_map__name(map), 15058 __btf_kind_str(btf_kind(map_type))); 15059 return libbpf_err(-EINVAL); 15060 } 15061 15062 len = btf_vlen(map_type); 15063 var = btf_var_secinfos(map_type); 15064 for (i = 0; i < len; i++, var++) { 15065 var_type = btf__type_by_id(btf, var->type); 15066 var_name = btf__name_by_offset(btf, var_type->name_off); 15067 if (strcmp(var_name, var_skel->name) == 0) { 15068 *var_skel->addr = map->mmaped + var->offset; 15069 break; 15070 } 15071 } 15072 } 15073 return 0; 15074 } 15075 15076 void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s) 15077 { 15078 if (!s) 15079 return; 15080 free(s->maps); 15081 free(s->progs); 15082 free(s->vars); 15083 free(s); 15084 } 15085 15086 int bpf_object__load_skeleton(struct bpf_object_skeleton *s) 15087 { 15088 int i, err; 15089 15090 err = bpf_object__load(*s->obj); 15091 if (err) { 15092 pr_warn("failed to load BPF skeleton '%s': %s\n", s->name, errstr(err)); 15093 return libbpf_err(err); 15094 } 15095 15096 for (i = 0; i < s->map_cnt; i++) { 15097 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz; 15098 struct bpf_map *map = *map_skel->map; 15099 15100 if (!map_skel->mmaped) 15101 continue; 15102 15103 if (map->def.type == BPF_MAP_TYPE_ARENA) 15104 *map_skel->mmaped = map->mmaped + map->obj->arena_data_off; 15105 else 15106 *map_skel->mmaped = map->mmaped; 15107 } 15108 15109 return 0; 15110 } 15111 15112 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) 15113 { 15114 int i, err; 15115 15116 for (i = 0; i < s->prog_cnt; i++) { 15117 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz; 15118 struct bpf_program *prog = *prog_skel->prog; 15119 struct bpf_link **link = prog_skel->link; 15120 15121 if (!prog->autoload || !prog->autoattach) 15122 continue; 15123 15124 /* auto-attaching not supported for this program */ 15125 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) 15126 continue; 15127 15128 /* if user already set the link manually, don't attempt auto-attach */ 15129 if (*link) 15130 continue; 15131 15132 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link); 15133 if (err) { 15134 pr_warn("prog '%s': failed to auto-attach: %s\n", 15135 bpf_program__name(prog), errstr(err)); 15136 return libbpf_err(err); 15137 } 15138 15139 /* It's possible that for some SEC() definitions auto-attach 15140 * is supported in some cases (e.g., if definition completely 15141 * specifies target information), but is not in other cases. 15142 * SEC("uprobe") is one such case. If user specified target 15143 * binary and function name, such BPF program can be 15144 * auto-attached. But if not, it shouldn't trigger skeleton's 15145 * attach to fail. It should just be skipped. 15146 * attach_fn signals such case with returning 0 (no error) and 15147 * setting link to NULL. 15148 */ 15149 } 15150 15151 15152 for (i = 0; i < s->map_cnt; i++) { 15153 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz; 15154 struct bpf_map *map = *map_skel->map; 15155 struct bpf_link **link; 15156 15157 if (!map->autocreate || !map->autoattach) 15158 continue; 15159 15160 /* only struct_ops maps can be attached */ 15161 if (!bpf_map__is_struct_ops(map)) 15162 continue; 15163 15164 /* skeleton is created with earlier version of bpftool, notify user */ 15165 if (s->map_skel_sz < offsetofend(struct bpf_map_skeleton, link)) { 15166 pr_warn("map '%s': BPF skeleton version is old, skipping map auto-attachment...\n", 15167 bpf_map__name(map)); 15168 continue; 15169 } 15170 15171 link = map_skel->link; 15172 if (!link) { 15173 pr_warn("map '%s': BPF map skeleton link is uninitialized\n", 15174 bpf_map__name(map)); 15175 continue; 15176 } 15177 15178 if (*link) 15179 continue; 15180 15181 *link = bpf_map__attach_struct_ops(map); 15182 if (!*link) { 15183 err = -errno; 15184 pr_warn("map '%s': failed to auto-attach: %s\n", 15185 bpf_map__name(map), errstr(err)); 15186 return libbpf_err(err); 15187 } 15188 } 15189 15190 return 0; 15191 } 15192 15193 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) 15194 { 15195 int i; 15196 15197 for (i = 0; i < s->prog_cnt; i++) { 15198 struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz; 15199 struct bpf_link **link = prog_skel->link; 15200 15201 bpf_link__destroy(*link); 15202 *link = NULL; 15203 } 15204 15205 if (s->map_skel_sz < sizeof(struct bpf_map_skeleton)) 15206 return; 15207 15208 for (i = 0; i < s->map_cnt; i++) { 15209 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz; 15210 struct bpf_link **link = map_skel->link; 15211 15212 if (link) { 15213 bpf_link__destroy(*link); 15214 *link = NULL; 15215 } 15216 } 15217 } 15218 15219 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) 15220 { 15221 if (!s) 15222 return; 15223 15224 bpf_object__detach_skeleton(s); 15225 if (s->obj) 15226 bpf_object__close(*s->obj); 15227 free(s->maps); 15228 free(s->progs); 15229 free(s); 15230 } 15231