1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ 2 /* Copyright (c) 2021 Facebook */ 3 #ifndef __SKEL_INTERNAL_H 4 #define __SKEL_INTERNAL_H 5 6 #ifdef __KERNEL__ 7 #include <linux/fdtable.h> 8 #include <linux/mm.h> 9 #include <linux/mman.h> 10 #include <linux/slab.h> 11 #include <linux/bpf.h> 12 #else 13 #include <unistd.h> 14 #include <sys/syscall.h> 15 #include <sys/mman.h> 16 #include <linux/keyctl.h> 17 #include <stdlib.h> 18 #include "bpf.h" 19 #endif 20 21 #ifndef __NR_bpf 22 # if defined(__mips__) && defined(_ABIO32) 23 # define __NR_bpf 4355 24 # elif defined(__mips__) && defined(_ABIN32) 25 # define __NR_bpf 6319 26 # elif defined(__mips__) && defined(_ABI64) 27 # define __NR_bpf 5315 28 # endif 29 #endif 30 31 /* This file is a base header for auto-generated *.lskel.h files. 32 * Its contents will change and may become part of auto-generation in the future. 33 * 34 * The layout of bpf_[map|prog]_desc and bpf_loader_ctx is feature dependent 35 * and will change from one version of libbpf to another and features 36 * requested during loader program generation. 37 */ 38 struct bpf_map_desc { 39 /* output of the loader prog */ 40 int map_fd; 41 /* input for the loader prog */ 42 __u32 max_entries; 43 __aligned_u64 initial_value; 44 }; 45 struct bpf_prog_desc { 46 int prog_fd; 47 }; 48 49 enum { 50 BPF_SKEL_KERNEL = (1ULL << 0), 51 }; 52 53 struct bpf_loader_ctx { 54 __u32 sz; 55 __u32 flags; 56 __u32 log_level; 57 __u32 log_size; 58 __u64 log_buf; 59 }; 60 61 struct bpf_load_and_run_opts { 62 struct bpf_loader_ctx *ctx; 63 const void *data; 64 const void *insns; 65 __u32 data_sz; 66 __u32 insns_sz; 67 const char *errstr; 68 void *signature; 69 __u32 signature_sz; 70 __s32 keyring_id; 71 void *excl_prog_hash; 72 __u32 excl_prog_hash_sz; 73 }; 74 75 long kern_sys_bpf(__u32 cmd, void *attr, __u32 attr_size); 76 77 static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, 78 unsigned int size) 79 { 80 #ifdef __KERNEL__ 81 return kern_sys_bpf(cmd, attr, size); 82 #else 83 return syscall(__NR_bpf, cmd, attr, size); 84 #endif 85 } 86 87 #ifdef __KERNEL__ 88 static inline int close(int fd) 89 { 90 return close_fd(fd); 91 } 92 93 static inline void *skel_alloc(size_t size) 94 { 95 struct bpf_loader_ctx *ctx = kzalloc(size, GFP_KERNEL); 96 97 if (!ctx) 98 return NULL; 99 ctx->flags |= BPF_SKEL_KERNEL; 100 return ctx; 101 } 102 103 static inline void skel_free(const void *p) 104 { 105 kfree(p); 106 } 107 108 /* skel->bss/rodata maps are populated the following way: 109 * 110 * For kernel use: 111 * skel_prep_map_data() allocates kernel memory that kernel module can directly access. 112 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. 113 * The loader program will perform probe_read_kernel() from maps.rodata.initial_value. 114 * skel_finalize_map_data() sets skel->rodata to point to actual value in a bpf map and 115 * does maps.rodata.initial_value = ~0ULL to signal skel_free_map_data() that kvfree 116 * is not necessary. 117 * 118 * For user space: 119 * skel_prep_map_data() mmaps anon memory into skel->rodata that can be accessed directly. 120 * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. 121 * The loader program will perform copy_from_user() from maps.rodata.initial_value. 122 * skel_finalize_map_data() remaps bpf array map value from the kernel memory into 123 * skel->rodata address. 124 * 125 * The "bpftool gen skeleton -L" command generates lskel.h that is suitable for 126 * both kernel and user space. The generated loader program does 127 * either bpf_probe_read_kernel() or bpf_copy_from_user() from initial_value 128 * depending on bpf_loader_ctx->flags. 129 */ 130 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) 131 { 132 if (addr != ~0ULL) 133 kvfree(p); 134 /* When addr == ~0ULL the 'p' points to 135 * ((struct bpf_array *)map)->value. See skel_finalize_map_data. 136 */ 137 } 138 139 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) 140 { 141 void *addr; 142 143 addr = kvmalloc(val_sz, GFP_KERNEL); 144 if (!addr) 145 return NULL; 146 memcpy(addr, val, val_sz); 147 return addr; 148 } 149 150 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) 151 { 152 struct bpf_map *map; 153 void *addr = NULL; 154 155 kvfree((void *) (long) *init_val); 156 *init_val = ~0ULL; 157 158 /* At this point bpf_load_and_run() finished without error and 159 * 'fd' is a valid bpf map FD. All sanity checks below should succeed. 160 */ 161 map = bpf_map_get(fd); 162 if (IS_ERR(map)) 163 return NULL; 164 if (map->map_type != BPF_MAP_TYPE_ARRAY) 165 goto out; 166 addr = ((struct bpf_array *)map)->value; 167 /* the addr stays valid, since FD is not closed */ 168 out: 169 bpf_map_put(map); 170 return addr; 171 } 172 173 #else 174 175 static inline void *skel_alloc(size_t size) 176 { 177 return calloc(1, size); 178 } 179 180 static inline void skel_free(void *p) 181 { 182 free(p); 183 } 184 185 static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) 186 { 187 munmap(p, sz); 188 } 189 190 static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) 191 { 192 void *addr; 193 194 addr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 195 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 196 if (addr == (void *) -1) 197 return NULL; 198 memcpy(addr, val, val_sz); 199 return addr; 200 } 201 202 static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) 203 { 204 void *addr; 205 206 addr = mmap((void *) (long) *init_val, mmap_sz, flags, MAP_SHARED | MAP_FIXED, fd, 0); 207 if (addr == (void *) -1) 208 return NULL; 209 return addr; 210 } 211 #endif 212 213 static inline int skel_closenz(int fd) 214 { 215 if (fd > 0) 216 return close(fd); 217 return -EINVAL; 218 } 219 220 #ifndef offsetofend 221 #define offsetofend(TYPE, MEMBER) \ 222 (offsetof(TYPE, MEMBER) + sizeof((((TYPE *)0)->MEMBER))) 223 #endif 224 225 static inline int skel_map_create(enum bpf_map_type map_type, 226 const char *map_name, 227 __u32 key_size, 228 __u32 value_size, 229 __u32 max_entries, 230 const void *excl_prog_hash, 231 __u32 excl_prog_hash_sz) 232 { 233 const size_t attr_sz = offsetofend(union bpf_attr, excl_prog_hash_size); 234 union bpf_attr attr; 235 236 memset(&attr, 0, attr_sz); 237 238 attr.map_type = map_type; 239 attr.excl_prog_hash = (unsigned long) excl_prog_hash; 240 attr.excl_prog_hash_size = excl_prog_hash_sz; 241 242 #ifdef __KERNEL__ 243 if (strscpy(attr.map_name, map_name) < 0) 244 return -EINVAL; 245 #else 246 strncpy(attr.map_name, map_name, sizeof(attr.map_name)); 247 #endif 248 attr.key_size = key_size; 249 attr.value_size = value_size; 250 attr.max_entries = max_entries; 251 252 return skel_sys_bpf(BPF_MAP_CREATE, &attr, attr_sz); 253 } 254 255 static inline int skel_map_update_elem(int fd, const void *key, 256 const void *value, __u64 flags) 257 { 258 const size_t attr_sz = offsetofend(union bpf_attr, flags); 259 union bpf_attr attr; 260 261 memset(&attr, 0, attr_sz); 262 attr.map_fd = fd; 263 attr.key = (long) key; 264 attr.value = (long) value; 265 attr.flags = flags; 266 267 return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); 268 } 269 270 static inline int skel_map_delete_elem(int fd, const void *key) 271 { 272 const size_t attr_sz = offsetofend(union bpf_attr, flags); 273 union bpf_attr attr; 274 275 memset(&attr, 0, attr_sz); 276 attr.map_fd = fd; 277 attr.key = (long)key; 278 279 return skel_sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz); 280 } 281 282 static inline int skel_map_get_fd_by_id(__u32 id) 283 { 284 const size_t attr_sz = offsetofend(union bpf_attr, flags); 285 union bpf_attr attr; 286 287 memset(&attr, 0, attr_sz); 288 attr.map_id = id; 289 290 return skel_sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz); 291 } 292 293 static inline int skel_raw_tracepoint_open(const char *name, int prog_fd) 294 { 295 const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd); 296 union bpf_attr attr; 297 298 memset(&attr, 0, attr_sz); 299 attr.raw_tracepoint.name = (long) name; 300 attr.raw_tracepoint.prog_fd = prog_fd; 301 302 return skel_sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz); 303 } 304 305 static inline int skel_link_create(int prog_fd, int target_fd, 306 enum bpf_attach_type attach_type) 307 { 308 const size_t attr_sz = offsetofend(union bpf_attr, link_create.iter_info_len); 309 union bpf_attr attr; 310 311 memset(&attr, 0, attr_sz); 312 attr.link_create.prog_fd = prog_fd; 313 attr.link_create.target_fd = target_fd; 314 attr.link_create.attach_type = attach_type; 315 316 return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz); 317 } 318 319 static inline int skel_map_freeze(int fd) 320 { 321 const size_t attr_sz = offsetofend(union bpf_attr, map_fd); 322 union bpf_attr attr; 323 324 memset(&attr, 0, attr_sz); 325 attr.map_fd = fd; 326 327 return skel_sys_bpf(BPF_MAP_FREEZE, &attr, attr_sz); 328 } 329 #ifdef __KERNEL__ 330 #define set_err 331 #else 332 #define set_err err = -errno 333 #endif 334 335 static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) 336 { 337 const size_t prog_load_attr_sz = offsetofend(union bpf_attr, keyring_id); 338 const size_t test_run_attr_sz = offsetofend(union bpf_attr, test); 339 int map_fd = -1, prog_fd = -1, key = 0, err; 340 union bpf_attr attr; 341 342 err = map_fd = skel_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, opts->data_sz, 1, 343 opts->excl_prog_hash, opts->excl_prog_hash_sz); 344 if (map_fd < 0) { 345 opts->errstr = "failed to create loader map"; 346 set_err; 347 goto out; 348 } 349 350 err = skel_map_update_elem(map_fd, &key, opts->data, 0); 351 if (err < 0) { 352 opts->errstr = "failed to update loader map"; 353 set_err; 354 goto out; 355 } 356 357 #ifndef __KERNEL__ 358 err = skel_map_freeze(map_fd); 359 if (err < 0) { 360 opts->errstr = "failed to freeze map"; 361 set_err; 362 goto out; 363 } 364 #endif 365 366 memset(&attr, 0, prog_load_attr_sz); 367 attr.prog_type = BPF_PROG_TYPE_SYSCALL; 368 attr.insns = (long) opts->insns; 369 attr.insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); 370 attr.license = (long) "Dual BSD/GPL"; 371 #ifndef __KERNEL__ 372 attr.signature = (long) opts->signature; 373 attr.signature_size = opts->signature_sz; 374 if (opts->signature) 375 attr.fd_array_cnt = 1; 376 #else 377 if (opts->signature || opts->signature_sz) 378 pr_warn("signatures are not supported from bpf_preload\n"); 379 #endif 380 attr.keyring_id = opts->keyring_id; 381 memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); 382 attr.fd_array = (long) &map_fd; 383 attr.log_level = opts->ctx->log_level; 384 attr.log_size = opts->ctx->log_size; 385 attr.log_buf = opts->ctx->log_buf; 386 attr.prog_flags = BPF_F_SLEEPABLE; 387 err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, prog_load_attr_sz); 388 if (prog_fd < 0) { 389 opts->errstr = "failed to load loader prog"; 390 set_err; 391 goto out; 392 } 393 394 memset(&attr, 0, test_run_attr_sz); 395 attr.test.prog_fd = prog_fd; 396 attr.test.ctx_in = (long) opts->ctx; 397 attr.test.ctx_size_in = opts->ctx->sz; 398 err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz); 399 if (err < 0 || (int)attr.test.retval < 0) { 400 if (err < 0) { 401 opts->errstr = "failed to execute loader prog"; 402 set_err; 403 } else { 404 opts->errstr = "error returned by loader prog"; 405 err = (int)attr.test.retval; 406 #ifndef __KERNEL__ 407 errno = -err; 408 #endif 409 } 410 goto out; 411 } 412 err = 0; 413 out: 414 if (map_fd >= 0) 415 close(map_fd); 416 if (prog_fd >= 0) 417 close(prog_fd); 418 return err; 419 } 420 421 #endif 422