1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (c) 2021 Facebook */ 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <string.h> 6 #include <errno.h> 7 #include <linux/filter.h> 8 #include <sys/param.h> 9 #include "btf.h" 10 #include "bpf.h" 11 #include "libbpf.h" 12 #include "libbpf_internal.h" 13 #include "hashmap.h" 14 #include "bpf_gen_internal.h" 15 #include "skel_internal.h" 16 #include <asm/byteorder.h> 17 18 #define MAX_USED_MAPS 64 19 #define MAX_USED_PROGS 32 20 #define MAX_KFUNC_DESCS 256 21 #define MAX_FD_ARRAY_SZ (MAX_USED_MAPS + MAX_KFUNC_DESCS) 22 23 /* The following structure describes the stack layout of the loader program. 24 * In addition R6 contains the pointer to context. 25 * R7 contains the result of the last sys_bpf command (typically error or FD). 26 * R9 contains the result of the last sys_close command. 27 * 28 * Naming convention: 29 * ctx - bpf program context 30 * stack - bpf program stack 31 * blob - bpf_attr-s, strings, insns, map data. 32 * All the bytes that loader prog will use for read/write. 33 */ 34 struct loader_stack { 35 __u32 btf_fd; 36 __u32 inner_map_fd; 37 __u32 prog_fd[MAX_USED_PROGS]; 38 }; 39 40 #define stack_off(field) \ 41 (__s16)(-sizeof(struct loader_stack) + offsetof(struct loader_stack, field)) 42 43 #define attr_field(attr, field) (attr + offsetof(union bpf_attr, field)) 44 45 static int blob_fd_array_off(struct bpf_gen *gen, int index) 46 { 47 return gen->fd_array + index * sizeof(int); 48 } 49 50 static int realloc_insn_buf(struct bpf_gen *gen, __u32 size) 51 { 52 size_t off = gen->insn_cur - gen->insn_start; 53 void *insn_start; 54 55 if (gen->error) 56 return gen->error; 57 if (size > INT32_MAX || off + size > INT32_MAX) { 58 gen->error = -ERANGE; 59 return -ERANGE; 60 } 61 insn_start = realloc(gen->insn_start, off + size); 62 if (!insn_start) { 63 gen->error = -ENOMEM; 64 free(gen->insn_start); 65 gen->insn_start = NULL; 66 return -ENOMEM; 67 } 68 gen->insn_start = insn_start; 69 gen->insn_cur = insn_start + off; 70 return 0; 71 } 72 73 static int realloc_data_buf(struct bpf_gen *gen, __u32 size) 74 { 75 size_t off = gen->data_cur - gen->data_start; 76 void *data_start; 77 78 if (gen->error) 79 return gen->error; 80 if (size > INT32_MAX || off + size > INT32_MAX) { 81 gen->error = -ERANGE; 82 return -ERANGE; 83 } 84 data_start = realloc(gen->data_start, off + size); 85 if (!data_start) { 86 gen->error = -ENOMEM; 87 free(gen->data_start); 88 gen->data_start = NULL; 89 return -ENOMEM; 90 } 91 gen->data_start = data_start; 92 gen->data_cur = data_start + off; 93 return 0; 94 } 95 96 static void emit(struct bpf_gen *gen, struct bpf_insn insn) 97 { 98 if (realloc_insn_buf(gen, sizeof(insn))) 99 return; 100 memcpy(gen->insn_cur, &insn, sizeof(insn)); 101 gen->insn_cur += sizeof(insn); 102 } 103 104 static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2) 105 { 106 emit(gen, insn1); 107 emit(gen, insn2); 108 } 109 110 static int add_data(struct bpf_gen *gen, const void *data, __u32 size); 111 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off); 112 113 void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) 114 { 115 size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz; 116 int i; 117 118 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int)); 119 gen->log_level = log_level; 120 /* save ctx pointer into R6 */ 121 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); 122 123 /* bzero stack */ 124 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); 125 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz)); 126 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz)); 127 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); 128 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); 129 130 /* amount of stack actually used, only used to calculate iterations, not stack offset */ 131 nr_progs_sz = offsetof(struct loader_stack, prog_fd[nr_progs]); 132 /* jump over cleanup code */ 133 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 134 /* size of cleanup code below (including map fd cleanup) */ 135 (nr_progs_sz / 4) * 3 + 2 + 136 /* 6 insns for emit_sys_close_blob, 137 * 6 insns for debug_regs in emit_sys_close_blob 138 */ 139 nr_maps * (6 + (gen->log_level ? 6 : 0)))); 140 141 /* remember the label where all error branches will jump to */ 142 gen->cleanup_label = gen->insn_cur - gen->insn_start; 143 /* emit cleanup code: close all temp FDs */ 144 for (i = 0; i < nr_progs_sz; i += 4) { 145 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i)); 146 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1)); 147 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); 148 } 149 for (i = 0; i < nr_maps; i++) 150 emit_sys_close_blob(gen, blob_fd_array_off(gen, i)); 151 /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */ 152 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7)); 153 emit(gen, BPF_EXIT_INSN()); 154 } 155 156 static int add_data(struct bpf_gen *gen, const void *data, __u32 size) 157 { 158 __u32 size8 = roundup(size, 8); 159 __u64 zero = 0; 160 void *prev; 161 162 if (realloc_data_buf(gen, size8)) 163 return 0; 164 prev = gen->data_cur; 165 if (data) { 166 memcpy(gen->data_cur, data, size); 167 memcpy(gen->data_cur + size, &zero, size8 - size); 168 } else { 169 memset(gen->data_cur, 0, size8); 170 } 171 gen->data_cur += size8; 172 return prev - gen->data_start; 173 } 174 175 /* Get index for map_fd/btf_fd slot in reserved fd_array, or in data relative 176 * to start of fd_array. Caller can decide if it is usable or not. 177 */ 178 static int add_map_fd(struct bpf_gen *gen) 179 { 180 if (gen->nr_maps == MAX_USED_MAPS) { 181 pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS); 182 gen->error = -E2BIG; 183 return 0; 184 } 185 return gen->nr_maps++; 186 } 187 188 static int add_kfunc_btf_fd(struct bpf_gen *gen) 189 { 190 int cur; 191 192 if (gen->nr_fd_array == MAX_KFUNC_DESCS) { 193 cur = add_data(gen, NULL, sizeof(int)); 194 return (cur - gen->fd_array) / sizeof(int); 195 } 196 return MAX_USED_MAPS + gen->nr_fd_array++; 197 } 198 199 static int insn_bytes_to_bpf_size(__u32 sz) 200 { 201 switch (sz) { 202 case 8: return BPF_DW; 203 case 4: return BPF_W; 204 case 2: return BPF_H; 205 case 1: return BPF_B; 206 default: return -1; 207 } 208 } 209 210 /* *(u64 *)(blob + off) = (u64)(void *)(blob + data) */ 211 static void emit_rel_store(struct bpf_gen *gen, int off, int data) 212 { 213 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 214 0, 0, 0, data)); 215 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 216 0, 0, 0, off)); 217 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0)); 218 } 219 220 static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off) 221 { 222 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, 223 0, 0, 0, blob_off)); 224 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0)); 225 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 226 0, 0, 0, off)); 227 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); 228 } 229 230 static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off) 231 { 232 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 233 0, 0, 0, blob_off)); 234 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0)); 235 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); 236 } 237 238 static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off, 239 bool check_non_zero) 240 { 241 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off)); 242 if (check_non_zero) 243 /* If value in ctx is zero don't update the blob. 244 * For example: when ctx->map.max_entries == 0, keep default max_entries from bpf.c 245 */ 246 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3)); 247 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 248 0, 0, 0, off)); 249 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); 250 } 251 252 static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off) 253 { 254 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); 255 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 256 0, 0, 0, off)); 257 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); 258 } 259 260 static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off) 261 { 262 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); 263 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); 264 } 265 266 static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size) 267 { 268 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd)); 269 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, 270 0, 0, 0, attr)); 271 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size)); 272 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf)); 273 /* remember the result in R7 */ 274 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 275 } 276 277 static bool is_simm16(__s64 value) 278 { 279 return value == (__s64)(__s16)value; 280 } 281 282 static void emit_check_err(struct bpf_gen *gen) 283 { 284 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1; 285 286 /* R7 contains result of last sys_bpf command. 287 * if (R7 < 0) goto cleanup; 288 */ 289 if (is_simm16(off)) { 290 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off)); 291 } else { 292 gen->error = -ERANGE; 293 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1)); 294 } 295 } 296 297 /* reg1 and reg2 should not be R1 - R5. They can be R0, R6 - R10 */ 298 static void emit_debug(struct bpf_gen *gen, int reg1, int reg2, 299 const char *fmt, va_list args) 300 { 301 char buf[1024]; 302 int addr, len, ret; 303 304 if (!gen->log_level) 305 return; 306 ret = vsnprintf(buf, sizeof(buf), fmt, args); 307 if (ret < 1024 - 7 && reg1 >= 0 && reg2 < 0) 308 /* The special case to accommodate common debug_ret(): 309 * to avoid specifying BPF_REG_7 and adding " r=%%d" to 310 * prints explicitly. 311 */ 312 strcat(buf, " r=%d"); 313 len = strlen(buf) + 1; 314 addr = add_data(gen, buf, len); 315 316 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 317 0, 0, 0, addr)); 318 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 319 if (reg1 >= 0) 320 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1)); 321 if (reg2 >= 0) 322 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2)); 323 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk)); 324 } 325 326 static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...) 327 { 328 va_list args; 329 330 va_start(args, fmt); 331 emit_debug(gen, reg1, reg2, fmt, args); 332 va_end(args); 333 } 334 335 static void debug_ret(struct bpf_gen *gen, const char *fmt, ...) 336 { 337 va_list args; 338 339 va_start(args, fmt); 340 emit_debug(gen, BPF_REG_7, -1, fmt, args); 341 va_end(args); 342 } 343 344 static void __emit_sys_close(struct bpf_gen *gen) 345 { 346 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 347 /* 2 is the number of the following insns 348 * * 6 is additional insns in debug_regs 349 */ 350 2 + (gen->log_level ? 6 : 0))); 351 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1)); 352 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); 353 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d"); 354 } 355 356 static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off) 357 { 358 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off)); 359 __emit_sys_close(gen); 360 } 361 362 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off) 363 { 364 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 365 0, 0, 0, blob_off)); 366 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0)); 367 __emit_sys_close(gen); 368 } 369 370 int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) 371 { 372 int i; 373 374 if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) { 375 pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n", 376 nr_progs, gen->nr_progs, nr_maps, gen->nr_maps); 377 gen->error = -EFAULT; 378 return gen->error; 379 } 380 emit_sys_close_stack(gen, stack_off(btf_fd)); 381 for (i = 0; i < gen->nr_progs; i++) 382 move_stack2ctx(gen, 383 sizeof(struct bpf_loader_ctx) + 384 sizeof(struct bpf_map_desc) * gen->nr_maps + 385 sizeof(struct bpf_prog_desc) * i + 386 offsetof(struct bpf_prog_desc, prog_fd), 4, 387 stack_off(prog_fd[i])); 388 for (i = 0; i < gen->nr_maps; i++) 389 move_blob2ctx(gen, 390 sizeof(struct bpf_loader_ctx) + 391 sizeof(struct bpf_map_desc) * i + 392 offsetof(struct bpf_map_desc, map_fd), 4, 393 blob_fd_array_off(gen, i)); 394 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0)); 395 emit(gen, BPF_EXIT_INSN()); 396 pr_debug("gen: finish %d\n", gen->error); 397 if (!gen->error) { 398 struct gen_loader_opts *opts = gen->opts; 399 400 opts->insns = gen->insn_start; 401 opts->insns_sz = gen->insn_cur - gen->insn_start; 402 opts->data = gen->data_start; 403 opts->data_sz = gen->data_cur - gen->data_start; 404 405 /* use target endianness for embedded loader */ 406 if (gen->swapped_endian) { 407 struct bpf_insn *insn = (struct bpf_insn *)opts->insns; 408 int insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); 409 410 for (i = 0; i < insn_cnt; i++) 411 bpf_insn_bswap(insn++); 412 } 413 } 414 return gen->error; 415 } 416 417 void bpf_gen__free(struct bpf_gen *gen) 418 { 419 if (!gen) 420 return; 421 free(gen->data_start); 422 free(gen->insn_start); 423 free(gen); 424 } 425 426 /* 427 * Fields of bpf_attr are set to values in native byte-order before being 428 * written to the target-bound data blob, and may need endian conversion. 429 * This macro allows providing the correct value in situ more simply than 430 * writing a separate converter for *all fields* of *all records* included 431 * in union bpf_attr. Note that sizeof(rval) should match the assignment 432 * target to avoid runtime problems. 433 */ 434 #define tgt_endian(rval) ({ \ 435 typeof(rval) _val = (rval); \ 436 if (gen->swapped_endian) { \ 437 switch (sizeof(_val)) { \ 438 case 1: break; \ 439 case 2: _val = bswap_16(_val); break; \ 440 case 4: _val = bswap_32(_val); break; \ 441 case 8: _val = bswap_64(_val); break; \ 442 default: pr_warn("unsupported bswap size!\n"); \ 443 } \ 444 } \ 445 _val; \ 446 }) 447 448 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, 449 __u32 btf_raw_size) 450 { 451 int attr_size = offsetofend(union bpf_attr, btf_log_level); 452 int btf_data, btf_load_attr; 453 union bpf_attr attr; 454 455 memset(&attr, 0, attr_size); 456 btf_data = add_data(gen, btf_raw_data, btf_raw_size); 457 458 attr.btf_size = tgt_endian(btf_raw_size); 459 btf_load_attr = add_data(gen, &attr, attr_size); 460 pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n", 461 btf_data, btf_raw_size, btf_load_attr, attr_size); 462 463 /* populate union bpf_attr with user provided log details */ 464 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4, 465 offsetof(struct bpf_loader_ctx, log_level), false); 466 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4, 467 offsetof(struct bpf_loader_ctx, log_size), false); 468 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8, 469 offsetof(struct bpf_loader_ctx, log_buf), false); 470 /* populate union bpf_attr with a pointer to the BTF data */ 471 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data); 472 /* emit BTF_LOAD command */ 473 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size); 474 debug_ret(gen, "btf_load size %d", btf_raw_size); 475 emit_check_err(gen); 476 /* remember btf_fd in the stack, if successful */ 477 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd))); 478 } 479 480 void bpf_gen__map_create(struct bpf_gen *gen, 481 enum bpf_map_type map_type, 482 const char *map_name, 483 __u32 key_size, __u32 value_size, __u32 max_entries, 484 struct bpf_map_create_opts *map_attr, int map_idx) 485 { 486 int attr_size = offsetofend(union bpf_attr, map_extra); 487 bool close_inner_map_fd = false; 488 int map_create_attr, idx; 489 union bpf_attr attr; 490 491 memset(&attr, 0, attr_size); 492 attr.map_type = tgt_endian(map_type); 493 attr.key_size = tgt_endian(key_size); 494 attr.value_size = tgt_endian(value_size); 495 attr.map_flags = tgt_endian(map_attr->map_flags); 496 attr.map_extra = tgt_endian(map_attr->map_extra); 497 if (map_name) 498 libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); 499 attr.numa_node = tgt_endian(map_attr->numa_node); 500 attr.map_ifindex = tgt_endian(map_attr->map_ifindex); 501 attr.max_entries = tgt_endian(max_entries); 502 attr.btf_key_type_id = tgt_endian(map_attr->btf_key_type_id); 503 attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id); 504 505 map_create_attr = add_data(gen, &attr, attr_size); 506 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n", 507 map_name, map_idx, map_type, map_attr->btf_value_type_id, 508 map_create_attr, attr_size); 509 510 if (map_attr->btf_value_type_id) 511 /* populate union bpf_attr with btf_fd saved in the stack earlier */ 512 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4, 513 stack_off(btf_fd)); 514 switch (map_type) { 515 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 516 case BPF_MAP_TYPE_HASH_OF_MAPS: 517 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4, 518 stack_off(inner_map_fd)); 519 close_inner_map_fd = true; 520 break; 521 default: 522 break; 523 } 524 /* conditionally update max_entries */ 525 if (map_idx >= 0) 526 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4, 527 sizeof(struct bpf_loader_ctx) + 528 sizeof(struct bpf_map_desc) * map_idx + 529 offsetof(struct bpf_map_desc, max_entries), 530 true /* check that max_entries != 0 */); 531 /* emit MAP_CREATE command */ 532 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); 533 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d", 534 map_name, map_idx, map_type, value_size, 535 map_attr->btf_value_type_id); 536 emit_check_err(gen); 537 /* remember map_fd in the stack, if successful */ 538 if (map_idx < 0) { 539 /* This bpf_gen__map_create() function is called with map_idx >= 0 540 * for all maps that libbpf loading logic tracks. 541 * It's called with -1 to create an inner map. 542 */ 543 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, 544 stack_off(inner_map_fd))); 545 } else if (map_idx != gen->nr_maps) { 546 gen->error = -EDOM; /* internal bug */ 547 return; 548 } else { 549 /* add_map_fd does gen->nr_maps++ */ 550 idx = add_map_fd(gen); 551 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 552 0, 0, 0, blob_fd_array_off(gen, idx))); 553 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0)); 554 } 555 if (close_inner_map_fd) 556 emit_sys_close_stack(gen, stack_off(inner_map_fd)); 557 } 558 559 void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, 560 enum bpf_attach_type type) 561 { 562 const char *prefix; 563 int kind, ret; 564 565 btf_get_kernel_prefix_kind(type, &prefix, &kind); 566 gen->attach_kind = kind; 567 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s", 568 prefix, attach_name); 569 if (ret >= sizeof(gen->attach_target)) 570 gen->error = -ENOSPC; 571 } 572 573 static void emit_find_attach_target(struct bpf_gen *gen) 574 { 575 int name, len = strlen(gen->attach_target) + 1; 576 577 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind); 578 name = add_data(gen, gen->attach_target, len); 579 580 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 581 0, 0, 0, name)); 582 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 583 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind)); 584 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); 585 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); 586 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 587 debug_ret(gen, "find_by_name_kind(%s,%d)", 588 gen->attach_target, gen->attach_kind); 589 emit_check_err(gen); 590 /* if successful, btf_id is in lower 32-bit of R7 and 591 * btf_obj_fd is in upper 32-bit 592 */ 593 } 594 595 void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, 596 bool is_typeless, bool is_ld64, int kind, int insn_idx) 597 { 598 struct ksym_relo_desc *relo; 599 600 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo)); 601 if (!relo) { 602 gen->error = -ENOMEM; 603 return; 604 } 605 gen->relos = relo; 606 relo += gen->relo_cnt; 607 relo->name = name; 608 relo->is_weak = is_weak; 609 relo->is_typeless = is_typeless; 610 relo->is_ld64 = is_ld64; 611 relo->kind = kind; 612 relo->insn_idx = insn_idx; 613 gen->relo_cnt++; 614 } 615 616 /* returns existing ksym_desc with ref incremented, or inserts a new one */ 617 static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo) 618 { 619 struct ksym_desc *kdesc; 620 int i; 621 622 for (i = 0; i < gen->nr_ksyms; i++) { 623 kdesc = &gen->ksyms[i]; 624 if (kdesc->kind == relo->kind && kdesc->is_ld64 == relo->is_ld64 && 625 !strcmp(kdesc->name, relo->name)) { 626 kdesc->ref++; 627 return kdesc; 628 } 629 } 630 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc)); 631 if (!kdesc) { 632 gen->error = -ENOMEM; 633 return NULL; 634 } 635 gen->ksyms = kdesc; 636 kdesc = &gen->ksyms[gen->nr_ksyms++]; 637 kdesc->name = relo->name; 638 kdesc->kind = relo->kind; 639 kdesc->ref = 1; 640 kdesc->off = 0; 641 kdesc->insn = 0; 642 kdesc->is_ld64 = relo->is_ld64; 643 return kdesc; 644 } 645 646 /* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7} 647 * Returns result in BPF_REG_7 648 */ 649 static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo) 650 { 651 int name_off, len = strlen(relo->name) + 1; 652 653 name_off = add_data(gen, relo->name, len); 654 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 655 0, 0, 0, name_off)); 656 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 657 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind)); 658 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); 659 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); 660 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 661 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind); 662 } 663 664 /* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7} 665 * Returns result in BPF_REG_7 666 * Returns u64 symbol addr in BPF_REG_9 667 */ 668 static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo) 669 { 670 int name_off, len = strlen(relo->name) + 1, res_off; 671 672 name_off = add_data(gen, relo->name, len); 673 res_off = add_data(gen, NULL, 8); /* res is u64 */ 674 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 675 0, 0, 0, name_off)); 676 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 677 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); 678 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE, 679 0, 0, 0, res_off)); 680 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4)); 681 emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name)); 682 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0)); 683 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 684 debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind); 685 } 686 687 /* Expects: 688 * BPF_REG_8 - pointer to instruction 689 * 690 * We need to reuse BTF fd for same symbol otherwise each relocation takes a new 691 * index, while kernel limits total kfunc BTFs to 256. For duplicate symbols, 692 * this would mean a new BTF fd index for each entry. By pairing symbol name 693 * with index, we get the insn->imm, insn->off pairing that kernel uses for 694 * kfunc_tab, which becomes the effective limit even though all of them may 695 * share same index in fd_array (such that kfunc_btf_tab has 1 element). 696 */ 697 static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) 698 { 699 struct ksym_desc *kdesc; 700 int btf_fd_idx; 701 702 kdesc = get_ksym_desc(gen, relo); 703 if (!kdesc) 704 return; 705 /* try to copy from existing bpf_insn */ 706 if (kdesc->ref > 1) { 707 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, 708 kdesc->insn + offsetof(struct bpf_insn, imm)); 709 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2, 710 kdesc->insn + offsetof(struct bpf_insn, off)); 711 goto log; 712 } 713 /* remember insn offset, so we can copy BTF ID and FD later */ 714 kdesc->insn = insn; 715 emit_bpf_find_by_name_kind(gen, relo); 716 if (!relo->is_weak) 717 emit_check_err(gen); 718 /* get index in fd_array to store BTF FD at */ 719 btf_fd_idx = add_kfunc_btf_fd(gen); 720 if (btf_fd_idx > INT16_MAX) { 721 pr_warn("BTF fd off %d for kfunc %s exceeds INT16_MAX, cannot process relocation\n", 722 btf_fd_idx, relo->name); 723 gen->error = -E2BIG; 724 return; 725 } 726 kdesc->off = btf_fd_idx; 727 /* jump to success case */ 728 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); 729 /* set value for imm, off as 0 */ 730 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); 731 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); 732 /* skip success case for ret < 0 */ 733 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10)); 734 /* store btf_id into insn[insn_idx].imm */ 735 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); 736 /* obtain fd in BPF_REG_9 */ 737 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7)); 738 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); 739 /* load fd_array slot pointer */ 740 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 741 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx))); 742 /* store BTF fd in slot, 0 for vmlinux */ 743 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0)); 744 /* jump to insn[insn_idx].off store if fd denotes module BTF */ 745 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2)); 746 /* set the default value for off */ 747 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); 748 /* skip BTF fd store for vmlinux BTF */ 749 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); 750 /* store index into insn[insn_idx].off */ 751 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx)); 752 log: 753 if (!gen->log_level) 754 return; 755 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, 756 offsetof(struct bpf_insn, imm))); 757 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, 758 offsetof(struct bpf_insn, off))); 759 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d", 760 relo->name, kdesc->ref); 761 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 762 0, 0, 0, blob_fd_array_off(gen, kdesc->off))); 763 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0)); 764 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd", 765 relo->name, kdesc->ref); 766 } 767 768 static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo, 769 int ref) 770 { 771 if (!gen->log_level) 772 return; 773 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, 774 offsetof(struct bpf_insn, imm))); 775 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) + 776 offsetof(struct bpf_insn, imm))); 777 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d", 778 relo->is_typeless, relo->is_weak, relo->name, ref); 779 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); 780 debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg", 781 relo->is_typeless, relo->is_weak, relo->name, ref); 782 } 783 784 /* Expects: 785 * BPF_REG_8 - pointer to instruction 786 */ 787 static void emit_relo_ksym_typeless(struct bpf_gen *gen, 788 struct ksym_relo_desc *relo, int insn) 789 { 790 struct ksym_desc *kdesc; 791 792 kdesc = get_ksym_desc(gen, relo); 793 if (!kdesc) 794 return; 795 /* try to copy from existing ldimm64 insn */ 796 if (kdesc->ref > 1) { 797 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, 798 kdesc->insn + offsetof(struct bpf_insn, imm)); 799 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, 800 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)); 801 goto log; 802 } 803 /* remember insn offset, so we can copy ksym addr later */ 804 kdesc->insn = insn; 805 /* skip typeless ksym_desc in fd closing loop in cleanup_relos */ 806 kdesc->typeless = true; 807 emit_bpf_kallsyms_lookup_name(gen, relo); 808 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1)); 809 emit_check_err(gen); 810 /* store lower half of addr into insn[insn_idx].imm */ 811 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm))); 812 /* store upper half of addr into insn[insn_idx + 1].imm */ 813 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); 814 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, 815 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm))); 816 log: 817 emit_ksym_relo_log(gen, relo, kdesc->ref); 818 } 819 820 static __u32 src_reg_mask(struct bpf_gen *gen) 821 { 822 #if defined(__LITTLE_ENDIAN_BITFIELD) /* src_reg,dst_reg,... */ 823 return gen->swapped_endian ? 0xf0 : 0x0f; 824 #elif defined(__BIG_ENDIAN_BITFIELD) /* dst_reg,src_reg,... */ 825 return gen->swapped_endian ? 0x0f : 0xf0; 826 #else 827 #error "Unsupported bit endianness, cannot proceed" 828 #endif 829 } 830 831 /* Expects: 832 * BPF_REG_8 - pointer to instruction 833 */ 834 static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) 835 { 836 struct ksym_desc *kdesc; 837 __u32 reg_mask; 838 839 kdesc = get_ksym_desc(gen, relo); 840 if (!kdesc) 841 return; 842 /* try to copy from existing ldimm64 insn */ 843 if (kdesc->ref > 1) { 844 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, 845 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)); 846 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, 847 kdesc->insn + offsetof(struct bpf_insn, imm)); 848 /* jump over src_reg adjustment if imm (btf_id) is not 0, reuse BPF_REG_0 from move_blob2blob 849 * If btf_id is zero, clear BPF_PSEUDO_BTF_ID flag in src_reg of ld_imm64 insn 850 */ 851 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3)); 852 goto clear_src_reg; 853 } 854 /* remember insn offset, so we can copy BTF ID and FD later */ 855 kdesc->insn = insn; 856 emit_bpf_find_by_name_kind(gen, relo); 857 if (!relo->is_weak) 858 emit_check_err(gen); 859 /* jump to success case */ 860 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); 861 /* set values for insn[insn_idx].imm, insn[insn_idx + 1].imm as 0 */ 862 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); 863 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0)); 864 /* skip success case for ret < 0 */ 865 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4)); 866 /* store btf_id into insn[insn_idx].imm */ 867 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); 868 /* store btf_obj_fd into insn[insn_idx + 1].imm */ 869 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); 870 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, 871 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm))); 872 /* skip src_reg adjustment */ 873 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3)); 874 clear_src_reg: 875 /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */ 876 reg_mask = src_reg_mask(gen); 877 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); 878 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask)); 879 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code))); 880 881 emit_ksym_relo_log(gen, relo, kdesc->ref); 882 } 883 884 void bpf_gen__record_relo_core(struct bpf_gen *gen, 885 const struct bpf_core_relo *core_relo) 886 { 887 struct bpf_core_relo *relos; 888 889 relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos)); 890 if (!relos) { 891 gen->error = -ENOMEM; 892 return; 893 } 894 gen->core_relos = relos; 895 relos += gen->core_relo_cnt; 896 memcpy(relos, core_relo, sizeof(*relos)); 897 gen->core_relo_cnt++; 898 } 899 900 static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns) 901 { 902 int insn; 903 904 pr_debug("gen: emit_relo (%d): %s at %d %s\n", 905 relo->kind, relo->name, relo->insn_idx, relo->is_ld64 ? "ld64" : "call"); 906 insn = insns + sizeof(struct bpf_insn) * relo->insn_idx; 907 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn)); 908 if (relo->is_ld64) { 909 if (relo->is_typeless) 910 emit_relo_ksym_typeless(gen, relo, insn); 911 else 912 emit_relo_ksym_btf(gen, relo, insn); 913 } else { 914 emit_relo_kfunc_btf(gen, relo, insn); 915 } 916 } 917 918 static void emit_relos(struct bpf_gen *gen, int insns) 919 { 920 int i; 921 922 for (i = 0; i < gen->relo_cnt; i++) 923 emit_relo(gen, gen->relos + i, insns); 924 } 925 926 static void cleanup_core_relo(struct bpf_gen *gen) 927 { 928 if (!gen->core_relo_cnt) 929 return; 930 free(gen->core_relos); 931 gen->core_relo_cnt = 0; 932 gen->core_relos = NULL; 933 } 934 935 static void cleanup_relos(struct bpf_gen *gen, int insns) 936 { 937 struct ksym_desc *kdesc; 938 int i, insn; 939 940 for (i = 0; i < gen->nr_ksyms; i++) { 941 kdesc = &gen->ksyms[i]; 942 /* only close fds for typed ksyms and kfuncs */ 943 if (kdesc->is_ld64 && !kdesc->typeless) { 944 /* close fd recorded in insn[insn_idx + 1].imm */ 945 insn = kdesc->insn; 946 insn += sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm); 947 emit_sys_close_blob(gen, insn); 948 } else if (!kdesc->is_ld64) { 949 emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off)); 950 if (kdesc->off < MAX_FD_ARRAY_SZ) 951 gen->nr_fd_array--; 952 } 953 } 954 if (gen->nr_ksyms) { 955 free(gen->ksyms); 956 gen->nr_ksyms = 0; 957 gen->ksyms = NULL; 958 } 959 if (gen->relo_cnt) { 960 free(gen->relos); 961 gen->relo_cnt = 0; 962 gen->relos = NULL; 963 } 964 cleanup_core_relo(gen); 965 } 966 967 /* Convert func, line, and core relo info blobs to target endianness */ 968 static void info_blob_bswap(struct bpf_gen *gen, int func_info, int line_info, 969 int core_relos, struct bpf_prog_load_opts *load_attr) 970 { 971 struct bpf_func_info *fi = gen->data_start + func_info; 972 struct bpf_line_info *li = gen->data_start + line_info; 973 struct bpf_core_relo *cr = gen->data_start + core_relos; 974 int i; 975 976 for (i = 0; i < load_attr->func_info_cnt; i++) 977 bpf_func_info_bswap(fi++); 978 979 for (i = 0; i < load_attr->line_info_cnt; i++) 980 bpf_line_info_bswap(li++); 981 982 for (i = 0; i < gen->core_relo_cnt; i++) 983 bpf_core_relo_bswap(cr++); 984 } 985 986 void bpf_gen__prog_load(struct bpf_gen *gen, 987 enum bpf_prog_type prog_type, const char *prog_name, 988 const char *license, struct bpf_insn *insns, size_t insn_cnt, 989 struct bpf_prog_load_opts *load_attr, int prog_idx) 990 { 991 int func_info_tot_sz = load_attr->func_info_cnt * 992 load_attr->func_info_rec_size; 993 int line_info_tot_sz = load_attr->line_info_cnt * 994 load_attr->line_info_rec_size; 995 int core_relo_tot_sz = gen->core_relo_cnt * 996 sizeof(struct bpf_core_relo); 997 int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos; 998 int attr_size = offsetofend(union bpf_attr, core_relo_rec_size); 999 union bpf_attr attr; 1000 1001 memset(&attr, 0, attr_size); 1002 /* add license string to blob of bytes */ 1003 license_off = add_data(gen, license, strlen(license) + 1); 1004 /* add insns to blob of bytes */ 1005 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn)); 1006 pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n", 1007 prog_idx, prog_type, insns_off, insn_cnt, license_off); 1008 1009 /* convert blob insns to target endianness */ 1010 if (gen->swapped_endian) { 1011 struct bpf_insn *insn = gen->data_start + insns_off; 1012 int i; 1013 1014 for (i = 0; i < insn_cnt; i++, insn++) 1015 bpf_insn_bswap(insn); 1016 } 1017 1018 attr.prog_type = tgt_endian(prog_type); 1019 attr.expected_attach_type = tgt_endian(load_attr->expected_attach_type); 1020 attr.attach_btf_id = tgt_endian(load_attr->attach_btf_id); 1021 attr.prog_ifindex = tgt_endian(load_attr->prog_ifindex); 1022 attr.kern_version = 0; 1023 attr.insn_cnt = tgt_endian((__u32)insn_cnt); 1024 attr.prog_flags = tgt_endian(load_attr->prog_flags); 1025 1026 attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size); 1027 attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt); 1028 func_info = add_data(gen, load_attr->func_info, func_info_tot_sz); 1029 pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n", 1030 func_info, load_attr->func_info_cnt, 1031 load_attr->func_info_rec_size); 1032 1033 attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size); 1034 attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt); 1035 line_info = add_data(gen, load_attr->line_info, line_info_tot_sz); 1036 pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n", 1037 line_info, load_attr->line_info_cnt, 1038 load_attr->line_info_rec_size); 1039 1040 attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo)); 1041 attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt); 1042 core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz); 1043 pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n", 1044 core_relos, gen->core_relo_cnt, 1045 sizeof(struct bpf_core_relo)); 1046 1047 /* convert all info blobs to target endianness */ 1048 if (gen->swapped_endian) 1049 info_blob_bswap(gen, func_info, line_info, core_relos, load_attr); 1050 1051 libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); 1052 prog_load_attr = add_data(gen, &attr, attr_size); 1053 pr_debug("gen: prog_load: attr: off %d size %d\n", 1054 prog_load_attr, attr_size); 1055 1056 /* populate union bpf_attr with a pointer to license */ 1057 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off); 1058 1059 /* populate union bpf_attr with a pointer to instructions */ 1060 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off); 1061 1062 /* populate union bpf_attr with a pointer to func_info */ 1063 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info); 1064 1065 /* populate union bpf_attr with a pointer to line_info */ 1066 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info); 1067 1068 /* populate union bpf_attr with a pointer to core_relos */ 1069 emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos); 1070 1071 /* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */ 1072 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array); 1073 1074 /* populate union bpf_attr with user provided log details */ 1075 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4, 1076 offsetof(struct bpf_loader_ctx, log_level), false); 1077 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4, 1078 offsetof(struct bpf_loader_ctx, log_size), false); 1079 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8, 1080 offsetof(struct bpf_loader_ctx, log_buf), false); 1081 /* populate union bpf_attr with btf_fd saved in the stack earlier */ 1082 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4, 1083 stack_off(btf_fd)); 1084 if (gen->attach_kind) { 1085 emit_find_attach_target(gen); 1086 /* populate union bpf_attr with btf_id and btf_obj_fd found by helper */ 1087 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 1088 0, 0, 0, prog_load_attr)); 1089 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, 1090 offsetof(union bpf_attr, attach_btf_id))); 1091 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); 1092 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, 1093 offsetof(union bpf_attr, attach_btf_obj_fd))); 1094 } 1095 emit_relos(gen, insns_off); 1096 /* emit PROG_LOAD command */ 1097 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size); 1098 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt); 1099 /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */ 1100 cleanup_relos(gen, insns_off); 1101 if (gen->attach_kind) { 1102 emit_sys_close_blob(gen, 1103 attr_field(prog_load_attr, attach_btf_obj_fd)); 1104 gen->attach_kind = 0; 1105 } 1106 emit_check_err(gen); 1107 /* remember prog_fd in the stack, if successful */ 1108 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, 1109 stack_off(prog_fd[gen->nr_progs]))); 1110 gen->nr_progs++; 1111 } 1112 1113 void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, 1114 __u32 value_size) 1115 { 1116 int attr_size = offsetofend(union bpf_attr, flags); 1117 int map_update_attr, value, key; 1118 union bpf_attr attr; 1119 int zero = 0; 1120 1121 memset(&attr, 0, attr_size); 1122 1123 value = add_data(gen, pvalue, value_size); 1124 key = add_data(gen, &zero, sizeof(zero)); 1125 1126 /* if (map_desc[map_idx].initial_value) { 1127 * if (ctx->flags & BPF_SKEL_KERNEL) 1128 * bpf_probe_read_kernel(value, value_size, initial_value); 1129 * else 1130 * bpf_copy_from_user(value, value_size, initial_value); 1131 * } 1132 */ 1133 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, 1134 sizeof(struct bpf_loader_ctx) + 1135 sizeof(struct bpf_map_desc) * map_idx + 1136 offsetof(struct bpf_map_desc, initial_value))); 1137 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); 1138 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 1139 0, 0, 0, value)); 1140 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); 1141 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, 1142 offsetof(struct bpf_loader_ctx, flags))); 1143 emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); 1144 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); 1145 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); 1146 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); 1147 1148 map_update_attr = add_data(gen, &attr, attr_size); 1149 pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n", 1150 map_idx, value, value_size, map_update_attr, attr_size); 1151 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, 1152 blob_fd_array_off(gen, map_idx)); 1153 emit_rel_store(gen, attr_field(map_update_attr, key), key); 1154 emit_rel_store(gen, attr_field(map_update_attr, value), value); 1155 /* emit MAP_UPDATE_ELEM command */ 1156 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); 1157 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size); 1158 emit_check_err(gen); 1159 } 1160 1161 void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot, 1162 int inner_map_idx) 1163 { 1164 int attr_size = offsetofend(union bpf_attr, flags); 1165 int map_update_attr, key; 1166 union bpf_attr attr; 1167 int tgt_slot; 1168 1169 memset(&attr, 0, attr_size); 1170 1171 tgt_slot = tgt_endian(slot); 1172 key = add_data(gen, &tgt_slot, sizeof(tgt_slot)); 1173 1174 map_update_attr = add_data(gen, &attr, attr_size); 1175 pr_debug("gen: populate_outer_map: outer %d key %d inner %d, attr: off %d size %d\n", 1176 outer_map_idx, slot, inner_map_idx, map_update_attr, attr_size); 1177 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, 1178 blob_fd_array_off(gen, outer_map_idx)); 1179 emit_rel_store(gen, attr_field(map_update_attr, key), key); 1180 emit_rel_store(gen, attr_field(map_update_attr, value), 1181 blob_fd_array_off(gen, inner_map_idx)); 1182 1183 /* emit MAP_UPDATE_ELEM command */ 1184 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); 1185 debug_ret(gen, "populate_outer_map outer %d key %d inner %d", 1186 outer_map_idx, slot, inner_map_idx); 1187 emit_check_err(gen); 1188 } 1189 1190 void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) 1191 { 1192 int attr_size = offsetofend(union bpf_attr, map_fd); 1193 int map_freeze_attr; 1194 union bpf_attr attr; 1195 1196 memset(&attr, 0, attr_size); 1197 map_freeze_attr = add_data(gen, &attr, attr_size); 1198 pr_debug("gen: map_freeze: idx %d, attr: off %d size %d\n", 1199 map_idx, map_freeze_attr, attr_size); 1200 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4, 1201 blob_fd_array_off(gen, map_idx)); 1202 /* emit MAP_FREEZE command */ 1203 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size); 1204 debug_ret(gen, "map_freeze"); 1205 emit_check_err(gen); 1206 } 1207