1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (c) 2021 Facebook */ 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <string.h> 6 #include <errno.h> 7 #include <asm/byteorder.h> 8 #include <linux/filter.h> 9 #include <sys/param.h> 10 #include "btf.h" 11 #include "bpf.h" 12 #include "libbpf.h" 13 #include "libbpf_internal.h" 14 #include "hashmap.h" 15 #include "bpf_gen_internal.h" 16 #include "skel_internal.h" 17 18 #define MAX_USED_MAPS 64 19 #define MAX_USED_PROGS 32 20 #define MAX_KFUNC_DESCS 256 21 #define MAX_FD_ARRAY_SZ (MAX_USED_MAPS + MAX_KFUNC_DESCS) 22 23 /* The following structure describes the stack layout of the loader program. 24 * In addition R6 contains the pointer to context. 25 * R7 contains the result of the last sys_bpf command (typically error or FD). 26 * R9 contains the result of the last sys_close command. 27 * 28 * Naming convention: 29 * ctx - bpf program context 30 * stack - bpf program stack 31 * blob - bpf_attr-s, strings, insns, map data. 32 * All the bytes that loader prog will use for read/write. 33 */ 34 struct loader_stack { 35 __u32 btf_fd; 36 __u32 inner_map_fd; 37 __u32 prog_fd[MAX_USED_PROGS]; 38 }; 39 40 #define stack_off(field) \ 41 (__s16)(-sizeof(struct loader_stack) + offsetof(struct loader_stack, field)) 42 43 #define attr_field(attr, field) (attr + offsetof(union bpf_attr, field)) 44 45 static int blob_fd_array_off(struct bpf_gen *gen, int index) 46 { 47 return gen->fd_array + index * sizeof(int); 48 } 49 50 static int realloc_insn_buf(struct bpf_gen *gen, __u32 size) 51 { 52 size_t off = gen->insn_cur - gen->insn_start; 53 void *insn_start; 54 55 if (gen->error) 56 return gen->error; 57 if (size > INT32_MAX || off + size > INT32_MAX) { 58 gen->error = -ERANGE; 59 return -ERANGE; 60 } 61 insn_start = realloc(gen->insn_start, off + size); 62 if (!insn_start) { 63 gen->error = -ENOMEM; 64 free(gen->insn_start); 65 gen->insn_start = NULL; 66 gen->insn_cur = NULL; 67 return -ENOMEM; 68 } 69 gen->insn_start = insn_start; 70 gen->insn_cur = insn_start + off; 71 return 0; 72 } 73 74 static int realloc_data_buf(struct bpf_gen *gen, __u32 size) 75 { 76 size_t off = gen->data_cur - gen->data_start; 77 void *data_start; 78 79 if (gen->error) 80 return gen->error; 81 if (size > INT32_MAX || off + size > INT32_MAX) { 82 gen->error = -ERANGE; 83 return -ERANGE; 84 } 85 data_start = realloc(gen->data_start, off + size); 86 if (!data_start) { 87 gen->error = -ENOMEM; 88 free(gen->data_start); 89 gen->data_start = NULL; 90 gen->data_cur = NULL; 91 return -ENOMEM; 92 } 93 gen->data_start = data_start; 94 gen->data_cur = data_start + off; 95 return 0; 96 } 97 98 static void emit(struct bpf_gen *gen, struct bpf_insn insn) 99 { 100 if (realloc_insn_buf(gen, sizeof(insn))) 101 return; 102 memcpy(gen->insn_cur, &insn, sizeof(insn)); 103 gen->insn_cur += sizeof(insn); 104 } 105 106 static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2) 107 { 108 emit(gen, insn1); 109 emit(gen, insn2); 110 } 111 112 static int add_data(struct bpf_gen *gen, const void *data, __u32 size); 113 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off); 114 static void emit_signature_match(struct bpf_gen *gen); 115 116 void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) 117 { 118 size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz; 119 int i; 120 121 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int)); 122 gen->log_level = log_level; 123 /* save ctx pointer into R6 */ 124 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); 125 126 /* bzero stack */ 127 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); 128 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz)); 129 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz)); 130 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); 131 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); 132 133 /* amount of stack actually used, only used to calculate iterations, not stack offset */ 134 nr_progs_sz = offsetof(struct loader_stack, prog_fd[nr_progs]); 135 /* jump over cleanup code */ 136 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 137 /* size of cleanup code below (including map fd cleanup) */ 138 (nr_progs_sz / 4) * 3 + 2 + 139 /* 6 insns for emit_sys_close_blob, 140 * 6 insns for debug_regs in emit_sys_close_blob 141 */ 142 nr_maps * (6 + (gen->log_level ? 6 : 0)))); 143 144 /* remember the label where all error branches will jump to */ 145 gen->cleanup_label = gen->insn_cur - gen->insn_start; 146 /* emit cleanup code: close all temp FDs */ 147 for (i = 0; i < nr_progs_sz; i += 4) { 148 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i)); 149 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1)); 150 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); 151 } 152 for (i = 0; i < nr_maps; i++) 153 emit_sys_close_blob(gen, blob_fd_array_off(gen, i)); 154 /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */ 155 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7)); 156 emit(gen, BPF_EXIT_INSN()); 157 if (OPTS_GET(gen->opts, gen_hash, false)) 158 emit_signature_match(gen); 159 } 160 161 static int add_data(struct bpf_gen *gen, const void *data, __u32 size) 162 { 163 __u32 size8 = roundup(size, 8); 164 __u64 zero = 0; 165 void *prev; 166 167 if (realloc_data_buf(gen, size8)) 168 return 0; 169 prev = gen->data_cur; 170 if (data) { 171 memcpy(gen->data_cur, data, size); 172 memcpy(gen->data_cur + size, &zero, size8 - size); 173 } else { 174 memset(gen->data_cur, 0, size8); 175 } 176 gen->data_cur += size8; 177 return prev - gen->data_start; 178 } 179 180 /* Get index for map_fd/btf_fd slot in reserved fd_array, or in data relative 181 * to start of fd_array. Caller can decide if it is usable or not. 182 */ 183 static int add_map_fd(struct bpf_gen *gen) 184 { 185 if (gen->nr_maps == MAX_USED_MAPS) { 186 pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS); 187 gen->error = -E2BIG; 188 return 0; 189 } 190 return gen->nr_maps++; 191 } 192 193 static int add_kfunc_btf_fd(struct bpf_gen *gen) 194 { 195 int cur; 196 197 if (gen->nr_fd_array == MAX_KFUNC_DESCS) { 198 cur = add_data(gen, NULL, sizeof(int)); 199 return (cur - gen->fd_array) / sizeof(int); 200 } 201 return MAX_USED_MAPS + gen->nr_fd_array++; 202 } 203 204 static int insn_bytes_to_bpf_size(__u32 sz) 205 { 206 switch (sz) { 207 case 8: return BPF_DW; 208 case 4: return BPF_W; 209 case 2: return BPF_H; 210 case 1: return BPF_B; 211 default: return -1; 212 } 213 } 214 215 /* *(u64 *)(blob + off) = (u64)(void *)(blob + data) */ 216 static void emit_rel_store(struct bpf_gen *gen, int off, int data) 217 { 218 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 219 0, 0, 0, data)); 220 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 221 0, 0, 0, off)); 222 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0)); 223 } 224 225 static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off) 226 { 227 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, 228 0, 0, 0, blob_off)); 229 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0)); 230 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 231 0, 0, 0, off)); 232 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); 233 } 234 235 static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off) 236 { 237 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 238 0, 0, 0, blob_off)); 239 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0)); 240 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); 241 } 242 243 static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off, 244 bool check_non_zero) 245 { 246 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off)); 247 if (check_non_zero) 248 /* If value in ctx is zero don't update the blob. 249 * For example: when ctx->map.max_entries == 0, keep default max_entries from bpf.c 250 */ 251 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3)); 252 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 253 0, 0, 0, off)); 254 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); 255 } 256 257 static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off) 258 { 259 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); 260 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 261 0, 0, 0, off)); 262 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); 263 } 264 265 static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off) 266 { 267 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); 268 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); 269 } 270 271 static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size) 272 { 273 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd)); 274 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, 275 0, 0, 0, attr)); 276 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size)); 277 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf)); 278 /* remember the result in R7 */ 279 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 280 } 281 282 static bool is_simm16(__s64 value) 283 { 284 return value == (__s64)(__s16)value; 285 } 286 287 static void emit_check_err(struct bpf_gen *gen) 288 { 289 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1; 290 291 /* R7 contains result of last sys_bpf command. 292 * if (R7 < 0) goto cleanup; 293 */ 294 if (is_simm16(off)) { 295 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off)); 296 } else { 297 gen->error = -ERANGE; 298 } 299 } 300 301 /* reg1 and reg2 should not be R1 - R5. They can be R0, R6 - R10 */ 302 static void emit_debug(struct bpf_gen *gen, int reg1, int reg2, 303 const char *fmt, va_list args) 304 { 305 char buf[1024]; 306 int addr, len, ret; 307 308 if (!gen->log_level) 309 return; 310 ret = vsnprintf(buf, sizeof(buf), fmt, args); 311 if (ret < 1024 - 7 && reg1 >= 0 && reg2 < 0) 312 /* The special case to accommodate common debug_ret(): 313 * to avoid specifying BPF_REG_7 and adding " r=%%d" to 314 * prints explicitly. 315 */ 316 strcat(buf, " r=%d"); 317 len = strlen(buf) + 1; 318 addr = add_data(gen, buf, len); 319 320 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 321 0, 0, 0, addr)); 322 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 323 if (reg1 >= 0) 324 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1)); 325 if (reg2 >= 0) 326 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2)); 327 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk)); 328 } 329 330 static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...) 331 { 332 va_list args; 333 334 va_start(args, fmt); 335 emit_debug(gen, reg1, reg2, fmt, args); 336 va_end(args); 337 } 338 339 static void debug_ret(struct bpf_gen *gen, const char *fmt, ...) 340 { 341 va_list args; 342 343 va_start(args, fmt); 344 emit_debug(gen, BPF_REG_7, -1, fmt, args); 345 va_end(args); 346 } 347 348 static void __emit_sys_close(struct bpf_gen *gen) 349 { 350 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 351 /* 2 is the number of the following insns 352 * * 6 is additional insns in debug_regs 353 */ 354 2 + (gen->log_level ? 6 : 0))); 355 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1)); 356 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); 357 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d"); 358 } 359 360 static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off) 361 { 362 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off)); 363 __emit_sys_close(gen); 364 } 365 366 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off) 367 { 368 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 369 0, 0, 0, blob_off)); 370 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0)); 371 __emit_sys_close(gen); 372 } 373 374 static void compute_sha_update_offsets(struct bpf_gen *gen); 375 376 int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) 377 { 378 int i; 379 380 if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) { 381 pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n", 382 nr_progs, gen->nr_progs, nr_maps, gen->nr_maps); 383 gen->error = -EFAULT; 384 return gen->error; 385 } 386 emit_sys_close_stack(gen, stack_off(btf_fd)); 387 for (i = 0; i < gen->nr_progs; i++) 388 move_stack2ctx(gen, 389 sizeof(struct bpf_loader_ctx) + 390 sizeof(struct bpf_map_desc) * gen->nr_maps + 391 sizeof(struct bpf_prog_desc) * i + 392 offsetof(struct bpf_prog_desc, prog_fd), 4, 393 stack_off(prog_fd[i])); 394 for (i = 0; i < gen->nr_maps; i++) 395 move_blob2ctx(gen, 396 sizeof(struct bpf_loader_ctx) + 397 sizeof(struct bpf_map_desc) * i + 398 offsetof(struct bpf_map_desc, map_fd), 4, 399 blob_fd_array_off(gen, i)); 400 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0)); 401 emit(gen, BPF_EXIT_INSN()); 402 if (!gen->error) { 403 struct gen_loader_opts *opts = gen->opts; 404 405 if (OPTS_GET(opts, gen_hash, false)) 406 compute_sha_update_offsets(gen); 407 408 opts->insns = gen->insn_start; 409 opts->insns_sz = gen->insn_cur - gen->insn_start; 410 opts->data = gen->data_start; 411 opts->data_sz = gen->data_cur - gen->data_start; 412 413 /* use target endianness for embedded loader */ 414 if (gen->swapped_endian) { 415 struct bpf_insn *insn = (struct bpf_insn *)opts->insns; 416 int insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); 417 418 for (i = 0; i < insn_cnt; i++) 419 bpf_insn_bswap(insn++); 420 } 421 } 422 pr_debug("gen: finish %s\n", errstr(gen->error)); 423 return gen->error; 424 } 425 426 void bpf_gen__free(struct bpf_gen *gen) 427 { 428 if (!gen) 429 return; 430 free(gen->data_start); 431 free(gen->insn_start); 432 free(gen); 433 } 434 435 /* 436 * Fields of bpf_attr are set to values in native byte-order before being 437 * written to the target-bound data blob, and may need endian conversion. 438 * This macro allows providing the correct value in situ more simply than 439 * writing a separate converter for *all fields* of *all records* included 440 * in union bpf_attr. Note that sizeof(rval) should match the assignment 441 * target to avoid runtime problems. 442 */ 443 #define tgt_endian(rval) ({ \ 444 typeof(rval) _val = (rval); \ 445 if (gen->swapped_endian) { \ 446 switch (sizeof(_val)) { \ 447 case 1: break; \ 448 case 2: _val = bswap_16(_val); break; \ 449 case 4: _val = bswap_32(_val); break; \ 450 case 8: _val = bswap_64(_val); break; \ 451 default: pr_warn("unsupported bswap size!\n"); \ 452 } \ 453 } \ 454 _val; \ 455 }) 456 457 static void compute_sha_update_offsets(struct bpf_gen *gen) 458 { 459 __u64 sha[SHA256_DWORD_SIZE]; 460 __u64 sha_dw; 461 int i; 462 463 libbpf_sha256(gen->data_start, gen->data_cur - gen->data_start, (__u8 *)sha); 464 for (i = 0; i < SHA256_DWORD_SIZE; i++) { 465 struct bpf_insn *insn = 466 (struct bpf_insn *)(gen->insn_start + gen->hash_insn_offset[i]); 467 sha_dw = tgt_endian(sha[i]); 468 insn[0].imm = (__u32)sha_dw; 469 insn[1].imm = sha_dw >> 32; 470 } 471 } 472 473 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, 474 __u32 btf_raw_size) 475 { 476 int attr_size = offsetofend(union bpf_attr, btf_log_level); 477 int btf_data, btf_load_attr; 478 union bpf_attr attr; 479 480 memset(&attr, 0, attr_size); 481 btf_data = add_data(gen, btf_raw_data, btf_raw_size); 482 483 attr.btf_size = tgt_endian(btf_raw_size); 484 btf_load_attr = add_data(gen, &attr, attr_size); 485 pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n", 486 btf_data, btf_raw_size, btf_load_attr, attr_size); 487 488 /* populate union bpf_attr with user provided log details */ 489 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4, 490 offsetof(struct bpf_loader_ctx, log_level), false); 491 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4, 492 offsetof(struct bpf_loader_ctx, log_size), false); 493 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8, 494 offsetof(struct bpf_loader_ctx, log_buf), false); 495 /* populate union bpf_attr with a pointer to the BTF data */ 496 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data); 497 /* emit BTF_LOAD command */ 498 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size); 499 debug_ret(gen, "btf_load size %d", btf_raw_size); 500 emit_check_err(gen); 501 /* remember btf_fd in the stack, if successful */ 502 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd))); 503 } 504 505 void bpf_gen__map_create(struct bpf_gen *gen, 506 enum bpf_map_type map_type, 507 const char *map_name, 508 __u32 key_size, __u32 value_size, __u32 max_entries, 509 struct bpf_map_create_opts *map_attr, int map_idx) 510 { 511 int attr_size = offsetofend(union bpf_attr, map_extra); 512 bool close_inner_map_fd = false; 513 int map_create_attr, idx; 514 union bpf_attr attr; 515 516 memset(&attr, 0, attr_size); 517 attr.map_type = tgt_endian(map_type); 518 attr.key_size = tgt_endian(key_size); 519 attr.value_size = tgt_endian(value_size); 520 attr.map_flags = tgt_endian(map_attr->map_flags); 521 attr.map_extra = tgt_endian(map_attr->map_extra); 522 if (map_name) 523 libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); 524 attr.numa_node = tgt_endian(map_attr->numa_node); 525 attr.map_ifindex = tgt_endian(map_attr->map_ifindex); 526 attr.max_entries = tgt_endian(max_entries); 527 attr.btf_key_type_id = tgt_endian(map_attr->btf_key_type_id); 528 attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id); 529 530 map_create_attr = add_data(gen, &attr, attr_size); 531 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n", 532 map_name, map_idx, map_type, map_attr->btf_value_type_id, 533 map_create_attr, attr_size); 534 535 if (map_attr->btf_value_type_id) 536 /* populate union bpf_attr with btf_fd saved in the stack earlier */ 537 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4, 538 stack_off(btf_fd)); 539 switch (map_type) { 540 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 541 case BPF_MAP_TYPE_HASH_OF_MAPS: 542 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4, 543 stack_off(inner_map_fd)); 544 close_inner_map_fd = true; 545 break; 546 default: 547 break; 548 } 549 550 /* 551 * Conditionally update max_entries from the host-supplied loader 552 * ctx. This sizes the map at runtime, but for a signed loader 553 * (gen_hash) it would let an untrusted host re-dimension the 554 * program's maps after emit_signature_match(), outside what the 555 * signature attests to. Keep the signer-provided max_entries 556 * baked into the blob in that case. 557 */ 558 if (map_idx >= 0 && !OPTS_GET(gen->opts, gen_hash, false)) 559 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4, 560 sizeof(struct bpf_loader_ctx) + 561 sizeof(struct bpf_map_desc) * map_idx + 562 offsetof(struct bpf_map_desc, max_entries), 563 true /* check that max_entries != 0 */); 564 565 /* emit MAP_CREATE command */ 566 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); 567 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d", 568 map_name, map_idx, map_type, value_size, 569 map_attr->btf_value_type_id); 570 emit_check_err(gen); 571 /* remember map_fd in the stack, if successful */ 572 if (map_idx < 0) { 573 /* This bpf_gen__map_create() function is called with map_idx >= 0 574 * for all maps that libbpf loading logic tracks. 575 * It's called with -1 to create an inner map. 576 */ 577 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, 578 stack_off(inner_map_fd))); 579 } else if (map_idx != gen->nr_maps) { 580 gen->error = -EDOM; /* internal bug */ 581 return; 582 } else { 583 /* add_map_fd does gen->nr_maps++ */ 584 idx = add_map_fd(gen); 585 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 586 0, 0, 0, blob_fd_array_off(gen, idx))); 587 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0)); 588 } 589 if (close_inner_map_fd) 590 emit_sys_close_stack(gen, stack_off(inner_map_fd)); 591 } 592 593 static void emit_signature_match(struct bpf_gen *gen) 594 { 595 __s64 off; 596 int i; 597 598 /* 599 * Reject if the metadata map is not exclusive. Without exclusivity 600 * the cached map->sha[] verified above can be stale: another BPF 601 * program with map access could have mutated the contents between 602 * BPF_OBJ_GET_INFO_BY_FD and loader execution. 603 */ 604 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX, 605 0, 0, 0, 0)); 606 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, SHA256_DIGEST_LENGTH)); 607 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2; 608 if (is_simm16(off)) { 609 emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL)); 610 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 1, off)); 611 } else { 612 gen->error = -ERANGE; 613 } 614 615 for (i = 0; i < SHA256_DWORD_SIZE; i++) { 616 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX, 617 0, 0, 0, 0)); 618 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, i * sizeof(__u64))); 619 gen->hash_insn_offset[i] = gen->insn_cur - gen->insn_start; 620 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_3, 0, 0, 0, 0, 0)); 621 622 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2; 623 if (is_simm16(off)) { 624 emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL)); 625 emit(gen, BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, off)); 626 } else { 627 gen->error = -ERANGE; 628 } 629 } 630 } 631 632 void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, 633 enum bpf_attach_type type) 634 { 635 const char *prefix; 636 int kind, ret; 637 638 btf_get_kernel_prefix_kind(type, &prefix, &kind); 639 gen->attach_kind = kind; 640 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s", 641 prefix, attach_name); 642 if (ret >= sizeof(gen->attach_target)) 643 gen->error = -ENOSPC; 644 } 645 646 static void emit_find_attach_target(struct bpf_gen *gen) 647 { 648 int name, len = strlen(gen->attach_target) + 1; 649 650 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind); 651 name = add_data(gen, gen->attach_target, len); 652 653 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 654 0, 0, 0, name)); 655 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 656 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind)); 657 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); 658 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); 659 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 660 debug_ret(gen, "find_by_name_kind(%s,%d)", 661 gen->attach_target, gen->attach_kind); 662 emit_check_err(gen); 663 /* if successful, btf_id is in lower 32-bit of R7 and 664 * btf_obj_fd is in upper 32-bit 665 */ 666 } 667 668 void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, 669 bool is_typeless, bool is_ld64, int kind, int insn_idx) 670 { 671 struct ksym_relo_desc *relo; 672 673 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo)); 674 if (!relo) { 675 gen->error = -ENOMEM; 676 return; 677 } 678 gen->relos = relo; 679 relo += gen->relo_cnt; 680 relo->name = name; 681 relo->is_weak = is_weak; 682 relo->is_typeless = is_typeless; 683 relo->is_ld64 = is_ld64; 684 relo->kind = kind; 685 relo->insn_idx = insn_idx; 686 gen->relo_cnt++; 687 } 688 689 /* returns existing ksym_desc with ref incremented, or inserts a new one */ 690 static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo) 691 { 692 struct ksym_desc *kdesc; 693 int i; 694 695 for (i = 0; i < gen->nr_ksyms; i++) { 696 kdesc = &gen->ksyms[i]; 697 if (kdesc->kind == relo->kind && kdesc->is_ld64 == relo->is_ld64 && 698 !strcmp(kdesc->name, relo->name)) { 699 kdesc->ref++; 700 return kdesc; 701 } 702 } 703 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc)); 704 if (!kdesc) { 705 gen->error = -ENOMEM; 706 return NULL; 707 } 708 gen->ksyms = kdesc; 709 kdesc = &gen->ksyms[gen->nr_ksyms++]; 710 kdesc->name = relo->name; 711 kdesc->kind = relo->kind; 712 kdesc->ref = 1; 713 kdesc->off = 0; 714 kdesc->insn = 0; 715 kdesc->is_ld64 = relo->is_ld64; 716 return kdesc; 717 } 718 719 /* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7} 720 * Returns result in BPF_REG_7 721 */ 722 static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo) 723 { 724 int name_off, len = strlen(relo->name) + 1; 725 726 name_off = add_data(gen, relo->name, len); 727 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 728 0, 0, 0, name_off)); 729 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 730 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind)); 731 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); 732 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); 733 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 734 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind); 735 } 736 737 /* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7} 738 * Returns result in BPF_REG_7 739 * Returns u64 symbol addr in BPF_REG_9 740 */ 741 static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo) 742 { 743 int name_off, len = strlen(relo->name) + 1, res_off; 744 745 name_off = add_data(gen, relo->name, len); 746 res_off = add_data(gen, NULL, 8); /* res is u64 */ 747 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 748 0, 0, 0, name_off)); 749 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 750 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); 751 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE, 752 0, 0, 0, res_off)); 753 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4)); 754 emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name)); 755 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0)); 756 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 757 debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind); 758 } 759 760 /* Expects: 761 * BPF_REG_8 - pointer to instruction 762 * 763 * We need to reuse BTF fd for same symbol otherwise each relocation takes a new 764 * index, while kernel limits total kfunc BTFs to 256. For duplicate symbols, 765 * this would mean a new BTF fd index for each entry. By pairing symbol name 766 * with index, we get the insn->imm, insn->off pairing that kernel uses for 767 * kfunc_tab, which becomes the effective limit even though all of them may 768 * share same index in fd_array (such that kfunc_btf_tab has 1 element). 769 */ 770 static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) 771 { 772 struct ksym_desc *kdesc; 773 int btf_fd_idx; 774 775 kdesc = get_ksym_desc(gen, relo); 776 if (!kdesc) 777 return; 778 /* try to copy from existing bpf_insn */ 779 if (kdesc->ref > 1) { 780 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, 781 kdesc->insn + offsetof(struct bpf_insn, imm)); 782 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2, 783 kdesc->insn + offsetof(struct bpf_insn, off)); 784 goto log; 785 } 786 /* remember insn offset, so we can copy BTF ID and FD later */ 787 kdesc->insn = insn; 788 emit_bpf_find_by_name_kind(gen, relo); 789 if (!relo->is_weak) 790 emit_check_err(gen); 791 /* get index in fd_array to store BTF FD at */ 792 btf_fd_idx = add_kfunc_btf_fd(gen); 793 if (btf_fd_idx > INT16_MAX) { 794 pr_warn("BTF fd off %d for kfunc %s exceeds INT16_MAX, cannot process relocation\n", 795 btf_fd_idx, relo->name); 796 gen->error = -E2BIG; 797 return; 798 } 799 kdesc->off = btf_fd_idx; 800 /* jump to success case */ 801 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); 802 /* set value for imm, off as 0 */ 803 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); 804 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); 805 /* skip success case for ret < 0 */ 806 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10)); 807 /* store btf_id into insn[insn_idx].imm */ 808 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); 809 /* obtain fd in BPF_REG_9 */ 810 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7)); 811 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); 812 /* load fd_array slot pointer */ 813 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 814 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx))); 815 /* store BTF fd in slot, 0 for vmlinux */ 816 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0)); 817 /* jump to insn[insn_idx].off store if fd denotes module BTF */ 818 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2)); 819 /* set the default value for off */ 820 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); 821 /* skip BTF fd store for vmlinux BTF */ 822 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); 823 /* store index into insn[insn_idx].off */ 824 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx)); 825 log: 826 if (!gen->log_level) 827 return; 828 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, 829 offsetof(struct bpf_insn, imm))); 830 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, 831 offsetof(struct bpf_insn, off))); 832 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d", 833 relo->name, kdesc->ref); 834 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 835 0, 0, 0, blob_fd_array_off(gen, kdesc->off))); 836 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0)); 837 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd", 838 relo->name, kdesc->ref); 839 } 840 841 static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo, 842 int ref) 843 { 844 if (!gen->log_level) 845 return; 846 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, 847 offsetof(struct bpf_insn, imm))); 848 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) + 849 offsetof(struct bpf_insn, imm))); 850 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d", 851 relo->is_typeless, relo->is_weak, relo->name, ref); 852 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); 853 debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg", 854 relo->is_typeless, relo->is_weak, relo->name, ref); 855 } 856 857 /* Expects: 858 * BPF_REG_8 - pointer to instruction 859 */ 860 static void emit_relo_ksym_typeless(struct bpf_gen *gen, 861 struct ksym_relo_desc *relo, int insn) 862 { 863 struct ksym_desc *kdesc; 864 865 kdesc = get_ksym_desc(gen, relo); 866 if (!kdesc) 867 return; 868 /* try to copy from existing ldimm64 insn */ 869 if (kdesc->ref > 1) { 870 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, 871 kdesc->insn + offsetof(struct bpf_insn, imm)); 872 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, 873 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)); 874 goto log; 875 } 876 /* remember insn offset, so we can copy ksym addr later */ 877 kdesc->insn = insn; 878 /* skip typeless ksym_desc in fd closing loop in cleanup_relos */ 879 kdesc->typeless = true; 880 emit_bpf_kallsyms_lookup_name(gen, relo); 881 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1)); 882 emit_check_err(gen); 883 /* store lower half of addr into insn[insn_idx].imm */ 884 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm))); 885 /* store upper half of addr into insn[insn_idx + 1].imm */ 886 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); 887 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, 888 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm))); 889 log: 890 emit_ksym_relo_log(gen, relo, kdesc->ref); 891 } 892 893 static __u32 src_reg_mask(struct bpf_gen *gen) 894 { 895 #if defined(__LITTLE_ENDIAN_BITFIELD) /* src_reg,dst_reg,... */ 896 return gen->swapped_endian ? 0xf0 : 0x0f; 897 #elif defined(__BIG_ENDIAN_BITFIELD) /* dst_reg,src_reg,... */ 898 return gen->swapped_endian ? 0x0f : 0xf0; 899 #else 900 #error "Unsupported bit endianness, cannot proceed" 901 #endif 902 } 903 904 /* Expects: 905 * BPF_REG_8 - pointer to instruction 906 */ 907 static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) 908 { 909 struct ksym_desc *kdesc; 910 __u32 reg_mask; 911 912 kdesc = get_ksym_desc(gen, relo); 913 if (!kdesc) 914 return; 915 /* try to copy from existing ldimm64 insn */ 916 if (kdesc->ref > 1) { 917 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, 918 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)); 919 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, 920 kdesc->insn + offsetof(struct bpf_insn, imm)); 921 /* jump over src_reg adjustment if imm (btf_id) is not 0, reuse BPF_REG_0 from move_blob2blob 922 * If btf_id is zero, clear BPF_PSEUDO_BTF_ID flag in src_reg of ld_imm64 insn 923 */ 924 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3)); 925 goto clear_src_reg; 926 } 927 /* remember insn offset, so we can copy BTF ID and FD later */ 928 kdesc->insn = insn; 929 emit_bpf_find_by_name_kind(gen, relo); 930 if (!relo->is_weak) 931 emit_check_err(gen); 932 /* jump to success case */ 933 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); 934 /* set values for insn[insn_idx].imm, insn[insn_idx + 1].imm as 0 */ 935 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); 936 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0)); 937 /* skip success case for ret < 0 */ 938 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4)); 939 /* store btf_id into insn[insn_idx].imm */ 940 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); 941 /* store btf_obj_fd into insn[insn_idx + 1].imm */ 942 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); 943 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, 944 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm))); 945 /* skip src_reg adjustment */ 946 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3)); 947 clear_src_reg: 948 /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */ 949 reg_mask = src_reg_mask(gen); 950 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); 951 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask)); 952 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code))); 953 954 emit_ksym_relo_log(gen, relo, kdesc->ref); 955 } 956 957 void bpf_gen__record_relo_core(struct bpf_gen *gen, 958 const struct bpf_core_relo *core_relo) 959 { 960 struct bpf_core_relo *relos; 961 962 relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos)); 963 if (!relos) { 964 gen->error = -ENOMEM; 965 return; 966 } 967 gen->core_relos = relos; 968 relos += gen->core_relo_cnt; 969 memcpy(relos, core_relo, sizeof(*relos)); 970 gen->core_relo_cnt++; 971 } 972 973 static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns) 974 { 975 int insn; 976 977 pr_debug("gen: emit_relo (%d): %s at %d %s\n", 978 relo->kind, relo->name, relo->insn_idx, relo->is_ld64 ? "ld64" : "call"); 979 insn = insns + sizeof(struct bpf_insn) * relo->insn_idx; 980 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn)); 981 if (relo->is_ld64) { 982 if (relo->is_typeless) 983 emit_relo_ksym_typeless(gen, relo, insn); 984 else 985 emit_relo_ksym_btf(gen, relo, insn); 986 } else { 987 emit_relo_kfunc_btf(gen, relo, insn); 988 } 989 } 990 991 static void emit_relos(struct bpf_gen *gen, int insns) 992 { 993 int i; 994 995 for (i = 0; i < gen->relo_cnt; i++) 996 emit_relo(gen, gen->relos + i, insns); 997 } 998 999 static void cleanup_core_relo(struct bpf_gen *gen) 1000 { 1001 if (!gen->core_relo_cnt) 1002 return; 1003 free(gen->core_relos); 1004 gen->core_relo_cnt = 0; 1005 gen->core_relos = NULL; 1006 } 1007 1008 static void cleanup_relos(struct bpf_gen *gen, int insns) 1009 { 1010 struct ksym_desc *kdesc; 1011 int i, insn; 1012 1013 for (i = 0; i < gen->nr_ksyms; i++) { 1014 kdesc = &gen->ksyms[i]; 1015 /* only close fds for typed ksyms and kfuncs */ 1016 if (kdesc->is_ld64 && !kdesc->typeless) { 1017 /* close fd recorded in insn[insn_idx + 1].imm */ 1018 insn = kdesc->insn; 1019 insn += sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm); 1020 emit_sys_close_blob(gen, insn); 1021 } else if (!kdesc->is_ld64) { 1022 emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off)); 1023 if (kdesc->off < MAX_FD_ARRAY_SZ) 1024 gen->nr_fd_array--; 1025 } 1026 } 1027 if (gen->nr_ksyms) { 1028 free(gen->ksyms); 1029 gen->nr_ksyms = 0; 1030 gen->ksyms = NULL; 1031 } 1032 if (gen->relo_cnt) { 1033 free(gen->relos); 1034 gen->relo_cnt = 0; 1035 gen->relos = NULL; 1036 } 1037 cleanup_core_relo(gen); 1038 } 1039 1040 /* Convert func, line, and core relo info blobs to target endianness */ 1041 static void info_blob_bswap(struct bpf_gen *gen, int func_info, int line_info, 1042 int core_relos, struct bpf_prog_load_opts *load_attr) 1043 { 1044 struct bpf_func_info *fi = gen->data_start + func_info; 1045 struct bpf_line_info *li = gen->data_start + line_info; 1046 struct bpf_core_relo *cr = gen->data_start + core_relos; 1047 int i; 1048 1049 for (i = 0; i < load_attr->func_info_cnt; i++) 1050 bpf_func_info_bswap(fi++); 1051 1052 for (i = 0; i < load_attr->line_info_cnt; i++) 1053 bpf_line_info_bswap(li++); 1054 1055 for (i = 0; i < gen->core_relo_cnt; i++) 1056 bpf_core_relo_bswap(cr++); 1057 } 1058 1059 void bpf_gen__prog_load(struct bpf_gen *gen, 1060 enum bpf_prog_type prog_type, const char *prog_name, 1061 const char *license, struct bpf_insn *insns, size_t insn_cnt, 1062 struct bpf_prog_load_opts *load_attr, int prog_idx) 1063 { 1064 int func_info_tot_sz = load_attr->func_info_cnt * 1065 load_attr->func_info_rec_size; 1066 int line_info_tot_sz = load_attr->line_info_cnt * 1067 load_attr->line_info_rec_size; 1068 int core_relo_tot_sz = gen->core_relo_cnt * 1069 sizeof(struct bpf_core_relo); 1070 int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos; 1071 int attr_size = offsetofend(union bpf_attr, core_relo_rec_size); 1072 union bpf_attr attr; 1073 1074 memset(&attr, 0, attr_size); 1075 /* add license string to blob of bytes */ 1076 license_off = add_data(gen, license, strlen(license) + 1); 1077 /* add insns to blob of bytes */ 1078 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn)); 1079 pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n", 1080 prog_idx, prog_type, insns_off, insn_cnt, license_off); 1081 1082 /* convert blob insns to target endianness */ 1083 if (gen->swapped_endian && !gen->error) { 1084 struct bpf_insn *insn = gen->data_start + insns_off; 1085 int i; 1086 1087 for (i = 0; i < insn_cnt; i++, insn++) 1088 bpf_insn_bswap(insn); 1089 } 1090 1091 attr.prog_type = tgt_endian(prog_type); 1092 attr.expected_attach_type = tgt_endian(load_attr->expected_attach_type); 1093 attr.attach_btf_id = tgt_endian(load_attr->attach_btf_id); 1094 attr.prog_ifindex = tgt_endian(load_attr->prog_ifindex); 1095 attr.kern_version = 0; 1096 attr.insn_cnt = tgt_endian((__u32)insn_cnt); 1097 attr.prog_flags = tgt_endian(load_attr->prog_flags); 1098 1099 attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size); 1100 attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt); 1101 func_info = add_data(gen, load_attr->func_info, func_info_tot_sz); 1102 pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n", 1103 func_info, load_attr->func_info_cnt, 1104 load_attr->func_info_rec_size); 1105 1106 attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size); 1107 attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt); 1108 line_info = add_data(gen, load_attr->line_info, line_info_tot_sz); 1109 pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n", 1110 line_info, load_attr->line_info_cnt, 1111 load_attr->line_info_rec_size); 1112 1113 attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo)); 1114 attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt); 1115 core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz); 1116 pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n", 1117 core_relos, gen->core_relo_cnt, 1118 sizeof(struct bpf_core_relo)); 1119 1120 /* convert all info blobs to target endianness */ 1121 if (gen->swapped_endian && !gen->error) 1122 info_blob_bswap(gen, func_info, line_info, core_relos, load_attr); 1123 1124 libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); 1125 prog_load_attr = add_data(gen, &attr, attr_size); 1126 pr_debug("gen: prog_load: attr: off %d size %d\n", 1127 prog_load_attr, attr_size); 1128 1129 /* populate union bpf_attr with a pointer to license */ 1130 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off); 1131 1132 /* populate union bpf_attr with a pointer to instructions */ 1133 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off); 1134 1135 /* populate union bpf_attr with a pointer to func_info */ 1136 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info); 1137 1138 /* populate union bpf_attr with a pointer to line_info */ 1139 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info); 1140 1141 /* populate union bpf_attr with a pointer to core_relos */ 1142 emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos); 1143 1144 /* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */ 1145 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array); 1146 1147 /* populate union bpf_attr with user provided log details */ 1148 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4, 1149 offsetof(struct bpf_loader_ctx, log_level), false); 1150 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4, 1151 offsetof(struct bpf_loader_ctx, log_size), false); 1152 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8, 1153 offsetof(struct bpf_loader_ctx, log_buf), false); 1154 /* populate union bpf_attr with btf_fd saved in the stack earlier */ 1155 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4, 1156 stack_off(btf_fd)); 1157 if (gen->attach_kind) { 1158 emit_find_attach_target(gen); 1159 /* populate union bpf_attr with btf_id and btf_obj_fd found by helper */ 1160 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 1161 0, 0, 0, prog_load_attr)); 1162 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, 1163 offsetof(union bpf_attr, attach_btf_id))); 1164 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); 1165 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, 1166 offsetof(union bpf_attr, attach_btf_obj_fd))); 1167 } 1168 emit_relos(gen, insns_off); 1169 /* emit PROG_LOAD command */ 1170 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size); 1171 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt); 1172 /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */ 1173 cleanup_relos(gen, insns_off); 1174 if (gen->attach_kind) { 1175 emit_sys_close_blob(gen, 1176 attr_field(prog_load_attr, attach_btf_obj_fd)); 1177 gen->attach_kind = 0; 1178 } 1179 emit_check_err(gen); 1180 /* remember prog_fd in the stack, if successful */ 1181 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, 1182 stack_off(prog_fd[gen->nr_progs]))); 1183 gen->nr_progs++; 1184 } 1185 1186 void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, 1187 __u32 value_size) 1188 { 1189 int attr_size = offsetofend(union bpf_attr, flags); 1190 int map_update_attr, value, key; 1191 union bpf_attr attr; 1192 int zero = 0; 1193 1194 memset(&attr, 0, attr_size); 1195 1196 value = add_data(gen, pvalue, value_size); 1197 key = add_data(gen, &zero, sizeof(zero)); 1198 1199 /* 1200 * if (map_desc[map_idx].initial_value) { 1201 * if (ctx->flags & BPF_SKEL_KERNEL) 1202 * bpf_probe_read_kernel(value, value_size, initial_value); 1203 * else 1204 * bpf_copy_from_user(value, value_size, initial_value); 1205 * } 1206 * 1207 * The runtime initial_value comes from the host-supplied loader 1208 * ctx and would overwrite the blob value after emit_signature_match() 1209 * has already validated map->sha[]. For a signed loader (gen_hash) 1210 * the attested blob value must be authoritative, so skip the override 1211 * and leave the hashed value in place. 1212 */ 1213 if (!OPTS_GET(gen->opts, gen_hash, false)) { 1214 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, 1215 sizeof(struct bpf_loader_ctx) + 1216 sizeof(struct bpf_map_desc) * map_idx + 1217 offsetof(struct bpf_map_desc, initial_value))); 1218 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); 1219 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 1220 0, 0, 0, value)); 1221 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); 1222 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, 1223 offsetof(struct bpf_loader_ctx, flags))); 1224 emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); 1225 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); 1226 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); 1227 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); 1228 } 1229 1230 map_update_attr = add_data(gen, &attr, attr_size); 1231 pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n", 1232 map_idx, value, value_size, map_update_attr, attr_size); 1233 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, 1234 blob_fd_array_off(gen, map_idx)); 1235 emit_rel_store(gen, attr_field(map_update_attr, key), key); 1236 emit_rel_store(gen, attr_field(map_update_attr, value), value); 1237 /* emit MAP_UPDATE_ELEM command */ 1238 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); 1239 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size); 1240 emit_check_err(gen); 1241 } 1242 1243 void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot, 1244 int inner_map_idx) 1245 { 1246 int attr_size = offsetofend(union bpf_attr, flags); 1247 int map_update_attr, key; 1248 union bpf_attr attr; 1249 int tgt_slot; 1250 1251 memset(&attr, 0, attr_size); 1252 1253 tgt_slot = tgt_endian(slot); 1254 key = add_data(gen, &tgt_slot, sizeof(tgt_slot)); 1255 1256 map_update_attr = add_data(gen, &attr, attr_size); 1257 pr_debug("gen: populate_outer_map: outer %d key %d inner %d, attr: off %d size %d\n", 1258 outer_map_idx, slot, inner_map_idx, map_update_attr, attr_size); 1259 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, 1260 blob_fd_array_off(gen, outer_map_idx)); 1261 emit_rel_store(gen, attr_field(map_update_attr, key), key); 1262 emit_rel_store(gen, attr_field(map_update_attr, value), 1263 blob_fd_array_off(gen, inner_map_idx)); 1264 1265 /* emit MAP_UPDATE_ELEM command */ 1266 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); 1267 debug_ret(gen, "populate_outer_map outer %d key %d inner %d", 1268 outer_map_idx, slot, inner_map_idx); 1269 emit_check_err(gen); 1270 } 1271 1272 void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) 1273 { 1274 int attr_size = offsetofend(union bpf_attr, map_fd); 1275 int map_freeze_attr; 1276 union bpf_attr attr; 1277 1278 memset(&attr, 0, attr_size); 1279 map_freeze_attr = add_data(gen, &attr, attr_size); 1280 pr_debug("gen: map_freeze: idx %d, attr: off %d size %d\n", 1281 map_idx, map_freeze_attr, attr_size); 1282 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4, 1283 blob_fd_array_off(gen, map_idx)); 1284 /* emit MAP_FREEZE command */ 1285 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size); 1286 debug_ret(gen, "map_freeze"); 1287 emit_check_err(gen); 1288 } 1289