1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 /* Copyright (c) 2021 Facebook */ 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <string.h> 6 #include <errno.h> 7 #include <asm/byteorder.h> 8 #include <linux/filter.h> 9 #include <sys/param.h> 10 #include "btf.h" 11 #include "bpf.h" 12 #include "libbpf.h" 13 #include "libbpf_internal.h" 14 #include "hashmap.h" 15 #include "bpf_gen_internal.h" 16 #include "skel_internal.h" 17 18 #define MAX_USED_MAPS 64 19 #define MAX_USED_PROGS 32 20 #define MAX_KFUNC_DESCS 256 21 #define MAX_FD_ARRAY_SZ (MAX_USED_MAPS + MAX_KFUNC_DESCS) 22 23 /* The following structure describes the stack layout of the loader program. 24 * In addition R6 contains the pointer to context. 25 * R7 contains the result of the last sys_bpf command (typically error or FD). 26 * R9 contains the result of the last sys_close command. 27 * 28 * Naming convention: 29 * ctx - bpf program context 30 * stack - bpf program stack 31 * blob - bpf_attr-s, strings, insns, map data. 32 * All the bytes that loader prog will use for read/write. 33 */ 34 struct loader_stack { 35 __u32 btf_fd; 36 __u32 inner_map_fd; 37 __u32 prog_fd[MAX_USED_PROGS]; 38 }; 39 40 #define stack_off(field) \ 41 (__s16)(-sizeof(struct loader_stack) + offsetof(struct loader_stack, field)) 42 43 #define attr_field(attr, field) (attr + offsetof(union bpf_attr, field)) 44 45 static int blob_fd_array_off(struct bpf_gen *gen, int index) 46 { 47 return gen->fd_array + index * sizeof(int); 48 } 49 50 static int realloc_insn_buf(struct bpf_gen *gen, __u32 size) 51 { 52 size_t off = gen->insn_cur - gen->insn_start; 53 void *insn_start; 54 55 if (gen->error) 56 return gen->error; 57 if (size > INT32_MAX || off + size > INT32_MAX) { 58 gen->error = -ERANGE; 59 return -ERANGE; 60 } 61 insn_start = realloc(gen->insn_start, off + size); 62 if (!insn_start) { 63 gen->error = -ENOMEM; 64 free(gen->insn_start); 65 gen->insn_start = NULL; 66 gen->insn_cur = NULL; 67 return -ENOMEM; 68 } 69 gen->insn_start = insn_start; 70 gen->insn_cur = insn_start + off; 71 return 0; 72 } 73 74 static int realloc_data_buf(struct bpf_gen *gen, __u32 size) 75 { 76 size_t off = gen->data_cur - gen->data_start; 77 void *data_start; 78 79 if (gen->error) 80 return gen->error; 81 if (size > INT32_MAX || off + size > INT32_MAX) { 82 gen->error = -ERANGE; 83 return -ERANGE; 84 } 85 data_start = realloc(gen->data_start, off + size); 86 if (!data_start) { 87 gen->error = -ENOMEM; 88 free(gen->data_start); 89 gen->data_start = NULL; 90 gen->data_cur = NULL; 91 return -ENOMEM; 92 } 93 gen->data_start = data_start; 94 gen->data_cur = data_start + off; 95 return 0; 96 } 97 98 static void emit(struct bpf_gen *gen, struct bpf_insn insn) 99 { 100 if (realloc_insn_buf(gen, sizeof(insn))) 101 return; 102 memcpy(gen->insn_cur, &insn, sizeof(insn)); 103 gen->insn_cur += sizeof(insn); 104 } 105 106 static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2) 107 { 108 emit(gen, insn1); 109 emit(gen, insn2); 110 } 111 112 static int add_data(struct bpf_gen *gen, const void *data, __u32 size); 113 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off); 114 115 void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) 116 { 117 size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz; 118 int i; 119 120 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int)); 121 gen->log_level = log_level; 122 /* save ctx pointer into R6 */ 123 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1)); 124 125 /* bzero stack */ 126 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10)); 127 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz)); 128 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz)); 129 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); 130 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); 131 132 /* amount of stack actually used, only used to calculate iterations, not stack offset */ 133 nr_progs_sz = offsetof(struct loader_stack, prog_fd[nr_progs]); 134 /* jump over cleanup code */ 135 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 136 /* size of cleanup code below (including map fd cleanup) */ 137 (nr_progs_sz / 4) * 3 + 2 + 138 /* 6 insns for emit_sys_close_blob, 139 * 6 insns for debug_regs in emit_sys_close_blob 140 */ 141 nr_maps * (6 + (gen->log_level ? 6 : 0)))); 142 143 /* remember the label where all error branches will jump to */ 144 gen->cleanup_label = gen->insn_cur - gen->insn_start; 145 /* emit cleanup code: close all temp FDs */ 146 for (i = 0; i < nr_progs_sz; i += 4) { 147 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i)); 148 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1)); 149 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); 150 } 151 for (i = 0; i < nr_maps; i++) 152 emit_sys_close_blob(gen, blob_fd_array_off(gen, i)); 153 /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */ 154 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7)); 155 emit(gen, BPF_EXIT_INSN()); 156 } 157 158 static int add_data(struct bpf_gen *gen, const void *data, __u32 size) 159 { 160 __u64 zero = 0; 161 __u32 size8; 162 void *prev; 163 164 if (size > INT32_MAX) { 165 gen->error = -ERANGE; 166 return 0; 167 } 168 size8 = roundup(size, 8); 169 170 if (realloc_data_buf(gen, size8)) 171 return 0; 172 prev = gen->data_cur; 173 if (data) { 174 memcpy(gen->data_cur, data, size); 175 memcpy(gen->data_cur + size, &zero, size8 - size); 176 } else { 177 memset(gen->data_cur, 0, size8); 178 } 179 gen->data_cur += size8; 180 return prev - gen->data_start; 181 } 182 183 /* Get index for map_fd/btf_fd slot in reserved fd_array, or in data relative 184 * to start of fd_array. Caller can decide if it is usable or not. 185 */ 186 static int add_map_fd(struct bpf_gen *gen) 187 { 188 if (gen->nr_maps == MAX_USED_MAPS) { 189 pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS); 190 gen->error = -E2BIG; 191 return 0; 192 } 193 return gen->nr_maps++; 194 } 195 196 static int add_kfunc_btf_fd(struct bpf_gen *gen) 197 { 198 int cur; 199 200 if (gen->nr_fd_array == MAX_KFUNC_DESCS) { 201 cur = add_data(gen, NULL, sizeof(int)); 202 return (cur - gen->fd_array) / sizeof(int); 203 } 204 return MAX_USED_MAPS + gen->nr_fd_array++; 205 } 206 207 static int insn_bytes_to_bpf_size(__u32 sz) 208 { 209 switch (sz) { 210 case 8: return BPF_DW; 211 case 4: return BPF_W; 212 case 2: return BPF_H; 213 case 1: return BPF_B; 214 default: return -1; 215 } 216 } 217 218 /* *(u64 *)(blob + off) = (u64)(void *)(blob + data) */ 219 static void emit_rel_store(struct bpf_gen *gen, int off, int data) 220 { 221 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 222 0, 0, 0, data)); 223 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 224 0, 0, 0, off)); 225 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0)); 226 } 227 228 static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off) 229 { 230 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, 231 0, 0, 0, blob_off)); 232 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0)); 233 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 234 0, 0, 0, off)); 235 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); 236 } 237 238 static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off) 239 { 240 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 241 0, 0, 0, blob_off)); 242 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0)); 243 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); 244 } 245 246 static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off, 247 bool check_non_zero) 248 { 249 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off)); 250 if (check_non_zero) 251 /* If value in ctx is zero don't update the blob. 252 * For example: when ctx->map.max_entries == 0, keep default max_entries from bpf.c 253 */ 254 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3)); 255 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 256 0, 0, 0, off)); 257 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); 258 } 259 260 static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off) 261 { 262 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); 263 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 264 0, 0, 0, off)); 265 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0)); 266 } 267 268 static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off) 269 { 270 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off)); 271 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off)); 272 } 273 274 static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size) 275 { 276 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd)); 277 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE, 278 0, 0, 0, attr)); 279 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size)); 280 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf)); 281 /* remember the result in R7 */ 282 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 283 } 284 285 static bool is_simm16(__s64 value) 286 { 287 return value == (__s64)(__s16)value; 288 } 289 290 static void emit_check_err(struct bpf_gen *gen) 291 { 292 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1; 293 294 /* R7 contains result of last sys_bpf command. 295 * if (R7 < 0) goto cleanup; 296 */ 297 if (is_simm16(off)) { 298 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off)); 299 } else { 300 gen->error = -ERANGE; 301 } 302 } 303 304 /* reg1 and reg2 should not be R1 - R5. They can be R0, R6 - R10 */ 305 static void emit_debug(struct bpf_gen *gen, int reg1, int reg2, 306 const char *fmt, va_list args) 307 { 308 char buf[1024]; 309 int addr, len, ret; 310 311 if (!gen->log_level) 312 return; 313 ret = vsnprintf(buf, sizeof(buf), fmt, args); 314 if (ret < 1024 - 7 && reg1 >= 0 && reg2 < 0) 315 /* The special case to accommodate common debug_ret(): 316 * to avoid specifying BPF_REG_7 and adding " r=%%d" to 317 * prints explicitly. 318 */ 319 strcat(buf, " r=%d"); 320 len = strlen(buf) + 1; 321 addr = add_data(gen, buf, len); 322 323 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 324 0, 0, 0, addr)); 325 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 326 if (reg1 >= 0) 327 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1)); 328 if (reg2 >= 0) 329 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2)); 330 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk)); 331 } 332 333 static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...) 334 { 335 va_list args; 336 337 va_start(args, fmt); 338 emit_debug(gen, reg1, reg2, fmt, args); 339 va_end(args); 340 } 341 342 static void debug_ret(struct bpf_gen *gen, const char *fmt, ...) 343 { 344 va_list args; 345 346 va_start(args, fmt); 347 emit_debug(gen, BPF_REG_7, -1, fmt, args); 348 va_end(args); 349 } 350 351 static void __emit_sys_close(struct bpf_gen *gen) 352 { 353 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 354 /* 2 is the number of the following insns 355 * * 6 is additional insns in debug_regs 356 */ 357 2 + (gen->log_level ? 6 : 0))); 358 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1)); 359 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close)); 360 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d"); 361 } 362 363 static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off) 364 { 365 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off)); 366 __emit_sys_close(gen); 367 } 368 369 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off) 370 { 371 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 372 0, 0, 0, blob_off)); 373 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0)); 374 __emit_sys_close(gen); 375 } 376 377 int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) 378 { 379 int i; 380 381 if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) { 382 pr_warn("nr_progs %d/%u nr_maps %d/%u mismatch\n", 383 nr_progs, gen->nr_progs, nr_maps, gen->nr_maps); 384 gen->error = -EFAULT; 385 return gen->error; 386 } 387 emit_sys_close_stack(gen, stack_off(btf_fd)); 388 for (i = 0; i < gen->nr_progs; i++) 389 move_stack2ctx(gen, 390 sizeof(struct bpf_loader_ctx) + 391 sizeof(struct bpf_map_desc) * gen->nr_maps + 392 sizeof(struct bpf_prog_desc) * i + 393 offsetof(struct bpf_prog_desc, prog_fd), 4, 394 stack_off(prog_fd[i])); 395 for (i = 0; i < gen->nr_maps; i++) 396 move_blob2ctx(gen, 397 sizeof(struct bpf_loader_ctx) + 398 sizeof(struct bpf_map_desc) * i + 399 offsetof(struct bpf_map_desc, map_fd), 4, 400 blob_fd_array_off(gen, i)); 401 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0)); 402 emit(gen, BPF_EXIT_INSN()); 403 if (!gen->error) { 404 struct gen_loader_opts *opts = gen->opts; 405 406 opts->insns = gen->insn_start; 407 opts->insns_sz = gen->insn_cur - gen->insn_start; 408 opts->data = gen->data_start; 409 opts->data_sz = gen->data_cur - gen->data_start; 410 411 /* use target endianness for embedded loader */ 412 if (gen->swapped_endian) { 413 struct bpf_insn *insn = (struct bpf_insn *)opts->insns; 414 int insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); 415 416 for (i = 0; i < insn_cnt; i++) 417 bpf_insn_bswap(insn++); 418 } 419 } 420 pr_debug("gen: finish %s\n", errstr(gen->error)); 421 return gen->error; 422 } 423 424 void bpf_gen__free(struct bpf_gen *gen) 425 { 426 if (!gen) 427 return; 428 free(gen->data_start); 429 free(gen->insn_start); 430 free(gen); 431 } 432 433 /* 434 * Fields of bpf_attr are set to values in native byte-order before being 435 * written to the target-bound data blob, and may need endian conversion. 436 * This macro allows providing the correct value in situ more simply than 437 * writing a separate converter for *all fields* of *all records* included 438 * in union bpf_attr. Note that sizeof(rval) should match the assignment 439 * target to avoid runtime problems. 440 */ 441 #define tgt_endian(rval) ({ \ 442 typeof(rval) _val = (rval); \ 443 if (gen->swapped_endian) { \ 444 switch (sizeof(_val)) { \ 445 case 1: break; \ 446 case 2: _val = bswap_16(_val); break; \ 447 case 4: _val = bswap_32(_val); break; \ 448 case 8: _val = bswap_64(_val); break; \ 449 default: pr_warn("unsupported bswap size!\n"); \ 450 } \ 451 } \ 452 _val; \ 453 }) 454 455 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, 456 __u32 btf_raw_size) 457 { 458 int attr_size = offsetofend(union bpf_attr, btf_log_level); 459 int btf_data, btf_load_attr; 460 union bpf_attr attr; 461 462 memset(&attr, 0, attr_size); 463 btf_data = add_data(gen, btf_raw_data, btf_raw_size); 464 465 attr.btf_size = tgt_endian(btf_raw_size); 466 btf_load_attr = add_data(gen, &attr, attr_size); 467 pr_debug("gen: load_btf: off %d size %u, attr: off %d size %d\n", 468 btf_data, btf_raw_size, btf_load_attr, attr_size); 469 470 /* populate union bpf_attr with user provided log details */ 471 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4, 472 offsetof(struct bpf_loader_ctx, log_level), false); 473 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4, 474 offsetof(struct bpf_loader_ctx, log_size), false); 475 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8, 476 offsetof(struct bpf_loader_ctx, log_buf), false); 477 /* populate union bpf_attr with a pointer to the BTF data */ 478 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data); 479 /* emit BTF_LOAD command */ 480 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size); 481 debug_ret(gen, "btf_load size %d", btf_raw_size); 482 emit_check_err(gen); 483 /* remember btf_fd in the stack, if successful */ 484 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd))); 485 } 486 487 void bpf_gen__map_create(struct bpf_gen *gen, 488 enum bpf_map_type map_type, 489 const char *map_name, 490 __u32 key_size, __u32 value_size, __u32 max_entries, 491 struct bpf_map_create_opts *map_attr, int map_idx) 492 { 493 int attr_size = offsetofend(union bpf_attr, map_extra); 494 bool close_inner_map_fd = false; 495 int map_create_attr, idx; 496 union bpf_attr attr; 497 498 memset(&attr, 0, attr_size); 499 attr.map_type = tgt_endian(map_type); 500 attr.key_size = tgt_endian(key_size); 501 attr.value_size = tgt_endian(value_size); 502 attr.map_flags = tgt_endian(map_attr->map_flags); 503 attr.map_extra = tgt_endian(map_attr->map_extra); 504 if (map_name) 505 libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); 506 attr.numa_node = tgt_endian(map_attr->numa_node); 507 attr.map_ifindex = tgt_endian(map_attr->map_ifindex); 508 attr.max_entries = tgt_endian(max_entries); 509 attr.btf_key_type_id = tgt_endian(map_attr->btf_key_type_id); 510 attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id); 511 512 map_create_attr = add_data(gen, &attr, attr_size); 513 pr_debug("gen: map_create: %s idx %d type %u value_type_id %u, attr: off %d size %d\n", 514 map_name, map_idx, map_type, map_attr->btf_value_type_id, 515 map_create_attr, attr_size); 516 517 if (map_attr->btf_value_type_id) 518 /* populate union bpf_attr with btf_fd saved in the stack earlier */ 519 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4, 520 stack_off(btf_fd)); 521 switch (map_type) { 522 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 523 case BPF_MAP_TYPE_HASH_OF_MAPS: 524 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4, 525 stack_off(inner_map_fd)); 526 close_inner_map_fd = true; 527 break; 528 default: 529 break; 530 } 531 532 /* 533 * Conditionally update max_entries from the host-supplied loader 534 * ctx. This sizes the map at runtime, but for a signed loader 535 * (gen_hash) it would let an untrusted host re-dimension the 536 * program's maps, outside what the signature attests to: the 537 * metadata blob is covered by the program signature and verified 538 * by the kernel at load time. Keep the signer-provided max_entries 539 * baked into the blob in that case. 540 */ 541 if (map_idx >= 0 && !OPTS_GET(gen->opts, gen_hash, false)) 542 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4, 543 sizeof(struct bpf_loader_ctx) + 544 sizeof(struct bpf_map_desc) * map_idx + 545 offsetof(struct bpf_map_desc, max_entries), 546 true /* check that max_entries != 0 */); 547 548 /* emit MAP_CREATE command */ 549 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size); 550 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d", 551 map_name, map_idx, map_type, value_size, 552 map_attr->btf_value_type_id); 553 emit_check_err(gen); 554 /* remember map_fd in the stack, if successful */ 555 if (map_idx < 0) { 556 /* This bpf_gen__map_create() function is called with map_idx >= 0 557 * for all maps that libbpf loading logic tracks. 558 * It's called with -1 to create an inner map. 559 */ 560 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, 561 stack_off(inner_map_fd))); 562 } else if (map_idx != gen->nr_maps) { 563 gen->error = -EDOM; /* internal bug */ 564 return; 565 } else { 566 /* add_map_fd does gen->nr_maps++ */ 567 idx = add_map_fd(gen); 568 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 569 0, 0, 0, blob_fd_array_off(gen, idx))); 570 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0)); 571 } 572 if (close_inner_map_fd) 573 emit_sys_close_stack(gen, stack_off(inner_map_fd)); 574 } 575 576 void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, 577 enum bpf_attach_type type) 578 { 579 const char *prefix; 580 int kind, ret; 581 582 btf_get_kernel_prefix_kind(type, &prefix, &kind); 583 gen->attach_kind = kind; 584 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s", 585 prefix, attach_name); 586 if (ret >= sizeof(gen->attach_target)) 587 gen->error = -ENOSPC; 588 } 589 590 static void emit_find_attach_target(struct bpf_gen *gen) 591 { 592 int name, len = strlen(gen->attach_target) + 1; 593 594 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind); 595 name = add_data(gen, gen->attach_target, len); 596 597 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 598 0, 0, 0, name)); 599 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 600 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind)); 601 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); 602 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); 603 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 604 debug_ret(gen, "find_by_name_kind(%s,%d)", 605 gen->attach_target, gen->attach_kind); 606 emit_check_err(gen); 607 /* if successful, btf_id is in lower 32-bit of R7 and 608 * btf_obj_fd is in upper 32-bit 609 */ 610 } 611 612 void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, 613 bool is_typeless, bool is_ld64, int kind, int insn_idx) 614 { 615 struct ksym_relo_desc *relo; 616 617 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo)); 618 if (!relo) { 619 gen->error = -ENOMEM; 620 return; 621 } 622 gen->relos = relo; 623 relo += gen->relo_cnt; 624 relo->name = name; 625 relo->is_weak = is_weak; 626 relo->is_typeless = is_typeless; 627 relo->is_ld64 = is_ld64; 628 relo->kind = kind; 629 relo->insn_idx = insn_idx; 630 gen->relo_cnt++; 631 } 632 633 /* returns existing ksym_desc with ref incremented, or inserts a new one */ 634 static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo) 635 { 636 struct ksym_desc *kdesc; 637 int i; 638 639 for (i = 0; i < gen->nr_ksyms; i++) { 640 kdesc = &gen->ksyms[i]; 641 if (kdesc->kind == relo->kind && kdesc->is_ld64 == relo->is_ld64 && 642 !strcmp(kdesc->name, relo->name)) { 643 kdesc->ref++; 644 return kdesc; 645 } 646 } 647 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc)); 648 if (!kdesc) { 649 gen->error = -ENOMEM; 650 return NULL; 651 } 652 gen->ksyms = kdesc; 653 kdesc = &gen->ksyms[gen->nr_ksyms++]; 654 kdesc->name = relo->name; 655 kdesc->kind = relo->kind; 656 kdesc->ref = 1; 657 kdesc->off = 0; 658 kdesc->insn = 0; 659 kdesc->is_ld64 = relo->is_ld64; 660 return kdesc; 661 } 662 663 /* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7} 664 * Returns result in BPF_REG_7 665 */ 666 static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo) 667 { 668 int name_off, len = strlen(relo->name) + 1; 669 670 name_off = add_data(gen, relo->name, len); 671 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 672 0, 0, 0, name_off)); 673 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 674 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind)); 675 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0)); 676 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind)); 677 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 678 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind); 679 } 680 681 /* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7} 682 * Returns result in BPF_REG_7 683 * Returns u64 symbol addr in BPF_REG_9 684 */ 685 static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo) 686 { 687 int name_off, len = strlen(relo->name) + 1, res_off; 688 689 name_off = add_data(gen, relo->name, len); 690 res_off = add_data(gen, NULL, 8); /* res is u64 */ 691 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 692 0, 0, 0, name_off)); 693 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len)); 694 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0)); 695 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE, 696 0, 0, 0, res_off)); 697 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4)); 698 emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name)); 699 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0)); 700 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0)); 701 debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind); 702 } 703 704 /* Expects: 705 * BPF_REG_8 - pointer to instruction 706 * 707 * We need to reuse BTF fd for same symbol otherwise each relocation takes a new 708 * index, while kernel limits total kfunc BTFs to 256. For duplicate symbols, 709 * this would mean a new BTF fd index for each entry. By pairing symbol name 710 * with index, we get the insn->imm, insn->off pairing that kernel uses for 711 * kfunc_tab, which becomes the effective limit even though all of them may 712 * share same index in fd_array (such that kfunc_btf_tab has 1 element). 713 */ 714 static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) 715 { 716 struct ksym_desc *kdesc; 717 int btf_fd_idx; 718 719 kdesc = get_ksym_desc(gen, relo); 720 if (!kdesc) 721 return; 722 /* try to copy from existing bpf_insn */ 723 if (kdesc->ref > 1) { 724 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, 725 kdesc->insn + offsetof(struct bpf_insn, imm)); 726 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2, 727 kdesc->insn + offsetof(struct bpf_insn, off)); 728 goto log; 729 } 730 /* remember insn offset, so we can copy BTF ID and FD later */ 731 kdesc->insn = insn; 732 emit_bpf_find_by_name_kind(gen, relo); 733 if (!relo->is_weak) 734 emit_check_err(gen); 735 /* get index in fd_array to store BTF FD at */ 736 btf_fd_idx = add_kfunc_btf_fd(gen); 737 if (btf_fd_idx > INT16_MAX) { 738 pr_warn("BTF fd off %d for kfunc %s exceeds INT16_MAX, cannot process relocation\n", 739 btf_fd_idx, relo->name); 740 gen->error = -E2BIG; 741 return; 742 } 743 kdesc->off = btf_fd_idx; 744 /* jump to success case */ 745 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); 746 /* set value for imm, off as 0 */ 747 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); 748 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); 749 /* skip success case for ret < 0 */ 750 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10)); 751 /* store btf_id into insn[insn_idx].imm */ 752 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); 753 /* obtain fd in BPF_REG_9 */ 754 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7)); 755 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); 756 /* load fd_array slot pointer */ 757 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 758 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx))); 759 /* store BTF fd in slot, 0 for vmlinux */ 760 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0)); 761 /* jump to insn[insn_idx].off store if fd denotes module BTF */ 762 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2)); 763 /* set the default value for off */ 764 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0)); 765 /* skip BTF fd store for vmlinux BTF */ 766 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); 767 /* store index into insn[insn_idx].off */ 768 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx)); 769 log: 770 if (!gen->log_level) 771 return; 772 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, 773 offsetof(struct bpf_insn, imm))); 774 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, 775 offsetof(struct bpf_insn, off))); 776 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d", 777 relo->name, kdesc->ref); 778 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 779 0, 0, 0, blob_fd_array_off(gen, kdesc->off))); 780 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0)); 781 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd", 782 relo->name, kdesc->ref); 783 } 784 785 static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo, 786 int ref) 787 { 788 if (!gen->log_level) 789 return; 790 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8, 791 offsetof(struct bpf_insn, imm))); 792 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) + 793 offsetof(struct bpf_insn, imm))); 794 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d", 795 relo->is_typeless, relo->is_weak, relo->name, ref); 796 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); 797 debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg", 798 relo->is_typeless, relo->is_weak, relo->name, ref); 799 } 800 801 /* Expects: 802 * BPF_REG_8 - pointer to instruction 803 */ 804 static void emit_relo_ksym_typeless(struct bpf_gen *gen, 805 struct ksym_relo_desc *relo, int insn) 806 { 807 struct ksym_desc *kdesc; 808 809 kdesc = get_ksym_desc(gen, relo); 810 if (!kdesc) 811 return; 812 /* try to copy from existing ldimm64 insn */ 813 if (kdesc->ref > 1) { 814 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, 815 kdesc->insn + offsetof(struct bpf_insn, imm)); 816 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, 817 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)); 818 goto log; 819 } 820 /* remember insn offset, so we can copy ksym addr later */ 821 kdesc->insn = insn; 822 /* skip typeless ksym_desc in fd closing loop in cleanup_relos */ 823 kdesc->typeless = true; 824 emit_bpf_kallsyms_lookup_name(gen, relo); 825 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1)); 826 emit_check_err(gen); 827 /* store lower half of addr into insn[insn_idx].imm */ 828 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm))); 829 /* store upper half of addr into insn[insn_idx + 1].imm */ 830 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32)); 831 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, 832 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm))); 833 log: 834 emit_ksym_relo_log(gen, relo, kdesc->ref); 835 } 836 837 static __u32 src_reg_mask(struct bpf_gen *gen) 838 { 839 #if defined(__LITTLE_ENDIAN_BITFIELD) /* src_reg,dst_reg,... */ 840 return gen->swapped_endian ? 0xf0 : 0x0f; 841 #elif defined(__BIG_ENDIAN_BITFIELD) /* dst_reg,src_reg,... */ 842 return gen->swapped_endian ? 0x0f : 0xf0; 843 #else 844 #error "Unsupported bit endianness, cannot proceed" 845 #endif 846 } 847 848 /* Expects: 849 * BPF_REG_8 - pointer to instruction 850 */ 851 static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn) 852 { 853 struct ksym_desc *kdesc; 854 __u32 reg_mask; 855 856 kdesc = get_ksym_desc(gen, relo); 857 if (!kdesc) 858 return; 859 /* try to copy from existing ldimm64 insn */ 860 if (kdesc->ref > 1) { 861 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4, 862 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)); 863 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4, 864 kdesc->insn + offsetof(struct bpf_insn, imm)); 865 /* jump over src_reg adjustment if imm (btf_id) is not 0, reuse BPF_REG_0 from move_blob2blob 866 * If btf_id is zero, clear BPF_PSEUDO_BTF_ID flag in src_reg of ld_imm64 insn 867 */ 868 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3)); 869 goto clear_src_reg; 870 } 871 /* remember insn offset, so we can copy BTF ID and FD later */ 872 kdesc->insn = insn; 873 emit_bpf_find_by_name_kind(gen, relo); 874 if (!relo->is_weak) 875 emit_check_err(gen); 876 /* jump to success case */ 877 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3)); 878 /* set values for insn[insn_idx].imm, insn[insn_idx + 1].imm as 0 */ 879 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0)); 880 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0)); 881 /* skip success case for ret < 0 */ 882 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4)); 883 /* store btf_id into insn[insn_idx].imm */ 884 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm))); 885 /* store btf_obj_fd into insn[insn_idx + 1].imm */ 886 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); 887 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, 888 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm))); 889 /* skip src_reg adjustment */ 890 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3)); 891 clear_src_reg: 892 /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */ 893 reg_mask = src_reg_mask(gen); 894 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code))); 895 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask)); 896 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code))); 897 898 emit_ksym_relo_log(gen, relo, kdesc->ref); 899 } 900 901 void bpf_gen__record_relo_core(struct bpf_gen *gen, 902 const struct bpf_core_relo *core_relo) 903 { 904 struct bpf_core_relo *relos; 905 906 relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos)); 907 if (!relos) { 908 gen->error = -ENOMEM; 909 return; 910 } 911 gen->core_relos = relos; 912 relos += gen->core_relo_cnt; 913 memcpy(relos, core_relo, sizeof(*relos)); 914 gen->core_relo_cnt++; 915 } 916 917 static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns) 918 { 919 int insn; 920 921 pr_debug("gen: emit_relo (%d): %s at %d %s\n", 922 relo->kind, relo->name, relo->insn_idx, relo->is_ld64 ? "ld64" : "call"); 923 insn = insns + sizeof(struct bpf_insn) * relo->insn_idx; 924 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn)); 925 if (relo->is_ld64) { 926 if (relo->is_typeless) 927 emit_relo_ksym_typeless(gen, relo, insn); 928 else 929 emit_relo_ksym_btf(gen, relo, insn); 930 } else { 931 emit_relo_kfunc_btf(gen, relo, insn); 932 } 933 } 934 935 static void emit_relos(struct bpf_gen *gen, int insns) 936 { 937 int i; 938 939 for (i = 0; i < gen->relo_cnt; i++) 940 emit_relo(gen, gen->relos + i, insns); 941 } 942 943 static void cleanup_core_relo(struct bpf_gen *gen) 944 { 945 if (!gen->core_relo_cnt) 946 return; 947 free(gen->core_relos); 948 gen->core_relo_cnt = 0; 949 gen->core_relos = NULL; 950 } 951 952 static void cleanup_relos(struct bpf_gen *gen, int insns) 953 { 954 struct ksym_desc *kdesc; 955 int i, insn; 956 957 for (i = 0; i < gen->nr_ksyms; i++) { 958 kdesc = &gen->ksyms[i]; 959 /* only close fds for typed ksyms and kfuncs */ 960 if (kdesc->is_ld64 && !kdesc->typeless) { 961 /* close fd recorded in insn[insn_idx + 1].imm */ 962 insn = kdesc->insn; 963 insn += sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm); 964 emit_sys_close_blob(gen, insn); 965 } else if (!kdesc->is_ld64) { 966 emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off)); 967 if (kdesc->off < MAX_FD_ARRAY_SZ) 968 gen->nr_fd_array--; 969 } 970 } 971 if (gen->nr_ksyms) { 972 free(gen->ksyms); 973 gen->nr_ksyms = 0; 974 gen->ksyms = NULL; 975 } 976 if (gen->relo_cnt) { 977 free(gen->relos); 978 gen->relo_cnt = 0; 979 gen->relos = NULL; 980 } 981 cleanup_core_relo(gen); 982 } 983 984 /* Convert func, line, and core relo info blobs to target endianness */ 985 static void info_blob_bswap(struct bpf_gen *gen, int func_info, int line_info, 986 int core_relos, struct bpf_prog_load_opts *load_attr) 987 { 988 struct bpf_func_info *fi = gen->data_start + func_info; 989 struct bpf_line_info *li = gen->data_start + line_info; 990 struct bpf_core_relo *cr = gen->data_start + core_relos; 991 int i; 992 993 for (i = 0; i < load_attr->func_info_cnt; i++) 994 bpf_func_info_bswap(fi++); 995 996 for (i = 0; i < load_attr->line_info_cnt; i++) 997 bpf_line_info_bswap(li++); 998 999 for (i = 0; i < gen->core_relo_cnt; i++) 1000 bpf_core_relo_bswap(cr++); 1001 } 1002 1003 void bpf_gen__prog_load(struct bpf_gen *gen, 1004 enum bpf_prog_type prog_type, const char *prog_name, 1005 const char *license, struct bpf_insn *insns, size_t insn_cnt, 1006 struct bpf_prog_load_opts *load_attr, int prog_idx) 1007 { 1008 int func_info_tot_sz = load_attr->func_info_cnt * 1009 load_attr->func_info_rec_size; 1010 int line_info_tot_sz = load_attr->line_info_cnt * 1011 load_attr->line_info_rec_size; 1012 int core_relo_tot_sz = gen->core_relo_cnt * 1013 sizeof(struct bpf_core_relo); 1014 int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos; 1015 int attr_size = offsetofend(union bpf_attr, core_relo_rec_size); 1016 union bpf_attr attr; 1017 1018 memset(&attr, 0, attr_size); 1019 /* add license string to blob of bytes */ 1020 license_off = add_data(gen, license, strlen(license) + 1); 1021 /* add insns to blob of bytes */ 1022 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn)); 1023 pr_debug("gen: prog_load: prog_idx %d type %u insn off %d insns_cnt %zu license off %d\n", 1024 prog_idx, prog_type, insns_off, insn_cnt, license_off); 1025 1026 /* convert blob insns to target endianness */ 1027 if (gen->swapped_endian && !gen->error) { 1028 struct bpf_insn *insn = gen->data_start + insns_off; 1029 int i; 1030 1031 for (i = 0; i < insn_cnt; i++, insn++) 1032 bpf_insn_bswap(insn); 1033 } 1034 1035 attr.prog_type = tgt_endian(prog_type); 1036 attr.expected_attach_type = tgt_endian(load_attr->expected_attach_type); 1037 attr.attach_btf_id = tgt_endian(load_attr->attach_btf_id); 1038 attr.prog_ifindex = tgt_endian(load_attr->prog_ifindex); 1039 attr.kern_version = 0; 1040 attr.insn_cnt = tgt_endian((__u32)insn_cnt); 1041 attr.prog_flags = tgt_endian(load_attr->prog_flags); 1042 1043 attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size); 1044 attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt); 1045 func_info = add_data(gen, load_attr->func_info, func_info_tot_sz); 1046 pr_debug("gen: prog_load: func_info: off %d cnt %u rec size %u\n", 1047 func_info, load_attr->func_info_cnt, 1048 load_attr->func_info_rec_size); 1049 1050 attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size); 1051 attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt); 1052 line_info = add_data(gen, load_attr->line_info, line_info_tot_sz); 1053 pr_debug("gen: prog_load: line_info: off %d cnt %u rec size %u\n", 1054 line_info, load_attr->line_info_cnt, 1055 load_attr->line_info_rec_size); 1056 1057 attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo)); 1058 attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt); 1059 core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz); 1060 pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zu\n", 1061 core_relos, gen->core_relo_cnt, 1062 sizeof(struct bpf_core_relo)); 1063 1064 /* convert all info blobs to target endianness */ 1065 if (gen->swapped_endian && !gen->error) 1066 info_blob_bswap(gen, func_info, line_info, core_relos, load_attr); 1067 1068 libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); 1069 prog_load_attr = add_data(gen, &attr, attr_size); 1070 pr_debug("gen: prog_load: attr: off %d size %d\n", 1071 prog_load_attr, attr_size); 1072 1073 /* populate union bpf_attr with a pointer to license */ 1074 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off); 1075 1076 /* populate union bpf_attr with a pointer to instructions */ 1077 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off); 1078 1079 /* populate union bpf_attr with a pointer to func_info */ 1080 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info); 1081 1082 /* populate union bpf_attr with a pointer to line_info */ 1083 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info); 1084 1085 /* populate union bpf_attr with a pointer to core_relos */ 1086 emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos); 1087 1088 /* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */ 1089 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array); 1090 1091 /* populate union bpf_attr with user provided log details */ 1092 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4, 1093 offsetof(struct bpf_loader_ctx, log_level), false); 1094 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4, 1095 offsetof(struct bpf_loader_ctx, log_size), false); 1096 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8, 1097 offsetof(struct bpf_loader_ctx, log_buf), false); 1098 /* populate union bpf_attr with btf_fd saved in the stack earlier */ 1099 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4, 1100 stack_off(btf_fd)); 1101 if (gen->attach_kind) { 1102 emit_find_attach_target(gen); 1103 /* populate union bpf_attr with btf_id and btf_obj_fd found by helper */ 1104 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE, 1105 0, 0, 0, prog_load_attr)); 1106 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, 1107 offsetof(union bpf_attr, attach_btf_id))); 1108 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32)); 1109 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, 1110 offsetof(union bpf_attr, attach_btf_obj_fd))); 1111 } 1112 emit_relos(gen, insns_off); 1113 /* emit PROG_LOAD command */ 1114 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size); 1115 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt); 1116 /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */ 1117 cleanup_relos(gen, insns_off); 1118 if (gen->attach_kind) { 1119 emit_sys_close_blob(gen, 1120 attr_field(prog_load_attr, attach_btf_obj_fd)); 1121 gen->attach_kind = 0; 1122 } 1123 emit_check_err(gen); 1124 /* remember prog_fd in the stack, if successful */ 1125 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, 1126 stack_off(prog_fd[gen->nr_progs]))); 1127 gen->nr_progs++; 1128 } 1129 1130 void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, 1131 __u32 value_size, __u64 flags) 1132 { 1133 int attr_size = offsetofend(union bpf_attr, flags); 1134 int map_update_attr, value, key; 1135 union bpf_attr attr; 1136 int zero = 0; 1137 1138 memset(&attr, 0, attr_size); 1139 attr.flags = tgt_endian(flags); 1140 1141 value = add_data(gen, pvalue, value_size); 1142 key = add_data(gen, &zero, sizeof(zero)); 1143 1144 /* 1145 * if (map_desc[map_idx].initial_value) { 1146 * if (ctx->flags & BPF_SKEL_KERNEL) 1147 * bpf_probe_read_kernel(value, value_size, initial_value); 1148 * else 1149 * bpf_copy_from_user(value, value_size, initial_value); 1150 * } 1151 * 1152 * The runtime initial_value comes from the host-supplied loader 1153 * ctx and would overwrite the blob value that the program signature 1154 * covers and the kernel verifies at load time. For a signed loader 1155 * (gen_hash) the attested blob value must be authoritative, so skip 1156 * the override and leave the signed value in place. 1157 */ 1158 if (!OPTS_GET(gen->opts, gen_hash, false)) { 1159 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, 1160 sizeof(struct bpf_loader_ctx) + 1161 sizeof(struct bpf_map_desc) * map_idx + 1162 offsetof(struct bpf_map_desc, initial_value))); 1163 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8)); 1164 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE, 1165 0, 0, 0, value)); 1166 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size)); 1167 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, 1168 offsetof(struct bpf_loader_ctx, flags))); 1169 emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2)); 1170 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user)); 1171 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1)); 1172 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel)); 1173 } 1174 1175 map_update_attr = add_data(gen, &attr, attr_size); 1176 pr_debug("gen: map_update_elem: idx %d, value: off %d size %u, attr: off %d size %d\n", 1177 map_idx, value, value_size, map_update_attr, attr_size); 1178 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, 1179 blob_fd_array_off(gen, map_idx)); 1180 emit_rel_store(gen, attr_field(map_update_attr, key), key); 1181 emit_rel_store(gen, attr_field(map_update_attr, value), value); 1182 /* emit MAP_UPDATE_ELEM command */ 1183 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); 1184 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size); 1185 emit_check_err(gen); 1186 } 1187 1188 void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot, 1189 int inner_map_idx) 1190 { 1191 int attr_size = offsetofend(union bpf_attr, flags); 1192 int map_update_attr, key; 1193 union bpf_attr attr; 1194 int tgt_slot; 1195 1196 memset(&attr, 0, attr_size); 1197 1198 tgt_slot = tgt_endian(slot); 1199 key = add_data(gen, &tgt_slot, sizeof(tgt_slot)); 1200 1201 map_update_attr = add_data(gen, &attr, attr_size); 1202 pr_debug("gen: populate_outer_map: outer %d key %d inner %d, attr: off %d size %d\n", 1203 outer_map_idx, slot, inner_map_idx, map_update_attr, attr_size); 1204 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, 1205 blob_fd_array_off(gen, outer_map_idx)); 1206 emit_rel_store(gen, attr_field(map_update_attr, key), key); 1207 emit_rel_store(gen, attr_field(map_update_attr, value), 1208 blob_fd_array_off(gen, inner_map_idx)); 1209 1210 /* emit MAP_UPDATE_ELEM command */ 1211 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size); 1212 debug_ret(gen, "populate_outer_map outer %d key %d inner %d", 1213 outer_map_idx, slot, inner_map_idx); 1214 emit_check_err(gen); 1215 } 1216 1217 void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) 1218 { 1219 int attr_size = offsetofend(union bpf_attr, map_fd); 1220 int map_freeze_attr; 1221 union bpf_attr attr; 1222 1223 memset(&attr, 0, attr_size); 1224 map_freeze_attr = add_data(gen, &attr, attr_size); 1225 pr_debug("gen: map_freeze: idx %d, attr: off %d size %d\n", 1226 map_idx, map_freeze_attr, attr_size); 1227 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4, 1228 blob_fd_array_off(gen, map_idx)); 1229 /* emit MAP_FREEZE command */ 1230 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size); 1231 debug_ret(gen, "map_freeze"); 1232 emit_check_err(gen); 1233 } 1234