Lines Matching full:gen
45 static int blob_fd_array_off(struct bpf_gen *gen, int index)
47 return gen->fd_array + index * sizeof(int);
50 static int realloc_insn_buf(struct bpf_gen *gen, __u32 size)
52 size_t off = gen->insn_cur - gen->insn_start;
55 if (gen->error)
56 return gen->error;
58 gen->error = -ERANGE;
61 insn_start = realloc(gen->insn_start, off + size);
63 gen->error = -ENOMEM;
64 free(gen->insn_start);
65 gen->insn_start = NULL;
66 gen->insn_cur = NULL;
69 gen->insn_start = insn_start;
70 gen->insn_cur = insn_start + off;
74 static int realloc_data_buf(struct bpf_gen *gen, __u32 size)
76 size_t off = gen->data_cur - gen->data_start;
79 if (gen->error)
80 return gen->error;
82 gen->error = -ERANGE;
85 data_start = realloc(gen->data_start, off + size);
87 gen->error = -ENOMEM;
88 free(gen->data_start);
89 gen->data_start = NULL;
90 gen->data_cur = NULL;
93 gen->data_start = data_start;
94 gen->data_cur = data_start + off;
98 static void emit(struct bpf_gen *gen, struct bpf_insn insn)
100 if (realloc_insn_buf(gen, sizeof(insn)))
102 memcpy(gen->insn_cur, &insn, sizeof(insn));
103 gen->insn_cur += sizeof(insn);
106 static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2)
108 emit(gen, insn1);
109 emit(gen, insn2);
112 static int add_data(struct bpf_gen *gen, const void *data, __u32 size);
113 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off);
114 static void emit_signature_match(struct bpf_gen *gen);
116 void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps)
121 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
122 gen->log_level = log_level;
124 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1));
127 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
128 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz));
129 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz));
130 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
131 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
136 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0,
142 nr_maps * (6 + (gen->log_level ? 6 : 0))));
145 gen->cleanup_label = gen->insn_cur - gen->insn_start;
148 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i));
149 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1));
150 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
153 emit_sys_close_blob(gen, blob_fd_array_off(gen, i));
155 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));
156 emit(gen, BPF_EXIT_INSN());
157 if (OPTS_GET(gen->opts, gen_hash, false))
158 emit_signature_match(gen);
161 static int add_data(struct bpf_gen *gen, const void *data, __u32 size)
168 gen->error = -ERANGE;
173 if (realloc_data_buf(gen, size8))
175 prev = gen->data_cur;
177 memcpy(gen->data_cur, data, size);
178 memcpy(gen->data_cur + size, &zero, size8 - size);
180 memset(gen->data_cur, 0, size8);
182 gen->data_cur += size8;
183 return prev - gen->data_start;
189 static int add_map_fd(struct bpf_gen *gen)
191 if (gen->nr_maps == MAX_USED_MAPS) {
193 gen->error = -E2BIG;
196 return gen->nr_maps++;
199 static int add_kfunc_btf_fd(struct bpf_gen *gen)
203 if (gen->nr_fd_array == MAX_KFUNC_DESCS) {
204 cur = add_data(gen, NULL, sizeof(int));
205 return (cur - gen->fd_array) / sizeof(int);
207 return MAX_USED_MAPS + gen->nr_fd_array++;
222 static void emit_rel_store(struct bpf_gen *gen, int off, int data)
224 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
226 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
228 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
231 static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off)
233 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
235 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0));
236 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
238 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
241 static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off)
243 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
245 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0));
246 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
249 static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,
252 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off));
257 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3));
258 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
260 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
263 static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off)
265 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
266 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
268 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
271 static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off)
273 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
274 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
277 static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size)
279 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd));
280 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
282 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size));
283 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf));
285 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
293 static void emit_check_err(struct bpf_gen *gen)
295 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1;
301 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off));
303 gen->error = -ERANGE;
308 static void emit_debug(struct bpf_gen *gen, int reg1, int reg2,
314 if (!gen->log_level)
324 addr = add_data(gen, buf, len);
326 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
328 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
330 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1));
332 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2));
333 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk));
336 static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...)
341 emit_debug(gen, reg1, reg2, fmt, args);
345 static void debug_ret(struct bpf_gen *gen, const char *fmt, ...)
350 emit_debug(gen, BPF_REG_7, -1, fmt, args);
354 static void __emit_sys_close(struct bpf_gen *gen)
356 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0,
360 2 + (gen->log_level ? 6 : 0)));
361 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1));
362 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
363 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d");
366 static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off)
368 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off));
369 __emit_sys_close(gen);
372 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off)
374 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
376 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0));
377 __emit_sys_close(gen);
380 static void compute_sha_update_offsets(struct bpf_gen *gen);
382 int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps)
386 if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) {
388 nr_progs, gen->nr_progs, nr_maps, gen->nr_maps);
389 gen->error = -EFAULT;
390 return gen->error;
392 emit_sys_close_stack(gen, stack_off(btf_fd));
393 for (i = 0; i < gen->nr_progs; i++)
394 move_stack2ctx(gen,
396 sizeof(struct bpf_map_desc) * gen->nr_maps +
400 for (i = 0; i < gen->nr_maps; i++)
401 move_blob2ctx(gen,
405 blob_fd_array_off(gen, i));
406 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
407 emit(gen, BPF_EXIT_INSN());
408 if (!gen->error) {
409 struct gen_loader_opts *opts = gen->opts;
412 compute_sha_update_offsets(gen);
414 opts->insns = gen->insn_start;
415 opts->insns_sz = gen->insn_cur - gen->insn_start;
416 opts->data = gen->data_start;
417 opts->data_sz = gen->data_cur - gen->data_start;
420 if (gen->swapped_endian) {
428 pr_debug("gen: finish %s\n", errstr(gen->error));
429 return gen->error;
432 void bpf_gen__free(struct bpf_gen *gen)
434 if (!gen)
436 free(gen->data_start);
437 free(gen->insn_start);
438 free(gen);
451 if (gen->swapped_endian) { \
463 static void compute_sha_update_offsets(struct bpf_gen *gen)
469 libbpf_sha256(gen->data_start, gen->data_cur - gen->data_start, (__u8 *)sha);
472 (struct bpf_insn *)(gen->insn_start + gen->hash_insn_offset[i]);
479 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,
487 btf_data = add_data(gen, btf_raw_data, btf_raw_size);
490 btf_load_attr = add_data(gen, &attr, attr_size);
491 pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n",
495 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4,
497 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4,
499 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8,
502 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data);
504 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size);
505 debug_ret(gen, "btf_load size %d", btf_raw_size);
506 emit_check_err(gen);
508 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd)));
511 void bpf_gen__map_create(struct bpf_gen *gen,
536 map_create_attr = add_data(gen, &attr, attr_size);
537 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n",
543 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4,
548 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4,
564 if (map_idx >= 0 && !OPTS_GET(gen->opts, gen_hash, false))
565 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4,
572 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size);
573 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d",
576 emit_check_err(gen);
583 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
585 } else if (map_idx != gen->nr_maps) {
586 gen->error = -EDOM; /* internal bug */
589 /* add_map_fd does gen->nr_maps++ */
590 idx = add_map_fd(gen);
591 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
592 0, 0, 0, blob_fd_array_off(gen, idx)));
593 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0));
596 emit_sys_close_stack(gen, stack_off(inner_map_fd));
599 static void emit_signature_match(struct bpf_gen *gen)
610 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX,
612 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, SHA256_DIGEST_LENGTH));
613 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2;
615 emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL));
616 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 1, off));
618 gen->error = -ERANGE;
622 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX,
624 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, i * sizeof(__u64)));
625 gen->hash_insn_offset[i] = gen->insn_cur - gen->insn_start;
626 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_3, 0, 0, 0, 0, 0));
628 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2;
630 emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL));
631 emit(gen, BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, off));
633 gen->error = -ERANGE;
638 void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name,
645 gen->attach_kind = kind;
646 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s",
648 if (ret >= sizeof(gen->attach_target))
649 gen->error = -ENOSPC;
652 static void emit_find_attach_target(struct bpf_gen *gen)
654 int name, len = strlen(gen->attach_target) + 1;
656 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind);
657 name = add_data(gen, gen->attach_target, len);
659 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
661 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
662 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind));
663 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
664 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
665 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
666 debug_ret(gen, "find_by_name_kind(%s,%d)",
667 gen->attach_target, gen->attach_kind);
668 emit_check_err(gen);
674 void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak,
679 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo));
681 gen->error = -ENOMEM;
684 gen->relos = relo;
685 relo += gen->relo_cnt;
692 gen->relo_cnt++;
696 static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo)
701 for (i = 0; i < gen->nr_ksyms; i++) {
702 kdesc = &gen->ksyms[i];
709 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc));
711 gen->error = -ENOMEM;
714 gen->ksyms = kdesc;
715 kdesc = &gen->ksyms[gen->nr_ksyms++];
728 static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo)
732 name_off = add_data(gen, relo->name, len);
733 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
735 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
736 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));
737 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
738 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
739 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
740 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);
747 static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo)
751 name_off = add_data(gen, relo->name, len);
752 res_off = add_data(gen, NULL, 8); /* res is u64 */
753 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
755 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
756 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
757 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE,
759 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4));
760 emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name));
761 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0));
762 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
763 debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind);
776 static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
781 kdesc = get_ksym_desc(gen, relo);
786 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
788 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2,
794 emit_bpf_find_by_name_kind(gen, relo);
796 emit_check_err(gen);
798 btf_fd_idx = add_kfunc_btf_fd(gen);
802 gen->error = -E2BIG;
807 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
809 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
810 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
812 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10));
814 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
816 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7));
817 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
819 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
820 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx)));
822 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0));
824 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2));
826 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
828 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
830 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx));
832 if (!gen->log_level)
834 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
836 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8,
838 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d",
840 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
841 0, 0, 0, blob_fd_array_off(gen, kdesc->off)));
842 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0));
843 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd",
847 static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo,
850 if (!gen->log_level)
852 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
854 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) +
856 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d",
858 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
859 debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg",
866 static void emit_relo_ksym_typeless(struct bpf_gen *gen,
871 kdesc = get_ksym_desc(gen, relo);
876 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
878 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
886 emit_bpf_kallsyms_lookup_name(gen, relo);
887 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1));
888 emit_check_err(gen);
890 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm)));
892 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
893 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9,
896 emit_ksym_relo_log(gen, relo, kdesc->ref);
899 static __u32 src_reg_mask(struct bpf_gen *gen)
902 return gen->swapped_endian ? 0xf0 : 0x0f;
904 return gen->swapped_endian ? 0x0f : 0xf0;
913 static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
918 kdesc = get_ksym_desc(gen, relo);
923 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
925 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
930 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3));
935 emit_bpf_find_by_name_kind(gen, relo);
937 emit_check_err(gen);
939 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
941 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
942 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0));
944 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4));
946 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
948 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
949 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7,
952 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3));
955 reg_mask = src_reg_mask(gen);
956 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
957 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask));
958 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code)));
960 emit_ksym_relo_log(gen, relo, kdesc->ref);
963 void bpf_gen__record_relo_core(struct bpf_gen *gen,
968 relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos));
970 gen->error = -ENOMEM;
973 gen->core_relos = relos;
974 relos += gen->core_relo_cnt;
976 gen->core_relo_cnt++;
979 static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)
983 pr_debug("gen: emit_relo (%d): %s at %d %s\n",
986 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn));
989 emit_relo_ksym_typeless(gen, relo, insn);
991 emit_relo_ksym_btf(gen, relo, insn);
993 emit_relo_kfunc_btf(gen, relo, insn);
997 static void emit_relos(struct bpf_gen *gen, int insns)
1001 for (i = 0; i < gen->relo_cnt; i++)
1002 emit_relo(gen, gen->relos + i, insns);
1005 static void cleanup_core_relo(struct bpf_gen *gen)
1007 if (!gen->core_relo_cnt)
1009 free(gen->core_relos);
1010 gen->core_relo_cnt = 0;
1011 gen->core_relos = NULL;
1014 static void cleanup_relos(struct bpf_gen *gen, int insns)
1019 for (i = 0; i < gen->nr_ksyms; i++) {
1020 kdesc = &gen->ksyms[i];
1026 emit_sys_close_blob(gen, insn);
1028 emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off));
1030 gen->nr_fd_array--;
1033 if (gen->nr_ksyms) {
1034 free(gen->ksyms);
1035 gen->nr_ksyms = 0;
1036 gen->ksyms = NULL;
1038 if (gen->relo_cnt) {
1039 free(gen->relos);
1040 gen->relo_cnt = 0;
1041 gen->relos = NULL;
1043 cleanup_core_relo(gen);
1047 static void info_blob_bswap(struct bpf_gen *gen, int func_info, int line_info,
1050 struct bpf_func_info *fi = gen->data_start + func_info;
1051 struct bpf_line_info *li = gen->data_start + line_info;
1052 struct bpf_core_relo *cr = gen->data_start + core_relos;
1061 for (i = 0; i < gen->core_relo_cnt; i++)
1065 void bpf_gen__prog_load(struct bpf_gen *gen,
1074 int core_relo_tot_sz = gen->core_relo_cnt *
1082 license_off = add_data(gen, license, strlen(license) + 1);
1084 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn));
1085 pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n",
1089 if (gen->swapped_endian && !gen->error) {
1090 struct bpf_insn *insn = gen->data_start + insns_off;
1107 func_info = add_data(gen, load_attr->func_info, func_info_tot_sz);
1108 pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n",
1114 line_info = add_data(gen, load_attr->line_info, line_info_tot_sz);
1115 pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n",
1120 attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt);
1121 core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz);
1122 pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n",
1123 core_relos, gen->core_relo_cnt,
1127 if (gen->swapped_endian && !gen->error)
1128 info_blob_bswap(gen, func_info, line_info, core_relos, load_attr);
1131 prog_load_attr = add_data(gen, &attr, attr_size);
1132 pr_debug("gen: prog_load: attr: off %d size %d\n",
1136 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off);
1139 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off);
1142 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info);
1145 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);
1148 emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos);
1151 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array);
1154 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,
1156 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4,
1158 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8,
1161 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4,
1163 if (gen->attach_kind) {
1164 emit_find_attach_target(gen);
1166 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
1168 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
1170 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
1171 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
1174 emit_relos(gen, insns_off);
1176 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size);
1177 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt);
1179 cleanup_relos(gen, insns_off);
1180 if (gen->attach_kind) {
1181 emit_sys_close_blob(gen,
1183 gen->attach_kind = 0;
1185 emit_check_err(gen);
1187 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
1188 stack_off(prog_fd[gen->nr_progs])));
1189 gen->nr_progs++;
1192 void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
1202 value = add_data(gen, pvalue, value_size);
1203 key = add_data(gen, &zero, sizeof(zero));
1219 if (!OPTS_GET(gen->opts, gen_hash, false)) {
1220 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6,
1224 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8));
1225 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
1227 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size));
1228 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6,
1230 emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2));
1231 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));
1232 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
1233 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
1236 map_update_attr = add_data(gen, &attr, attr_size);
1237 pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n",
1239 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1240 blob_fd_array_off(gen, map_idx));
1241 emit_rel_store(gen, attr_field(map_update_attr, key), key);
1242 emit_rel_store(gen, attr_field(map_update_attr, value), value);
1244 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
1245 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size);
1246 emit_check_err(gen);
1249 void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot,
1260 key = add_data(gen, &tgt_slot, sizeof(tgt_slot));
1262 map_update_attr = add_data(gen, &attr, attr_size);
1263 pr_debug("gen: populate_outer_map: outer %d key %d inner %d, attr: off %d size %d\n",
1265 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1266 blob_fd_array_off(gen, outer_map_idx));
1267 emit_rel_store(gen, attr_field(map_update_attr, key), key);
1268 emit_rel_store(gen, attr_field(map_update_attr, value),
1269 blob_fd_array_off(gen, inner_map_idx));
1272 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
1273 debug_ret(gen, "populate_outer_map outer %d key %d inner %d",
1275 emit_check_err(gen);
1278 void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx)
1285 map_freeze_attr = add_data(gen, &attr, attr_size);
1286 pr_debug("gen: map_freeze: idx %d, attr: off %d size %d\n",
1288 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
1289 blob_fd_array_off(gen, map_idx));
1291 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);
1292 debug_ret(gen, "map_freeze");
1293 emit_check_err(gen);