45 static int blob_fd_array_off(struct bpf_gen *gen, int index)  in blob_fd_array_off()  argument
47 	return gen->fd_array + index * sizeof(int);  in blob_fd_array_off()
50 static int realloc_insn_buf(struct bpf_gen *gen, __u32 size)  in realloc_insn_buf()  argument
52 	size_t off = gen->insn_cur - gen->insn_start;  in realloc_insn_buf()
55 	if (gen->error)  in realloc_insn_buf()
56 		return gen->error;  in realloc_insn_buf()
58 		gen->error = -ERANGE;  in realloc_insn_buf()
61 	insn_start = realloc(gen->insn_start, off + size);  in realloc_insn_buf()
63 		gen->error = -ENOMEM;  in realloc_insn_buf()
64 		free(gen->insn_start);  in realloc_insn_buf()
65 		gen->insn_start = NULL;  in realloc_insn_buf()
68 	gen->insn_start = insn_start;  in realloc_insn_buf()
69 	gen->insn_cur = insn_start + off;  in realloc_insn_buf()
73 static int realloc_data_buf(struct bpf_gen *gen, __u32 size)  in realloc_data_buf()  argument
75 	size_t off = gen->data_cur - gen->data_start;  in realloc_data_buf()
78 	if (gen->error)  in realloc_data_buf()
79 		return gen->error;  in realloc_data_buf()
81 		gen->error = -ERANGE;  in realloc_data_buf()
84 	data_start = realloc(gen->data_start, off + size);  in realloc_data_buf()
86 		gen->error = -ENOMEM;  in realloc_data_buf()
87 		free(gen->data_start);  in realloc_data_buf()
88 		gen->data_start = NULL;  in realloc_data_buf()
91 	gen->data_start = data_start;  in realloc_data_buf()
92 	gen->data_cur = data_start + off;  in realloc_data_buf()
96 static void emit(struct bpf_gen *gen, struct bpf_insn insn)  in emit()  argument
98 	if (realloc_insn_buf(gen, sizeof(insn)))  in emit()
100 	memcpy(gen->insn_cur, &insn, sizeof(insn));  in emit()
101 	gen->insn_cur += sizeof(insn);  in emit()
104 static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2)  in emit2()  argument
106 	emit(gen, insn1);  in emit2()
107 	emit(gen, insn2);  in emit2()
110 static int add_data(struct bpf_gen *gen, const void *data, __u32 size);
111 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off);
112 static void emit_signature_match(struct bpf_gen *gen);
114 void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps)  in bpf_gen__init()  argument
119 	gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));  in bpf_gen__init()
120 	gen->log_level = log_level;  in bpf_gen__init()
122 	emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1));  in bpf_gen__init()
125 	emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));  in bpf_gen__init()
126 	emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz));  in bpf_gen__init()
127 	emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz));  in bpf_gen__init()
128 	emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));  in bpf_gen__init()
129 	emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));  in bpf_gen__init()
134 	emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0,  in bpf_gen__init()
140 			      nr_maps * (6 + (gen->log_level ? 6 : 0))));  in bpf_gen__init()
143 	gen->cleanup_label = gen->insn_cur - gen->insn_start;  in bpf_gen__init()
146 		emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i));  in bpf_gen__init()
147 		emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1));  in bpf_gen__init()
148 		emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));  in bpf_gen__init()
151 		emit_sys_close_blob(gen, blob_fd_array_off(gen, i));  in bpf_gen__init()
153 	emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));  in bpf_gen__init()
154 	emit(gen, BPF_EXIT_INSN());  in bpf_gen__init()
155 	if (OPTS_GET(gen->opts, gen_hash, false))  in bpf_gen__init()
156 		emit_signature_match(gen);  in bpf_gen__init()
159 static int add_data(struct bpf_gen *gen, const void *data, __u32 size)  in add_data()  argument
165 	if (realloc_data_buf(gen, size8))  in add_data()
167 	prev = gen->data_cur;  in add_data()
169 		memcpy(gen->data_cur, data, size);  in add_data()
170 		memcpy(gen->data_cur + size, &zero, size8 - size);  in add_data()
172 		memset(gen->data_cur, 0, size8);  in add_data()
174 	gen->data_cur += size8;  in add_data()
175 	return prev - gen->data_start;  in add_data()
181 static int add_map_fd(struct bpf_gen *gen)  in add_map_fd()  argument
183 	if (gen->nr_maps == MAX_USED_MAPS) {  in add_map_fd()
185 		gen->error = -E2BIG;  in add_map_fd()
188 	return gen->nr_maps++;  in add_map_fd()
191 static int add_kfunc_btf_fd(struct bpf_gen *gen)  in add_kfunc_btf_fd()  argument
195 	if (gen->nr_fd_array == MAX_KFUNC_DESCS) {  in add_kfunc_btf_fd()
196 		cur = add_data(gen, NULL, sizeof(int));  in add_kfunc_btf_fd()
197 		return (cur - gen->fd_array) / sizeof(int);  in add_kfunc_btf_fd()
199 	return MAX_USED_MAPS + gen->nr_fd_array++;  in add_kfunc_btf_fd()
214 static void emit_rel_store(struct bpf_gen *gen, int off, int data)  in emit_rel_store()  argument
216 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,  in emit_rel_store()
218 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,  in emit_rel_store()
220 	emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));  in emit_rel_store()
223 static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off)  in move_blob2blob()  argument
225 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,  in move_blob2blob()
227 	emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0));  in move_blob2blob()
228 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,  in move_blob2blob()
230 	emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));  in move_blob2blob()
233 static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off)  in move_blob2ctx()  argument
235 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,  in move_blob2ctx()
237 	emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0));  in move_blob2ctx()
238 	emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));  in move_blob2ctx()
241 static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,  in move_ctx2blob()  argument
244 	emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off));  in move_ctx2blob()
249 		emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3));  in move_ctx2blob()
250 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,  in move_ctx2blob()
252 	emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));  in move_ctx2blob()
255 static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off)  in move_stack2blob()  argument
257 	emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));  in move_stack2blob()
258 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,  in move_stack2blob()
260 	emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));  in move_stack2blob()
263 static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off)  in move_stack2ctx()  argument
265 	emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));  in move_stack2ctx()
266 	emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));  in move_stack2ctx()
269 static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size)  in emit_sys_bpf()  argument
271 	emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd));  in emit_sys_bpf()
272 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,  in emit_sys_bpf()
274 	emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size));  in emit_sys_bpf()
275 	emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf));  in emit_sys_bpf()
277 	emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));  in emit_sys_bpf()
285 static void emit_check_err(struct bpf_gen *gen)  in emit_check_err()  argument
287 	__s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1;  in emit_check_err()
293 		emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off));  in emit_check_err()
295 		gen->error = -ERANGE;  in emit_check_err()
296 		emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, -1));  in emit_check_err()
301 static void emit_debug(struct bpf_gen *gen, int reg1, int reg2,  in emit_debug()  argument
307 	if (!gen->log_level)  in emit_debug()
317 	addr = add_data(gen, buf, len);  in emit_debug()
319 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,  in emit_debug()
321 	emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));  in emit_debug()
323 		emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1));  in emit_debug()
325 		emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2));  in emit_debug()
326 	emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk));  in emit_debug()
329 static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...)  in debug_regs()  argument
334 	emit_debug(gen, reg1, reg2, fmt, args);  in debug_regs()
338 static void debug_ret(struct bpf_gen *gen, const char *fmt, ...)  in debug_ret()  argument
343 	emit_debug(gen, BPF_REG_7, -1, fmt, args);  in debug_ret()
347 static void __emit_sys_close(struct bpf_gen *gen)  in __emit_sys_close()  argument
349 	emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0,  in __emit_sys_close()
353 			      2 + (gen->log_level ? 6 : 0)));  in __emit_sys_close()
354 	emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1));  in __emit_sys_close()
355 	emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));  in __emit_sys_close()
356 	debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d");  in __emit_sys_close()
359 static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off)  in emit_sys_close_stack()  argument
361 	emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off));  in emit_sys_close_stack()
362 	__emit_sys_close(gen);  in emit_sys_close_stack()
365 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off)  in emit_sys_close_blob()  argument
367 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,  in emit_sys_close_blob()
369 	emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0));  in emit_sys_close_blob()
370 	__emit_sys_close(gen);  in emit_sys_close_blob()
373 static void compute_sha_update_offsets(struct bpf_gen *gen);
375 int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps)  in bpf_gen__finish()  argument
379 	if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) {  in bpf_gen__finish()
381 			nr_progs, gen->nr_progs, nr_maps, gen->nr_maps);  in bpf_gen__finish()
382 		gen->error = -EFAULT;  in bpf_gen__finish()
383 		return gen->error;  in bpf_gen__finish()
385 	emit_sys_close_stack(gen, stack_off(btf_fd));  in bpf_gen__finish()
386 	for (i = 0; i < gen->nr_progs; i++)  in bpf_gen__finish()
387 		move_stack2ctx(gen,  in bpf_gen__finish()
389 			       sizeof(struct bpf_map_desc) * gen->nr_maps +  in bpf_gen__finish()
393 	for (i = 0; i < gen->nr_maps; i++)  in bpf_gen__finish()
394 		move_blob2ctx(gen,  in bpf_gen__finish()
398 			      blob_fd_array_off(gen, i));  in bpf_gen__finish()
399 	emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));  in bpf_gen__finish()
400 	emit(gen, BPF_EXIT_INSN());  in bpf_gen__finish()
401 	if (OPTS_GET(gen->opts, gen_hash, false))  in bpf_gen__finish()
402 		compute_sha_update_offsets(gen);  in bpf_gen__finish()
404 	pr_debug("gen: finish %s\n", errstr(gen->error));  in bpf_gen__finish()
405 	if (!gen->error) {  in bpf_gen__finish()
406 		struct gen_loader_opts *opts = gen->opts;  in bpf_gen__finish()
408 		opts->insns = gen->insn_start;  in bpf_gen__finish()
409 		opts->insns_sz = gen->insn_cur - gen->insn_start;  in bpf_gen__finish()
410 		opts->data = gen->data_start;  in bpf_gen__finish()
411 		opts->data_sz = gen->data_cur - gen->data_start;  in bpf_gen__finish()
414 		if (gen->swapped_endian) {  in bpf_gen__finish()
422 	return gen->error;  in bpf_gen__finish()
425 void bpf_gen__free(struct bpf_gen *gen)  in bpf_gen__free()  argument
427 	if (!gen)  in bpf_gen__free()
429 	free(gen->data_start);  in bpf_gen__free()
430 	free(gen->insn_start);  in bpf_gen__free()
431 	free(gen);  in bpf_gen__free()
444 	if (gen->swapped_endian) {				\
456 static void compute_sha_update_offsets(struct bpf_gen *gen)  in compute_sha_update_offsets()  argument
462 	libbpf_sha256(gen->data_start, gen->data_cur - gen->data_start, (__u8 *)sha);  in compute_sha_update_offsets()
465 			(struct bpf_insn *)(gen->insn_start + gen->hash_insn_offset[i]);  in compute_sha_update_offsets()
472 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,  in bpf_gen__load_btf()  argument
480 	btf_data = add_data(gen, btf_raw_data, btf_raw_size);  in bpf_gen__load_btf()
483 	btf_load_attr = add_data(gen, &attr, attr_size);  in bpf_gen__load_btf()
488 	move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4,  in bpf_gen__load_btf()
490 	move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4,  in bpf_gen__load_btf()
492 	move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8,  in bpf_gen__load_btf()
495 	emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data);  in bpf_gen__load_btf()
497 	emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size);  in bpf_gen__load_btf()
498 	debug_ret(gen, "btf_load size %d", btf_raw_size);  in bpf_gen__load_btf()
499 	emit_check_err(gen);  in bpf_gen__load_btf()
501 	emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd)));  in bpf_gen__load_btf()
504 void bpf_gen__map_create(struct bpf_gen *gen,  in bpf_gen__map_create()  argument
529 	map_create_attr = add_data(gen, &attr, attr_size);  in bpf_gen__map_create()
536 		move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4,  in bpf_gen__map_create()
541 		move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4,  in bpf_gen__map_create()
550 		move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4,  in bpf_gen__map_create()
556 	emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size);  in bpf_gen__map_create()
557 	debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d",  in bpf_gen__map_create()
560 	emit_check_err(gen);  in bpf_gen__map_create()
567 		emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,  in bpf_gen__map_create()
569 	} else if (map_idx != gen->nr_maps) {  in bpf_gen__map_create()
570 		gen->error = -EDOM; /* internal bug */  in bpf_gen__map_create()
574 		idx = add_map_fd(gen);  in bpf_gen__map_create()
575 		emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,  in bpf_gen__map_create()
576 						 0, 0, 0, blob_fd_array_off(gen, idx)));  in bpf_gen__map_create()
577 		emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0));  in bpf_gen__map_create()
580 		emit_sys_close_stack(gen, stack_off(inner_map_fd));  in bpf_gen__map_create()
583 static void emit_signature_match(struct bpf_gen *gen)  in emit_signature_match()  argument
589 		emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX,  in emit_signature_match()
591 		emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, i * sizeof(__u64)));  in emit_signature_match()
592 		gen->hash_insn_offset[i] = gen->insn_cur - gen->insn_start;  in emit_signature_match()
593 		emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_3, 0, 0, 0, 0, 0));  in emit_signature_match()
595 		off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2;  in emit_signature_match()
597 			emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL));  in emit_signature_match()
598 			emit(gen, BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, off));  in emit_signature_match()
600 			gen->error = -ERANGE;  in emit_signature_match()
605 void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name,  in bpf_gen__record_attach_target()  argument
612 	gen->attach_kind = kind;  in bpf_gen__record_attach_target()
613 	ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s",  in bpf_gen__record_attach_target()
615 	if (ret >= sizeof(gen->attach_target))  in bpf_gen__record_attach_target()
616 		gen->error = -ENOSPC;  in bpf_gen__record_attach_target()
619 static void emit_find_attach_target(struct bpf_gen *gen)  in emit_find_attach_target()  argument
621 	int name, len = strlen(gen->attach_target) + 1;  in emit_find_attach_target()
623 	pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind);  in emit_find_attach_target()
624 	name = add_data(gen, gen->attach_target, len);  in emit_find_attach_target()
626 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,  in emit_find_attach_target()
628 	emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));  in emit_find_attach_target()
629 	emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind));  in emit_find_attach_target()
630 	emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));  in emit_find_attach_target()
631 	emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));  in emit_find_attach_target()
632 	emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));  in emit_find_attach_target()
633 	debug_ret(gen, "find_by_name_kind(%s,%d)",  in emit_find_attach_target()
634 		  gen->attach_target, gen->attach_kind);  in emit_find_attach_target()
635 	emit_check_err(gen);  in emit_find_attach_target()
641 void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak,  in bpf_gen__record_extern()  argument
646 	relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo));  in bpf_gen__record_extern()
648 		gen->error = -ENOMEM;  in bpf_gen__record_extern()
651 	gen->relos = relo;  in bpf_gen__record_extern()
652 	relo += gen->relo_cnt;  in bpf_gen__record_extern()
659 	gen->relo_cnt++;  in bpf_gen__record_extern()
663 static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo)  in get_ksym_desc()  argument
668 	for (i = 0; i < gen->nr_ksyms; i++) {  in get_ksym_desc()
669 		kdesc = &gen->ksyms[i];  in get_ksym_desc()
676 	kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc));  in get_ksym_desc()
678 		gen->error = -ENOMEM;  in get_ksym_desc()
681 	gen->ksyms = kdesc;  in get_ksym_desc()
682 	kdesc = &gen->ksyms[gen->nr_ksyms++];  in get_ksym_desc()
695 static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo)  in emit_bpf_find_by_name_kind()  argument
699 	name_off = add_data(gen, relo->name, len);  in emit_bpf_find_by_name_kind()
700 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,  in emit_bpf_find_by_name_kind()
702 	emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));  in emit_bpf_find_by_name_kind()
703 	emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));  in emit_bpf_find_by_name_kind()
704 	emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));  in emit_bpf_find_by_name_kind()
705 	emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));  in emit_bpf_find_by_name_kind()
706 	emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));  in emit_bpf_find_by_name_kind()
707 	debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);  in emit_bpf_find_by_name_kind()
714 static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo)  in emit_bpf_kallsyms_lookup_name()  argument
718 	name_off = add_data(gen, relo->name, len);  in emit_bpf_kallsyms_lookup_name()
719 	res_off = add_data(gen, NULL, 8); /* res is u64 */  in emit_bpf_kallsyms_lookup_name()
720 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,  in emit_bpf_kallsyms_lookup_name()
722 	emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));  in emit_bpf_kallsyms_lookup_name()
723 	emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));  in emit_bpf_kallsyms_lookup_name()
724 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE,  in emit_bpf_kallsyms_lookup_name()
726 	emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4));  in emit_bpf_kallsyms_lookup_name()
727 	emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name));  in emit_bpf_kallsyms_lookup_name()
728 	emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0));  in emit_bpf_kallsyms_lookup_name()
729 	emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));  in emit_bpf_kallsyms_lookup_name()
730 	debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind);  in emit_bpf_kallsyms_lookup_name()
743 static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)  in emit_relo_kfunc_btf()  argument
748 	kdesc = get_ksym_desc(gen, relo);  in emit_relo_kfunc_btf()
753 		move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,  in emit_relo_kfunc_btf()
755 		move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2,  in emit_relo_kfunc_btf()
761 	emit_bpf_find_by_name_kind(gen, relo);  in emit_relo_kfunc_btf()
763 		emit_check_err(gen);  in emit_relo_kfunc_btf()
765 	btf_fd_idx = add_kfunc_btf_fd(gen);  in emit_relo_kfunc_btf()
769 		gen->error = -E2BIG;  in emit_relo_kfunc_btf()
774 	emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));  in emit_relo_kfunc_btf()
776 	emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));  in emit_relo_kfunc_btf()
777 	emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));  in emit_relo_kfunc_btf()
779 	emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10));  in emit_relo_kfunc_btf()
781 	emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));  in emit_relo_kfunc_btf()
783 	emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7));  in emit_relo_kfunc_btf()
784 	emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));  in emit_relo_kfunc_btf()
786 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,  in emit_relo_kfunc_btf()
787 					 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx)));  in emit_relo_kfunc_btf()
789 	emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0));  in emit_relo_kfunc_btf()
791 	emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2));  in emit_relo_kfunc_btf()
793 	emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));  in emit_relo_kfunc_btf()
795 	emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));  in emit_relo_kfunc_btf()
797 	emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx));  in emit_relo_kfunc_btf()
799 	if (!gen->log_level)  in emit_relo_kfunc_btf()
801 	emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,  in emit_relo_kfunc_btf()
803 	emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8,  in emit_relo_kfunc_btf()
805 	debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d",  in emit_relo_kfunc_btf()
807 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,  in emit_relo_kfunc_btf()
808 					 0, 0, 0, blob_fd_array_off(gen, kdesc->off)));  in emit_relo_kfunc_btf()
809 	emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0));  in emit_relo_kfunc_btf()
810 	debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd",  in emit_relo_kfunc_btf()
814 static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo,  in emit_ksym_relo_log()  argument
817 	if (!gen->log_level)  in emit_ksym_relo_log()
819 	emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,  in emit_ksym_relo_log()
821 	emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) +  in emit_ksym_relo_log()
823 	debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d",  in emit_ksym_relo_log()
825 	emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));  in emit_ksym_relo_log()
826 	debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg",  in emit_ksym_relo_log()
833 static void emit_relo_ksym_typeless(struct bpf_gen *gen,  in emit_relo_ksym_typeless()  argument
838 	kdesc = get_ksym_desc(gen, relo);  in emit_relo_ksym_typeless()
843 		move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,  in emit_relo_ksym_typeless()
845 		move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,  in emit_relo_ksym_typeless()
853 	emit_bpf_kallsyms_lookup_name(gen, relo);  in emit_relo_ksym_typeless()
854 	emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1));  in emit_relo_ksym_typeless()
855 	emit_check_err(gen);  in emit_relo_ksym_typeless()
857 	emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm)));  in emit_relo_ksym_typeless()
859 	emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));  in emit_relo_ksym_typeless()
860 	emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9,  in emit_relo_ksym_typeless()
863 	emit_ksym_relo_log(gen, relo, kdesc->ref);  in emit_relo_ksym_typeless()
866 static __u32 src_reg_mask(struct bpf_gen *gen)  in src_reg_mask()  argument
869 	return gen->swapped_endian ? 0xf0 : 0x0f;  in src_reg_mask()
871 	return gen->swapped_endian ? 0x0f : 0xf0;  in src_reg_mask()
880 static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)  in emit_relo_ksym_btf()  argument
885 	kdesc = get_ksym_desc(gen, relo);  in emit_relo_ksym_btf()
890 		move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,  in emit_relo_ksym_btf()
892 		move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,  in emit_relo_ksym_btf()
897 		emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3));  in emit_relo_ksym_btf()
902 	emit_bpf_find_by_name_kind(gen, relo);  in emit_relo_ksym_btf()
904 		emit_check_err(gen);  in emit_relo_ksym_btf()
906 	emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));  in emit_relo_ksym_btf()
908 	emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));  in emit_relo_ksym_btf()
909 …emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0…  in emit_relo_ksym_btf()
911 	emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4));  in emit_relo_ksym_btf()
913 	emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));  in emit_relo_ksym_btf()
915 	emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));  in emit_relo_ksym_btf()
916 	emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7,  in emit_relo_ksym_btf()
919 	emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3));  in emit_relo_ksym_btf()
922 	reg_mask = src_reg_mask(gen);  in emit_relo_ksym_btf()
923 	emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));  in emit_relo_ksym_btf()
924 	emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask));  in emit_relo_ksym_btf()
925 	emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code)));  in emit_relo_ksym_btf()
927 	emit_ksym_relo_log(gen, relo, kdesc->ref);  in emit_relo_ksym_btf()
930 void bpf_gen__record_relo_core(struct bpf_gen *gen,  in bpf_gen__record_relo_core()  argument
935 	relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos));  in bpf_gen__record_relo_core()
937 		gen->error = -ENOMEM;  in bpf_gen__record_relo_core()
940 	gen->core_relos = relos;  in bpf_gen__record_relo_core()
941 	relos += gen->core_relo_cnt;  in bpf_gen__record_relo_core()
943 	gen->core_relo_cnt++;  in bpf_gen__record_relo_core()
946 static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)  in emit_relo()  argument
953 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn));  in emit_relo()
956 			emit_relo_ksym_typeless(gen, relo, insn);  in emit_relo()
958 			emit_relo_ksym_btf(gen, relo, insn);  in emit_relo()
960 		emit_relo_kfunc_btf(gen, relo, insn);  in emit_relo()
964 static void emit_relos(struct bpf_gen *gen, int insns)  in emit_relos()  argument
968 	for (i = 0; i < gen->relo_cnt; i++)  in emit_relos()
969 		emit_relo(gen, gen->relos + i, insns);  in emit_relos()
972 static void cleanup_core_relo(struct bpf_gen *gen)  in cleanup_core_relo()  argument
974 	if (!gen->core_relo_cnt)  in cleanup_core_relo()
976 	free(gen->core_relos);  in cleanup_core_relo()
977 	gen->core_relo_cnt = 0;  in cleanup_core_relo()
978 	gen->core_relos = NULL;  in cleanup_core_relo()
981 static void cleanup_relos(struct bpf_gen *gen, int insns)  in cleanup_relos()  argument
986 	for (i = 0; i < gen->nr_ksyms; i++) {  in cleanup_relos()
987 		kdesc = &gen->ksyms[i];  in cleanup_relos()
993 			emit_sys_close_blob(gen, insn);  in cleanup_relos()
995 			emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off));  in cleanup_relos()
997 				gen->nr_fd_array--;  in cleanup_relos()
1000 	if (gen->nr_ksyms) {  in cleanup_relos()
1001 		free(gen->ksyms);  in cleanup_relos()
1002 		gen->nr_ksyms = 0;  in cleanup_relos()
1003 		gen->ksyms = NULL;  in cleanup_relos()
1005 	if (gen->relo_cnt) {  in cleanup_relos()
1006 		free(gen->relos);  in cleanup_relos()
1007 		gen->relo_cnt = 0;  in cleanup_relos()
1008 		gen->relos = NULL;  in cleanup_relos()
1010 	cleanup_core_relo(gen);  in cleanup_relos()
1014 static void info_blob_bswap(struct bpf_gen *gen, int func_info, int line_info,  in info_blob_bswap()  argument
1017 	struct bpf_func_info *fi = gen->data_start + func_info;  in info_blob_bswap()
1018 	struct bpf_line_info *li = gen->data_start + line_info;  in info_blob_bswap()
1019 	struct bpf_core_relo *cr = gen->data_start + core_relos;  in info_blob_bswap()
1028 	for (i = 0; i < gen->core_relo_cnt; i++)  in info_blob_bswap()
1032 void bpf_gen__prog_load(struct bpf_gen *gen,  in bpf_gen__prog_load()  argument
1041 	int core_relo_tot_sz = gen->core_relo_cnt *  in bpf_gen__prog_load()
1049 	license_off = add_data(gen, license, strlen(license) + 1);  in bpf_gen__prog_load()
1051 	insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn));  in bpf_gen__prog_load()
1056 	if (gen->swapped_endian) {  in bpf_gen__prog_load()
1057 		struct bpf_insn *insn = gen->data_start + insns_off;  in bpf_gen__prog_load()
1074 	func_info = add_data(gen, load_attr->func_info, func_info_tot_sz);  in bpf_gen__prog_load()
1081 	line_info = add_data(gen, load_attr->line_info, line_info_tot_sz);  in bpf_gen__prog_load()
1087 	attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt);  in bpf_gen__prog_load()
1088 	core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz);  in bpf_gen__prog_load()
1090 		 core_relos, gen->core_relo_cnt,  in bpf_gen__prog_load()
1094 	if (gen->swapped_endian)  in bpf_gen__prog_load()
1095 		info_blob_bswap(gen, func_info, line_info, core_relos, load_attr);  in bpf_gen__prog_load()
1098 	prog_load_attr = add_data(gen, &attr, attr_size);  in bpf_gen__prog_load()
1103 	emit_rel_store(gen, attr_field(prog_load_attr, license), license_off);  in bpf_gen__prog_load()
1106 	emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off);  in bpf_gen__prog_load()
1109 	emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info);  in bpf_gen__prog_load()
1112 	emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);  in bpf_gen__prog_load()
1115 	emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos);  in bpf_gen__prog_load()
1118 	emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array);  in bpf_gen__prog_load()
1121 	move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,  in bpf_gen__prog_load()
1123 	move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4,  in bpf_gen__prog_load()
1125 	move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8,  in bpf_gen__prog_load()
1128 	move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4,  in bpf_gen__prog_load()
1130 	if (gen->attach_kind) {  in bpf_gen__prog_load()
1131 		emit_find_attach_target(gen);  in bpf_gen__prog_load()
1133 		emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,  in bpf_gen__prog_load()
1135 		emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,  in bpf_gen__prog_load()
1137 		emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));  in bpf_gen__prog_load()
1138 		emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,  in bpf_gen__prog_load()
1141 	emit_relos(gen, insns_off);  in bpf_gen__prog_load()
1143 	emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size);  in bpf_gen__prog_load()
1144 	debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt);  in bpf_gen__prog_load()
1146 	cleanup_relos(gen, insns_off);  in bpf_gen__prog_load()
1147 	if (gen->attach_kind) {  in bpf_gen__prog_load()
1148 		emit_sys_close_blob(gen,  in bpf_gen__prog_load()
1150 		gen->attach_kind = 0;  in bpf_gen__prog_load()
1152 	emit_check_err(gen);  in bpf_gen__prog_load()
1154 	emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,  in bpf_gen__prog_load()
1155 			      stack_off(prog_fd[gen->nr_progs])));  in bpf_gen__prog_load()
1156 	gen->nr_progs++;  in bpf_gen__prog_load()
1159 void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,  in bpf_gen__map_update_elem()  argument
1169 	value = add_data(gen, pvalue, value_size);  in bpf_gen__map_update_elem()
1170 	key = add_data(gen, &zero, sizeof(zero));  in bpf_gen__map_update_elem()
1179 	emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6,  in bpf_gen__map_update_elem()
1183 	emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8));  in bpf_gen__map_update_elem()
1184 	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,  in bpf_gen__map_update_elem()
1186 	emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size));  in bpf_gen__map_update_elem()
1187 	emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6,  in bpf_gen__map_update_elem()
1189 	emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2));  in bpf_gen__map_update_elem()
1190 	emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));  in bpf_gen__map_update_elem()
1191 	emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));  in bpf_gen__map_update_elem()
1192 	emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));  in bpf_gen__map_update_elem()
1194 	map_update_attr = add_data(gen, &attr, attr_size);  in bpf_gen__map_update_elem()
1197 	move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,  in bpf_gen__map_update_elem()
1198 		       blob_fd_array_off(gen, map_idx));  in bpf_gen__map_update_elem()
1199 	emit_rel_store(gen, attr_field(map_update_attr, key), key);  in bpf_gen__map_update_elem()
1200 	emit_rel_store(gen, attr_field(map_update_attr, value), value);  in bpf_gen__map_update_elem()
1202 	emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);  in bpf_gen__map_update_elem()
1203 	debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size);  in bpf_gen__map_update_elem()
1204 	emit_check_err(gen);  in bpf_gen__map_update_elem()
1207 void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot,  in bpf_gen__populate_outer_map()  argument
1218 	key = add_data(gen, &tgt_slot, sizeof(tgt_slot));  in bpf_gen__populate_outer_map()
1220 	map_update_attr = add_data(gen, &attr, attr_size);  in bpf_gen__populate_outer_map()
1223 	move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,  in bpf_gen__populate_outer_map()
1224 		       blob_fd_array_off(gen, outer_map_idx));  in bpf_gen__populate_outer_map()
1225 	emit_rel_store(gen, attr_field(map_update_attr, key), key);  in bpf_gen__populate_outer_map()
1226 	emit_rel_store(gen, attr_field(map_update_attr, value),  in bpf_gen__populate_outer_map()
1227 		       blob_fd_array_off(gen, inner_map_idx));  in bpf_gen__populate_outer_map()
1230 	emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);  in bpf_gen__populate_outer_map()
1231 	debug_ret(gen, "populate_outer_map outer %d key %d inner %d",  in bpf_gen__populate_outer_map()
1233 	emit_check_err(gen);  in bpf_gen__populate_outer_map()
1236 void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx)  in bpf_gen__map_freeze()  argument
1243 	map_freeze_attr = add_data(gen, &attr, attr_size);  in bpf_gen__map_freeze()
1246 	move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,  in bpf_gen__map_freeze()
1247 		       blob_fd_array_off(gen, map_idx));  in bpf_gen__map_freeze()
1249 	emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);  in bpf_gen__map_freeze()
1250 	debug_ret(gen, "map_freeze");  in bpf_gen__map_freeze()
1251 	emit_check_err(gen);  in bpf_gen__map_freeze()