463 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_sys_lookup_elem()  local
471 	kvalue = &st_map->kvalue;  in bpf_struct_ops_map_sys_lookup_elem()
483 	memcpy(uvalue, st_map->uvalue, map->value_size);  in bpf_struct_ops_map_sys_lookup_elem()
502 static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_put_progs()  argument
506 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_put_progs()
507 		if (!st_map->links[i])  in bpf_struct_ops_map_put_progs()
509 		bpf_link_put(st_map->links[i]);  in bpf_struct_ops_map_put_progs()
510 		st_map->links[i] = NULL;  in bpf_struct_ops_map_put_progs()
514 static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_free_image()  argument
518 	for (i = 0; i < st_map->image_pages_cnt; i++)  in bpf_struct_ops_map_free_image()
519 		bpf_struct_ops_image_free(st_map->image_pages[i]);  in bpf_struct_ops_map_free_image()
520 	st_map->image_pages_cnt = 0;  in bpf_struct_ops_map_free_image()
620 static void bpf_struct_ops_map_add_ksyms(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_add_ksyms()  argument
624 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_add_ksyms()
625 		if (!st_map->ksyms[i])  in bpf_struct_ops_map_add_ksyms()
627 		bpf_image_ksym_add(st_map->ksyms[i]);  in bpf_struct_ops_map_add_ksyms()
631 static void bpf_struct_ops_map_del_ksyms(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_del_ksyms()  argument
635 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_del_ksyms()
636 		if (!st_map->ksyms[i])  in bpf_struct_ops_map_del_ksyms()
638 		bpf_image_ksym_del(st_map->ksyms[i]);  in bpf_struct_ops_map_del_ksyms()
642 static void bpf_struct_ops_map_free_ksyms(struct bpf_struct_ops_map *st_map)  in bpf_struct_ops_map_free_ksyms()  argument
646 	for (i = 0; i < st_map->funcs_cnt; i++) {  in bpf_struct_ops_map_free_ksyms()
647 		if (!st_map->ksyms[i])  in bpf_struct_ops_map_free_ksyms()
649 		kfree(st_map->ksyms[i]);  in bpf_struct_ops_map_free_ksyms()
650 		st_map->ksyms[i] = NULL;  in bpf_struct_ops_map_free_ksyms()
657 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_update_elem()  local
658 	const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;  in bpf_struct_ops_map_update_elem()
679 	err = check_zero_holes(st_map->btf, st_ops_desc->value_type, value);  in bpf_struct_ops_map_update_elem()
684 	err = check_zero_holes(st_map->btf, t, uvalue->data);  in bpf_struct_ops_map_update_elem()
695 	uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;  in bpf_struct_ops_map_update_elem()
696 	kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;  in bpf_struct_ops_map_update_elem()
698 	mutex_lock(&st_map->lock);  in bpf_struct_ops_map_update_elem()
710 	plink = st_map->links;  in bpf_struct_ops_map_update_elem()
711 	pksym = st_map->ksyms;  in bpf_struct_ops_map_update_elem()
712 	tname = btf_name_by_offset(st_map->btf, t->name_off);  in bpf_struct_ops_map_update_elem()
722 		mname = btf_name_by_offset(st_map->btf, member->name_off);  in bpf_struct_ops_map_update_elem()
723 		ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);  in bpf_struct_ops_map_update_elem()
748 			mtype = btf_type_by_id(st_map->btf, member->type);  in bpf_struct_ops_map_update_elem()
749 			mtype = btf_resolve_size(st_map->btf, mtype, &msize);  in bpf_struct_ops_map_update_elem()
804 						st_map->image_pages_cnt < MAX_TRAMP_IMAGE_PAGES);  in bpf_struct_ops_map_update_elem()
809 			st_map->image_pages[st_map->image_pages_cnt++] = image;  in bpf_struct_ops_map_update_elem()
831 	for (i = 0; i < st_map->image_pages_cnt; i++) {  in bpf_struct_ops_map_update_elem()
832 		err = arch_protect_bpf_trampoline(st_map->image_pages[i],  in bpf_struct_ops_map_update_elem()
838 	if (st_map->map.map_flags & BPF_F_LINK) {  in bpf_struct_ops_map_update_elem()
872 	bpf_struct_ops_map_free_ksyms(st_map);  in bpf_struct_ops_map_update_elem()
873 	bpf_struct_ops_map_free_image(st_map);  in bpf_struct_ops_map_update_elem()
874 	bpf_struct_ops_map_put_progs(st_map);  in bpf_struct_ops_map_update_elem()
879 	mutex_unlock(&st_map->lock);  in bpf_struct_ops_map_update_elem()
881 		bpf_struct_ops_map_add_ksyms(st_map);  in bpf_struct_ops_map_update_elem()
888 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_delete_elem()  local
890 	st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_delete_elem()
891 	if (st_map->map.map_flags & BPF_F_LINK)  in bpf_struct_ops_map_delete_elem()
894 	prev_state = cmpxchg(&st_map->kvalue.common.state,  in bpf_struct_ops_map_delete_elem()
899 		st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, NULL);  in bpf_struct_ops_map_delete_elem()
916 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_seq_show_elem()  local
926 		btf_type_seq_show(st_map->btf,  in bpf_struct_ops_map_seq_show_elem()
937 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in __bpf_struct_ops_map_free()  local
939 	if (st_map->links)  in __bpf_struct_ops_map_free()
940 		bpf_struct_ops_map_put_progs(st_map);  in __bpf_struct_ops_map_free()
941 	if (st_map->ksyms)  in __bpf_struct_ops_map_free()
942 		bpf_struct_ops_map_free_ksyms(st_map);  in __bpf_struct_ops_map_free()
943 	bpf_map_area_free(st_map->links);  in __bpf_struct_ops_map_free()
944 	bpf_map_area_free(st_map->ksyms);  in __bpf_struct_ops_map_free()
945 	bpf_struct_ops_map_free_image(st_map);  in __bpf_struct_ops_map_free()
946 	bpf_map_area_free(st_map->uvalue);  in __bpf_struct_ops_map_free()
947 	bpf_map_area_free(st_map);  in __bpf_struct_ops_map_free()
952 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_free()  local
958 	if (btf_is_module(st_map->btf))  in bpf_struct_ops_map_free()
959 		module_put(st_map->st_ops_desc->st_ops->owner);  in bpf_struct_ops_map_free()
961 	bpf_struct_ops_map_del_ksyms(st_map);  in bpf_struct_ops_map_free()
1009 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_alloc()  local
1055 	st_map_size = sizeof(*st_map) +  in bpf_struct_ops_map_alloc()
1061 	st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);  in bpf_struct_ops_map_alloc()
1062 	if (!st_map) {  in bpf_struct_ops_map_alloc()
1067 	st_map->st_ops_desc = st_ops_desc;  in bpf_struct_ops_map_alloc()
1068 	map = &st_map->map;  in bpf_struct_ops_map_alloc()
1070 	st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);  in bpf_struct_ops_map_alloc()
1071 	st_map->funcs_cnt = count_func_ptrs(btf, t);  in bpf_struct_ops_map_alloc()
1072 	st_map->links =  in bpf_struct_ops_map_alloc()
1073 		bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_link *),  in bpf_struct_ops_map_alloc()
1076 	st_map->ksyms =  in bpf_struct_ops_map_alloc()
1077 		bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_ksym *),  in bpf_struct_ops_map_alloc()
1079 	if (!st_map->uvalue || !st_map->links || !st_map->ksyms) {  in bpf_struct_ops_map_alloc()
1083 	st_map->btf = btf;  in bpf_struct_ops_map_alloc()
1085 	mutex_init(&st_map->lock);  in bpf_struct_ops_map_alloc()
1100 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_map_mem_usage()  local
1101 	const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc;  in bpf_struct_ops_map_mem_usage()
1105 	usage = sizeof(*st_map) +  in bpf_struct_ops_map_mem_usage()
1108 	usage += st_map->funcs_cnt * sizeof(struct bpf_link *);  in bpf_struct_ops_map_mem_usage()
1109 	usage += st_map->funcs_cnt * sizeof(struct bpf_ksym *);  in bpf_struct_ops_map_mem_usage()
1134 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_get()  local
1138 	st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);  in bpf_struct_ops_get()
1140 	map = __bpf_map_inc_not_zero(&st_map->map, false);  in bpf_struct_ops_get()
1147 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_put()  local
1150 	st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue);  in bpf_struct_ops_put()
1152 	bpf_map_put(&st_map->map);  in bpf_struct_ops_put()
1164 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_valid_to_reg()  local
1169 		smp_load_acquire(&st_map->kvalue.common.state) == BPF_STRUCT_OPS_STATE_READY;  in bpf_struct_ops_valid_to_reg()
1175 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_link_dealloc()  local
1178 	st_map = (struct bpf_struct_ops_map *)  in bpf_struct_ops_map_link_dealloc()
1180 	if (st_map) {  in bpf_struct_ops_map_link_dealloc()
1181 		st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);  in bpf_struct_ops_map_link_dealloc()
1182 		bpf_map_put(&st_map->map);  in bpf_struct_ops_map_link_dealloc()
1219 	struct bpf_struct_ops_map *st_map, *old_st_map;  in bpf_struct_ops_map_link_update()  local
1225 	st_map = container_of(new_map, struct bpf_struct_ops_map, map);  in bpf_struct_ops_map_link_update()
1230 	if (!st_map->st_ops_desc->st_ops->update)  in bpf_struct_ops_map_link_update()
1247 	if (st_map->st_ops_desc != old_st_map->st_ops_desc) {  in bpf_struct_ops_map_link_update()
1252 	err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data, link);  in bpf_struct_ops_map_link_update()
1269 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_map_link_detach()  local
1279 	st_map = container_of(map, struct bpf_struct_ops_map, map);  in bpf_struct_ops_map_link_detach()
1281 	st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);  in bpf_struct_ops_map_link_detach()
1287 	bpf_map_put(&st_map->map);  in bpf_struct_ops_map_link_detach()
1319 	struct bpf_struct_ops_map *st_map;  in bpf_struct_ops_link_create()  local
1327 	st_map = (struct bpf_struct_ops_map *)map;  in bpf_struct_ops_link_create()
1351 	err = st_map->st_ops_desc->st_ops->reg(st_map->kvalue.data, &link->link);  in bpf_struct_ops_link_create()
1371 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;  in bpf_map_struct_ops_info_fill()  local
1373 	info->btf_vmlinux_id = btf_obj_id(st_map->btf);  in bpf_map_struct_ops_info_fill()