318 		struct bpf_prog_list *pl;  in cgroup_bpf_release()  local
321 		hlist_for_each_entry_safe(pl, pltmp, progs, node) {  in cgroup_bpf_release()
322 			hlist_del(&pl->node);  in cgroup_bpf_release()
323 			if (pl->prog) {  in cgroup_bpf_release()
324 				if (pl->prog->expected_attach_type == BPF_LSM_CGROUP)  in cgroup_bpf_release()
325 					bpf_trampoline_unlink_cgroup_shim(pl->prog);  in cgroup_bpf_release()
326 				bpf_prog_put(pl->prog);  in cgroup_bpf_release()
328 			if (pl->link) {  in cgroup_bpf_release()
329 				if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP)  in cgroup_bpf_release()
330 					bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog);  in cgroup_bpf_release()
331 				bpf_cgroup_link_auto_detach(pl->link);  in cgroup_bpf_release()
333 			kfree(pl);  in cgroup_bpf_release()
372 static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl)  in prog_list_prog()  argument
374 	if (pl->prog)  in prog_list_prog()
375 		return pl->prog;  in prog_list_prog()
376 	if (pl->link)  in prog_list_prog()
377 		return pl->link->link.prog;  in prog_list_prog()
386 	struct bpf_prog_list *pl;  in prog_list_length()  local
389 	hlist_for_each_entry(pl, head, node) {  in prog_list_length()
390 		if (!prog_list_prog(pl))  in prog_list_length()
392 		if (preorder_cnt && (pl->flags & BPF_F_PREORDER))  in prog_list_length()
438 	struct bpf_prog_list *pl;  in compute_effective_progs()  local
463 		hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {  in compute_effective_progs()
464 			if (!prog_list_prog(pl))  in compute_effective_progs()
467 			if (pl->flags & BPF_F_PREORDER) {  in compute_effective_progs()
474 			item->prog = prog_list_prog(pl);  in compute_effective_progs()
476 						   pl->storage);  in compute_effective_progs()
628 	struct bpf_prog_list *pl;  in find_attach_entry()  local
634 		return hlist_entry(progs->first, typeof(*pl), node);  in find_attach_entry()
637 	hlist_for_each_entry(pl, progs, node) {  in find_attach_entry()
638 		if (prog && pl->prog == prog && prog != replace_prog)  in find_attach_entry()
641 		if (link && pl->link == link)  in find_attach_entry()
648 		hlist_for_each_entry(pl, progs, node) {  in find_attach_entry()
649 			if (pl->prog == replace_prog)  in find_attach_entry()
651 				return pl;  in find_attach_entry()
686 	struct bpf_prog_list *pltmp, *pl = ERR_PTR(-EINVAL);  in get_prog_list()  local
736 			pl = pltmp;  in get_prog_list()
741 	pl = ERR_PTR(-ENOENT);  in get_prog_list()
747 	return pl;  in get_prog_list()
750 static int insert_pl_to_hlist(struct bpf_prog_list *pl, struct hlist_head *progs,  in insert_pl_to_hlist()  argument
761 		hlist_add_head(&pl->node, progs);  in insert_pl_to_hlist()
763 		hlist_add_before(&pl->node, &pltmp->node);  in insert_pl_to_hlist()
765 		hlist_add_behind(&pl->node, &pltmp->node);  in insert_pl_to_hlist()
797 	struct bpf_prog_list *pl;  in __cgroup_bpf_attach()  local
836 	pl = find_attach_entry(progs, prog, link, replace_prog,  in __cgroup_bpf_attach()
838 	if (IS_ERR(pl))  in __cgroup_bpf_attach()
839 		return PTR_ERR(pl);  in __cgroup_bpf_attach()
845 	if (pl) {  in __cgroup_bpf_attach()
846 		old_prog = pl->prog;  in __cgroup_bpf_attach()
848 		pl = kmalloc_obj(*pl);  in __cgroup_bpf_attach()
849 		if (!pl) {  in __cgroup_bpf_attach()
854 		err = insert_pl_to_hlist(pl, progs, prog, link, flags, id_or_fd);  in __cgroup_bpf_attach()
856 			kfree(pl);  in __cgroup_bpf_attach()
862 	pl->prog = prog;  in __cgroup_bpf_attach()
863 	pl->link = link;  in __cgroup_bpf_attach()
864 	pl->flags = flags;  in __cgroup_bpf_attach()
865 	bpf_cgroup_storages_assign(pl->storage, storage);  in __cgroup_bpf_attach()
895 		pl->prog = old_prog;  in __cgroup_bpf_attach()
896 		pl->link = NULL;  in __cgroup_bpf_attach()
900 		hlist_del(&pl->node);  in __cgroup_bpf_attach()
901 		kfree(pl);  in __cgroup_bpf_attach()
931 	struct bpf_prog_list *pl;  in replace_effective_prog()  local
948 			hlist_for_each_entry(pl, head, node) {  in replace_effective_prog()
949 				if (!prog_list_prog(pl))  in replace_effective_prog()
951 				if (pl->link == link)  in replace_effective_prog()
982 	struct bpf_prog_list *pl;  in __cgroup_bpf_replace()  local
995 	hlist_for_each_entry(pl, progs, node) {  in __cgroup_bpf_replace()
996 		if (pl->link == link) {  in __cgroup_bpf_replace()
1040 	struct bpf_prog_list *pl;  in find_detach_entry()  local
1050 		return hlist_entry(progs->first, typeof(*pl), node);  in find_detach_entry()
1060 	hlist_for_each_entry(pl, progs, node) {  in find_detach_entry()
1061 		if (pl->prog == prog && pl->link == link)  in find_detach_entry()
1062 			return pl;  in find_detach_entry()
1083 	struct bpf_prog_list *pl;  in purge_effective_progs()  local
1101 			hlist_for_each_entry(pl, head, node) {  in purge_effective_progs()
1102 				if (!prog_list_prog(pl))  in purge_effective_progs()
1104 				if (pl->prog == prog && pl->link == link)  in purge_effective_progs()
1141 	struct bpf_prog_list *pl;  in __cgroup_bpf_detach()  local
1165 	pl = find_detach_entry(progs, prog, link, flags & BPF_F_ALLOW_MULTI);  in __cgroup_bpf_detach()
1166 	if (IS_ERR(pl))  in __cgroup_bpf_detach()
1167 		return PTR_ERR(pl);  in __cgroup_bpf_detach()
1170 	old_prog = pl->prog;  in __cgroup_bpf_detach()
1171 	pl->prog = NULL;  in __cgroup_bpf_detach()
1172 	pl->link = NULL;  in __cgroup_bpf_detach()
1176 		pl->prog = old_prog;  in __cgroup_bpf_detach()
1177 		pl->link = link;  in __cgroup_bpf_detach()
1182 	hlist_del(&pl->node);  in __cgroup_bpf_detach()
1185 	kfree(pl);  in __cgroup_bpf_detach()
1281 			struct bpf_prog_list *pl;  in __cgroup_bpf_query()  local
1288 			hlist_for_each_entry(pl, progs, node) {  in __cgroup_bpf_query()
1289 				prog = prog_list_prog(pl);  in __cgroup_bpf_query()