68 	const struct bpf_prog *prog;  in bpf_prog_run_array_cg()  local
79 	while ((prog = READ_ONCE(item->prog))) {  in bpf_prog_run_array_cg()
81 		func_ret = run_prog(prog, ctx);  in bpf_prog_run_array_cg()
238 				     struct bpf_prog *prog,  in bpf_cgroup_storages_alloc()  argument
249 		map = prog->aux->cgroup_storage[stype];  in bpf_cgroup_storages_alloc()
257 		storages[stype] = bpf_cgroup_storage_alloc(prog, stype);  in bpf_cgroup_storages_alloc()
323 			if (pl->prog) {  in cgroup_bpf_release()
324 				if (pl->prog->expected_attach_type == BPF_LSM_CGROUP)  in cgroup_bpf_release()
325 					bpf_trampoline_unlink_cgroup_shim(pl->prog);  in cgroup_bpf_release()
326 				bpf_prog_put(pl->prog);  in cgroup_bpf_release()
329 				if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP)  in cgroup_bpf_release()
330 					bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog);  in cgroup_bpf_release()
374 	if (pl->prog)  in prog_list_prog()
375 		return pl->prog;  in prog_list_prog()
377 		return pl->link->link.prog;  in prog_list_prog()
474 			item->prog = prog_list_prog(pl);  in compute_effective_progs()
623 					       struct bpf_prog *prog,  in find_attach_entry()  argument
638 		if (prog && pl->prog == prog && prog != replace_prog)  in find_attach_entry()
649 			if (pl->prog == replace_prog)  in find_attach_entry()
673 	struct bpf_prog *prog = ERR_PTR(-EINVAL);  in bpf_get_anchor_prog()  local
676 		prog = bpf_prog_by_id(id_or_fd);  in bpf_get_anchor_prog()
678 		prog = bpf_prog_get(id_or_fd);  in bpf_get_anchor_prog()
679 	return prog;  in bpf_get_anchor_prog()
682 static struct bpf_prog_list *get_prog_list(struct hlist_head *progs, struct bpf_prog *prog,  in get_prog_list()  argument
698 		if ((is_link && !link) || (!is_link && !prog))  in get_prog_list()
732 		if ((anchor_prog && anchor_prog == pltmp->prog) ||  in get_prog_list()
751 			      struct bpf_prog *prog, struct bpf_cgroup_link *link,  in insert_pl_to_hlist()  argument
756 	pltmp = get_prog_list(progs, prog, link, flags, id_or_fd);  in insert_pl_to_hlist()
786 			       struct bpf_prog *prog, struct bpf_prog *replace_prog,  in __cgroup_bpf_attach()  argument
795 	struct bpf_prog *new_prog = prog ? : link->link.prog;  in __cgroup_bpf_attach()
808 	if (link && (prog || replace_prog))  in __cgroup_bpf_attach()
836 	pl = find_attach_entry(progs, prog, link, replace_prog,  in __cgroup_bpf_attach()
842 				      prog ? : link->link.prog, cgrp))  in __cgroup_bpf_attach()
846 		old_prog = pl->prog;  in __cgroup_bpf_attach()
854 		err = insert_pl_to_hlist(pl, progs, prog, link, flags, id_or_fd);  in __cgroup_bpf_attach()
862 	pl->prog = prog;  in __cgroup_bpf_attach()
895 		pl->prog = old_prog;  in __cgroup_bpf_attach()
907 			     struct bpf_prog *prog, struct bpf_prog *replace_prog,  in cgroup_bpf_attach()  argument
915 	ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags,  in cgroup_bpf_attach()
962 		WRITE_ONCE(item->prog, link->link.prog);  in replace_effective_prog()
992 	if (link->link.prog->type != new_prog->type)  in __cgroup_bpf_replace()
1005 	old_prog = xchg(&link->link.prog, new_prog);  in __cgroup_bpf_replace()
1025 	if (old_prog && link->prog != old_prog) {  in cgroup_bpf_replace()
1036 					       struct bpf_prog *prog,  in find_detach_entry()  argument
1053 	if (!prog && !link)  in find_detach_entry()
1061 		if (pl->prog == prog && pl->link == link)  in find_detach_entry()
1077 static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog,  in purge_effective_progs()  argument
1104 				if (pl->prog == prog && pl->link == link)  in purge_effective_progs()
1135 static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,  in __cgroup_bpf_detach()  argument
1146 	if (prog)  in __cgroup_bpf_detach()
1147 		attach_btf_id = prog->aux->attach_btf_id;  in __cgroup_bpf_detach()
1149 		attach_btf_id = link->link.prog->aux->attach_btf_id;  in __cgroup_bpf_detach()
1161 	if (prog && link)  in __cgroup_bpf_detach()
1165 	pl = find_detach_entry(progs, prog, link, flags & BPF_F_ALLOW_MULTI);  in __cgroup_bpf_detach()
1170 	old_prog = pl->prog;  in __cgroup_bpf_detach()
1171 	pl->prog = NULL;  in __cgroup_bpf_detach()
1176 		pl->prog = old_prog;  in __cgroup_bpf_detach()
1198 static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,  in cgroup_bpf_detach()  argument
1204 	ret = __cgroup_bpf_detach(cgrp, prog, NULL, type, revision);  in cgroup_bpf_detach()
1282 			struct bpf_prog *prog;  in __cgroup_bpf_query()  local
1289 				prog = prog_list_prog(pl);  in __cgroup_bpf_query()
1290 				id = prog->aux->id;  in __cgroup_bpf_query()
1326 			   enum bpf_prog_type ptype, struct bpf_prog *prog)  in cgroup_bpf_prog_attach()  argument
1345 	ret = cgroup_bpf_attach(cgrp, prog, replace_prog, NULL,  in cgroup_bpf_prog_attach()
1357 	struct bpf_prog *prog;  in cgroup_bpf_prog_detach()  local
1365 	prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);  in cgroup_bpf_prog_detach()
1366 	if (IS_ERR(prog))  in cgroup_bpf_prog_detach()
1367 		prog = NULL;  in cgroup_bpf_prog_detach()
1369 	ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, attr->expected_revision);  in cgroup_bpf_prog_detach()
1370 	if (prog)  in cgroup_bpf_prog_detach()
1371 		bpf_prog_put(prog);  in cgroup_bpf_prog_detach()
1400 		bpf_trampoline_unlink_cgroup_shim(cg_link->link.prog);  in bpf_cgroup_link_release()
1477 int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)  in cgroup_bpf_link_attach()  argument
1497 		      prog, attr->link_create.attach_type);  in cgroup_bpf_link_attach()
1814 cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)  in cgroup_dev_func_proto()  argument
1818 	func_proto = cgroup_common_func_proto(func_id, prog);  in cgroup_dev_func_proto()
1826 		return bpf_base_func_proto(func_id, prog);  in cgroup_dev_func_proto()
1832 				       const struct bpf_prog *prog,  in cgroup_dev_is_valid_access()  argument
2361 sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)  in sysctl_func_proto()  argument
2365 	func_proto = cgroup_common_func_proto(func_id, prog);  in sysctl_func_proto()
2383 		return bpf_base_func_proto(func_id, prog);  in sysctl_func_proto()
2388 				   const struct bpf_prog *prog,  in sysctl_is_valid_access()  argument
2417 				     struct bpf_prog *prog, u32 *target_size)  in sysctl_convert_ctx_access()  argument
2504 cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)  in cg_sockopt_func_proto()  argument
2508 	func_proto = cgroup_common_func_proto(func_id, prog);  in cg_sockopt_func_proto()
2521 		if (prog->expected_attach_type == BPF_CGROUP_SETSOCKOPT)  in cg_sockopt_func_proto()
2525 		if (prog->expected_attach_type == BPF_CGROUP_SETSOCKOPT)  in cg_sockopt_func_proto()
2536 		return bpf_base_func_proto(func_id, prog);  in cg_sockopt_func_proto()
2542 				       const struct bpf_prog *prog,  in cg_sockopt_is_valid_access()  argument
2558 			return prog->expected_attach_type ==  in cg_sockopt_is_valid_access()
2565 			return prog->expected_attach_type ==  in cg_sockopt_is_valid_access()
2593 		return prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT;  in cg_sockopt_is_valid_access()
2617 					 struct bpf_prog *prog,  in cg_sockopt_convert_ctx_access()  argument
2694 				   const struct bpf_prog *prog)  in cg_sockopt_get_prologue()  argument
2713 cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)  in cgroup_common_func_proto()  argument
2719 		switch (prog->expected_attach_type) {  in cgroup_common_func_proto()
2737 		switch (prog->expected_attach_type) {  in cgroup_common_func_proto()