2  *  Generic process-grouping system.
12  *  --------------------------------------------------
14  *  Copyright (C) 2004-2006 Silicon Graphics, Inc.
17  *  sysfs is Copyright (c) 2001-3 Patrick Mochel
19  *  2003-10-10 Written by Simon Derr.
20  *  2003-10-22 Updates by Stephen Hemminger.
21  *  2004 May-July Rework by Paul Jackson.
22  *  ---------------------------------------------------
31 #include "cgroup-internal.h"
33 #include <linux/bpf-cgroup.h>
48 #include <linux/percpu-rwsem.h>
74  * that attempts to access what would be a 0-element array (i.e. sized
84  * css_set_lock protects task->cgroups pointer, the list of css_set
109  * Protects cgroup_file->kn for !self csses.  It synchronizes notifications
110  * against file removal/re-creation across css hiding.
195  * Also, as csses are always appended to the parent's ->children list, it
261  * cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID
277  * cgroup_on_dfl - test whether a cgroup is on the default hierarchy
287  * - Mount options "noprefix", "xattr", "clone_children", "release_agent"
290  * - When mounting an existing superblock, mount options should match.
292  * - rename(2) is disallowed.
294  * - "tasks" is removed.  Everything should be at process granularity.  Use
297  * - "cgroup.procs" is not sorted.  pids will be unique unless they got
298  *   recycled in-between reads.
300  * - "release_agent" and "notify_on_release" are removed.  Replacement
303  * - "cgroup.clone_children" is removed.
305  * - "cgroup.subtree_populated" is available.  Its value is 0 if the cgroup
310  * - cpuset: tasks will be kept in empty cpusets when hotplug happens and
311  *   take masks of ancestors with non-empty cpus/mems, instead of being
314  * - cpuset: a task can be moved into an empty cpuset, and again it takes
317  * - blkcg: blk-throttle becomes properly hierarchical.
321 	return cgrp->root == &cgrp_dfl_root;  in cgroup_on_dfl()
357 	return cgrp->nr_populated_csets;  in cgroup_has_tasks()
362 	return cgrp->dom_cgrp != cgrp;  in cgroup_is_threaded()
370 	 * the no-internal-process constraint, so it can serve as a thread  in cgroup_is_mixable()
388 	if (cgrp->nr_populated_domain_children)  in cgroup_can_be_thread_root()
392 	if (cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask)  in cgroup_can_be_thread_root()
406 	if (cgrp->nr_threaded_children)  in cgroup_is_thread_root()
414 	    (cgrp->subtree_control & cgrp_dfl_threaded_ss_mask))  in cgroup_is_thread_root()
442 	u16 root_ss_mask = cgrp->root->subsys_mask;  in cgroup_control()
445 		u16 ss_mask = parent->subtree_control;  in cgroup_control()
465 		u16 ss_mask = parent->subtree_ss_mask;  in cgroup_ss_mask()
473 	return cgrp->root->subsys_mask;  in cgroup_ss_mask()
477  * cgroup_css - obtain a cgroup's css for the specified subsystem
479  * @ss: the subsystem of interest (%NULL returns @cgrp->self)
491 		return rcu_dereference_check(cgrp->subsys[ss->id],  in cgroup_css()
494 		return &cgrp->self;  in cgroup_css()
498  * cgroup_e_css_by_mask - obtain a cgroup's effective css for the specified ss
500  * @ss: the subsystem of interest (%NULL returns @cgrp->self)
505  * function is guaranteed to return non-NULL css.
513 		return &cgrp->self;  in cgroup_e_css_by_mask()
519 	while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) {  in cgroup_e_css_by_mask()
529  * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
557 	return init_css_set.subsys[ss->id];  in cgroup_e_css()
561  * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem
589 	css = init_css_set.subsys[ss->id];  in cgroup_get_e_css()
604  * __cgroup_task_count - count the number of tasks in a cgroup. The caller
615 	list_for_each_entry(link, &cgrp->cset_links, cset_link)  in __cgroup_task_count()
616 		count += link->cset->nr_tasks;  in __cgroup_task_count()
622  * cgroup_task_count - count the number of tasks in a cgroup.
638 	struct cgroup *cgrp = of->kn->parent->priv;  in of_css()
649 	if (CGROUP_HAS_SUBSYS_CONFIG && cft->ss)  in of_css()
650 		return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);  in of_css()
652 		return &cgrp->self;  in of_css()
657  * for_each_css - iterate all css's of a cgroup
667 				(cgrp)->subsys[(ssid)],			\
672  * do_each_subsys_mask - filter for_each_subsys with a bitmask
677  * The block will only run for cases where the ssid-th bit (1 << ssid) of
697 	list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
707 		       (dsct) = (d_css)->cgroup;			\
716 		       (dsct) = (d_css)->cgroup;			\
722  * The default css_set - used by init and its children prior to any
725  * reference-counted, to improve performance when child cgroups
742 	 * The following field is re-initialized when this cset gets linked
754 	return cset->dom_cset != cset;  in css_set_threaded()
758  * css_set_populated - does a css_set contain any tasks?
761  * css_set_populated() should be the same as !!cset->nr_tasks at steady
764  * properly updated. Hence, we can't just look at ->nr_tasks here.
770 	return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks);  in css_set_populated()
774  * cgroup_update_populated - update the populated count of a cgroup
779  * task or losing the last.  Update @cgrp->nr_populated_* accordingly.  The
785  * @cgrp->nr_populated_csets and @cgrp->nr_populated_children are zero and
793 	int adj = populated ? 1 : -1;  in cgroup_update_populated()
801 			cgrp->nr_populated_csets += adj;  in cgroup_update_populated()
804 				cgrp->nr_populated_threaded_children += adj;  in cgroup_update_populated()
806 				cgrp->nr_populated_domain_children += adj;  in cgroup_update_populated()
815 		cgroup_file_notify(&cgrp->events_file);  in cgroup_update_populated()
823  * css_set_update_populated - update populated state of a css_set
836 	list_for_each_entry(link, &cset->cgrp_links, cgrp_link)  in css_set_update_populated()
837 		cgroup_update_populated(link->cgrp, populated);  in css_set_update_populated()
851 	list_for_each_entry_safe(it, pos, &cset->task_iters, iters_node)  in css_set_skip_task_iters()
856  * css_set_move_task - move a task from one css_set to another
860  * @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks
880 		WARN_ON_ONCE(list_empty(&task->cg_list));  in css_set_move_task()
883 		list_del_init(&task->cg_list);  in css_set_move_task()
887 		WARN_ON_ONCE(!list_empty(&task->cg_list));  in css_set_move_task()
896 		WARN_ON_ONCE(task->flags & PF_EXITING);  in css_set_move_task()
899 		list_add_tail(&task->cg_list, use_mg_tasks ? &to_cset->mg_tasks :  in css_set_move_task()
900 							     &to_cset->tasks);  in css_set_move_task()
933 	if (!refcount_dec_and_test(&cset->refcount))  in put_css_set_locked()
936 	WARN_ON_ONCE(!list_empty(&cset->threaded_csets));  in put_css_set_locked()
940 		list_del(&cset->e_cset_node[ssid]);  in put_css_set_locked()
941 		css_put(cset->subsys[ssid]);  in put_css_set_locked()
943 	hash_del(&cset->hlist);  in put_css_set_locked()
944 	css_set_count--;  in put_css_set_locked()
946 	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {  in put_css_set_locked()
947 		list_del(&link->cset_link);  in put_css_set_locked()
948 		list_del(&link->cgrp_link);  in put_css_set_locked()
949 		if (cgroup_parent(link->cgrp))  in put_css_set_locked()
950 			cgroup_put(link->cgrp);  in put_css_set_locked()
955 		list_del(&cset->threaded_csets_node);  in put_css_set_locked()
956 		put_css_set_locked(cset->dom_cset);  in put_css_set_locked()
963  * compare_css_sets - helper function for find_existing_css_set().
967  * @template: desired set of css pointers in css_set (pre-calculated)
985 	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))  in compare_css_sets()
993 		new_dfl_cgrp = old_cset->dfl_cgrp;  in compare_css_sets()
995 	if (new_dfl_cgrp->dom_cgrp != cset->dom_cset->dfl_cgrp)  in compare_css_sets()
1004 	l1 = &cset->cgrp_links;  in compare_css_sets()
1005 	l2 = &old_cset->cgrp_links;  in compare_css_sets()
1010 		l1 = l1->next;  in compare_css_sets()
1011 		l2 = l2->next;  in compare_css_sets()
1012 		/* See if we reached the end - both lists are equal length. */  in compare_css_sets()
1013 		if (l1 == &cset->cgrp_links) {  in compare_css_sets()
1014 			BUG_ON(l2 != &old_cset->cgrp_links);  in compare_css_sets()
1017 			BUG_ON(l2 == &old_cset->cgrp_links);  in compare_css_sets()
1022 		cgrp1 = link1->cgrp;  in compare_css_sets()
1023 		cgrp2 = link2->cgrp;  in compare_css_sets()
1025 		BUG_ON(cgrp1->root != cgrp2->root);  in compare_css_sets()
1034 		if (cgrp1->root == new_cgrp->root) {  in compare_css_sets()
1046  * find_existing_css_set - init css array and find the matching css_set
1055 	struct cgroup_root *root = cgrp->root;  in find_existing_css_set()
1067 		if (root->subsys_mask & (1UL << i)) {  in find_existing_css_set()
1078 			template[i] = old_cset->subsys[i];  in find_existing_css_set()
1100 		list_del(&link->cset_link);  in free_cgrp_cset_links()
1106  * allocate_cgrp_cset_links - allocate cgrp_cset_links
1111  * through ->cset_link.  Returns 0 on success or -errno.
1124 			return -ENOMEM;  in allocate_cgrp_cset_links()
1126 		list_add(&link->cset_link, tmp_links);  in allocate_cgrp_cset_links()
1132  * link_css_set - a helper function to link a css_set to a cgroup
1145 		cset->dfl_cgrp = cgrp;  in link_css_set()
1148 	link->cset = cset;  in link_css_set()
1149 	link->cgrp = cgrp;  in link_css_set()
1155 	list_move_tail(&link->cset_link, &cgrp->cset_links);  in link_css_set()
1156 	list_add_tail(&link->cgrp_link, &cset->cgrp_links);  in link_css_set()
1163  * find_css_set - return a new css_set with one cgroup updated
1204 	refcount_set(&cset->refcount, 1);  in find_css_set()
1205 	cset->dom_cset = cset;  in find_css_set()
1206 	INIT_LIST_HEAD(&cset->tasks);  in find_css_set()
1207 	INIT_LIST_HEAD(&cset->mg_tasks);  in find_css_set()
1208 	INIT_LIST_HEAD(&cset->dying_tasks);  in find_css_set()
1209 	INIT_LIST_HEAD(&cset->task_iters);  in find_css_set()
1210 	INIT_LIST_HEAD(&cset->threaded_csets);  in find_css_set()
1211 	INIT_HLIST_NODE(&cset->hlist);  in find_css_set()
1212 	INIT_LIST_HEAD(&cset->cgrp_links);  in find_css_set()
1213 	INIT_LIST_HEAD(&cset->mg_src_preload_node);  in find_css_set()
1214 	INIT_LIST_HEAD(&cset->mg_dst_preload_node);  in find_css_set()
1215 	INIT_LIST_HEAD(&cset->mg_node);  in find_css_set()
1219 	memcpy(cset->subsys, template, sizeof(cset->subsys));  in find_css_set()
1223 	list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {  in find_css_set()
1224 		struct cgroup *c = link->cgrp;  in find_css_set()
1226 		if (c->root == cgrp->root)  in find_css_set()
1236 	key = css_set_hash(cset->subsys);  in find_css_set()
1237 	hash_add(css_set_table, &cset->hlist, key);  in find_css_set()
1240 		struct cgroup_subsys_state *css = cset->subsys[ssid];  in find_css_set()
1242 		list_add_tail(&cset->e_cset_node[ssid],  in find_css_set()
1243 			      &css->cgroup->e_csets[ssid]);  in find_css_set()
1255 	if (cgroup_is_threaded(cset->dfl_cgrp)) {  in find_css_set()
1258 		dcset = find_css_set(cset, cset->dfl_cgrp->dom_cgrp);  in find_css_set()
1265 		cset->dom_cset = dcset;  in find_css_set()
1266 		list_add_tail(&cset->threaded_csets_node,  in find_css_set()
1267 			      &dcset->threaded_csets);  in find_css_set()
1276 	struct cgroup *root_cgrp = kernfs_root_to_node(kf_root)->priv;  in cgroup_root_from_kf()
1278 	return root_cgrp->root;  in cgroup_root_from_kf()
1283 	bool favoring = root->flags & CGRP_ROOT_FAVOR_DYNMODS;  in cgroup_favor_dynmods()
1288 		root->flags |= CGRP_ROOT_FAVOR_DYNMODS;  in cgroup_favor_dynmods()
1291 		root->flags &= ~CGRP_ROOT_FAVOR_DYNMODS;  in cgroup_favor_dynmods()
1305 	root->hierarchy_id = id;  in cgroup_init_root_id()
1313 	idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);  in cgroup_exit_root_id()
1323 	struct cgroup *cgrp = &root->cgrp;  in cgroup_destroy_root()
1330 	BUG_ON(atomic_read(&root->nr_cgrps));  in cgroup_destroy_root()
1331 	BUG_ON(!list_empty(&cgrp->self.children));  in cgroup_destroy_root()
1334 	WARN_ON(rebind_subsystems(&cgrp_dfl_root, root->subsys_mask));  in cgroup_destroy_root()
1342 	list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {  in cgroup_destroy_root()
1343 		list_del(&link->cset_link);  in cgroup_destroy_root()
1344 		list_del(&link->cgrp_link);  in cgroup_destroy_root()
1350 	WARN_ON_ONCE(list_empty(&root->root_list));  in cgroup_destroy_root()
1351 	list_del_rcu(&root->root_list);  in cgroup_destroy_root()
1352 	cgroup_root_count--;  in cgroup_destroy_root()
1362 	kernfs_destroy_root(root->kf_root);  in cgroup_destroy_root()
1375 		res_cgroup = &root->cgrp;  in __cset_cgroup_from_root()
1377 		res_cgroup = cset->dfl_cgrp;  in __cset_cgroup_from_root()
1382 		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {  in __cset_cgroup_from_root()
1383 			struct cgroup *c = link->cgrp;  in __cset_cgroup_from_root()
1385 			if (c->root == root) {  in __cset_cgroup_from_root()
1418 	cset = current->nsproxy->cgroup_ns->root_cset;  in current_cgns_cgroup_from_root()
1425 	 * be umounted. Therefore, we can ensure that the res is non-NULL.  in current_cgns_cgroup_from_root()
1436  * - Internal rcu_read_lock is unnecessary because we don't dereference any rcu
1438  * - css_set_lock is not needed because we just read cset->dfl_cgrp.
1439  * - As a bonus returned cgrp is pinned with the current because it cannot
1446 	if (current->nsproxy) {  in current_cgns_cgroup_dfl()
1447 		cset = current->nsproxy->cgroup_ns->root_cset;  in current_cgns_cgroup_dfl()
1479 	 * No need to lock the task - since we hold css_set_lock the  in task_cgroup_from_root()
1516 	struct cgroup_subsys *ss = cft->ss;  in cgroup_file_name()
1518 	if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&  in cgroup_file_name()
1519 	    !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) {  in cgroup_file_name()
1520 		const char *dbg = (cft->flags & CFTYPE_DEBUG) ? ".__DEBUG__." : "";  in cgroup_file_name()
1523 			 dbg, cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name,  in cgroup_file_name()
1524 			 cft->name);  in cgroup_file_name()
1526 		strscpy(buf, cft->name, CGROUP_FILE_NAME_MAX);  in cgroup_file_name()
1532  * cgroup_file_mode - deduce file mode of a control file
1541 	if (cft->read_u64 || cft->read_s64 || cft->seq_show)  in cgroup_file_mode()
1544 	if (cft->write_u64 || cft->write_s64 || cft->write) {  in cgroup_file_mode()
1545 		if (cft->flags & CFTYPE_WORLD_WRITABLE)  in cgroup_file_mode()
1555  * cgroup_calc_subtree_ss_mask - calculate subtree_ss_mask
1560  * enabled together through its ->depends_on mask.  In such cases, more
1580 			new_ss_mask |= ss->depends_on;  in cgroup_calc_subtree_ss_mask()
1585 		 * happen only if some depended-upon subsystems were bound  in cgroup_calc_subtree_ss_mask()
1586 		 * to non-default hierarchies.  in cgroup_calc_subtree_ss_mask()
1599  * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
1613 		cgrp = kn->priv;  in cgroup_kn_unlock()
1615 		cgrp = kn->parent->priv;  in cgroup_kn_unlock()
1624  * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
1638  * including self-removal.
1645 		cgrp = kn->priv;  in cgroup_kn_lock_live()
1647 		cgrp = kn->parent->priv;  in cgroup_kn_lock_live()
1677 	if (cft->file_offset) {  in cgroup_rm_file()
1678 		struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss);  in cgroup_rm_file()
1679 		struct cgroup_file *cfile = (void *)css + cft->file_offset;  in cgroup_rm_file()
1682 		cfile->kn = NULL;  in cgroup_rm_file()
1685 		del_timer_sync(&cfile->notify_timer);  in cgroup_rm_file()
1688 	kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));  in cgroup_rm_file()
1692  * css_clear_dir - remove subsys files in a cgroup directory
1697 	struct cgroup *cgrp = css->cgroup;  in css_clear_dir()
1700 	if (!(css->flags & CSS_VISIBLE))  in css_clear_dir()
1703 	css->flags &= ~CSS_VISIBLE;  in css_clear_dir()
1705 	if (!css->ss) {  in css_clear_dir()
1717 		list_for_each_entry(cfts, &css->ss->cfts, node)  in css_clear_dir()
1723  * css_populate_dir - create subsys files in a cgroup directory
1730 	struct cgroup *cgrp = css->cgroup;  in css_populate_dir()
1734 	if (css->flags & CSS_VISIBLE)  in css_populate_dir()
1737 	if (!css->ss) {  in css_populate_dir()
1760 		list_for_each_entry(cfts, &css->ss->cfts, node) {  in css_populate_dir()
1769 	css->flags |= CSS_VISIBLE;  in css_populate_dir()
1773 	list_for_each_entry(cfts, &css->ss->cfts, node) {  in css_populate_dir()
1783 	struct cgroup *dcgrp = &dst_root->cgrp;  in rebind_subsystems()
1792 		 * If @ss has non-root csses attached to it, can't move.  in rebind_subsystems()
1796 		if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) &&  in rebind_subsystems()
1797 		    !ss->implicit_on_dfl)  in rebind_subsystems()
1798 			return -EBUSY;  in rebind_subsystems()
1800 		/* can't move between two non-dummy roots either */  in rebind_subsystems()
1801 		if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)  in rebind_subsystems()
1802 			return -EBUSY;  in rebind_subsystems()
1808 		if (ss->root == &cgrp_dfl_root)  in rebind_subsystems()
1826 		struct cgroup_root *src_root = ss->root;  in rebind_subsystems()
1827 		struct cgroup *scgrp = &src_root->cgrp;  in rebind_subsystems()
1836 			src_root->subsys_mask &= ~(1 << ssid);  in rebind_subsystems()
1842 		RCU_INIT_POINTER(scgrp->subsys[ssid], NULL);  in rebind_subsystems()
1843 		rcu_assign_pointer(dcgrp->subsys[ssid], css);  in rebind_subsystems()
1844 		ss->root = dst_root;  in rebind_subsystems()
1847 		css->cgroup = dcgrp;  in rebind_subsystems()
1848 		WARN_ON(!list_empty(&dcgrp->e_csets[ss->id]));  in rebind_subsystems()
1849 		list_for_each_entry_safe(cset, cset_pos, &scgrp->e_csets[ss->id],  in rebind_subsystems()
1850 					 e_cset_node[ss->id]) {  in rebind_subsystems()
1851 			list_move_tail(&cset->e_cset_node[ss->id],  in rebind_subsystems()
1852 				       &dcgrp->e_csets[ss->id]);  in rebind_subsystems()
1855 			 * patch in-flight iterators to preserve correct iteration.  in rebind_subsystems()
1857 			 * finished when it->cset_pos meets it->cset_head, so only  in rebind_subsystems()
1858 			 * update it->cset_head is enough here.  in rebind_subsystems()
1860 			list_for_each_entry(it, &cset->task_iters, iters_node)  in rebind_subsystems()
1861 				if (it->cset_head == &scgrp->e_csets[ss->id])  in rebind_subsystems()
1862 					it->cset_head = &dcgrp->e_csets[ss->id];  in rebind_subsystems()
1866 		if (ss->css_rstat_flush) {  in rebind_subsystems()
1867 			list_del_rcu(&css->rstat_css_node);  in rebind_subsystems()
1869 			list_add_rcu(&css->rstat_css_node,  in rebind_subsystems()
1870 				     &dcgrp->rstat_css_list);  in rebind_subsystems()
1874 		dst_root->subsys_mask |= 1 << ssid;  in rebind_subsystems()
1878 			dcgrp->subtree_control |= 1 << ssid;  in rebind_subsystems()
1885 				ss->name, ret);  in rebind_subsystems()
1887 		if (ss->bind)  in rebind_subsystems()
1888 			ss->bind(css);  in rebind_subsystems()
1891 	kernfs_activate(dcgrp->kn);  in rebind_subsystems()
1905 		return -ENOMEM;  in cgroup_show_path()
1909 	len = kernfs_path_from_node(kf_node, ns_cgroup->kn, buf, PATH_MAX);  in cgroup_show_path()
1912 	if (len == -E2BIG)  in cgroup_show_path()
1913 		len = -ERANGE;  in cgroup_show_path()
1954 		ctx->flags |= CGRP_ROOT_NS_DELEGATE;  in cgroup2_parse_param()
1957 		ctx->flags |= CGRP_ROOT_FAVOR_DYNMODS;  in cgroup2_parse_param()
1960 		ctx->flags |= CGRP_ROOT_MEMORY_LOCAL_EVENTS;  in cgroup2_parse_param()
1963 		ctx->flags |= CGRP_ROOT_MEMORY_RECURSIVE_PROT;  in cgroup2_parse_param()
1966 		ctx->flags |= CGRP_ROOT_MEMORY_HUGETLB_ACCOUNTING;  in cgroup2_parse_param()
1969 		ctx->flags |= CGRP_ROOT_PIDS_LOCAL_EVENTS;  in cgroup2_parse_param()
1972 	return -EINVAL;  in cgroup2_parse_param()
1977 	struct cgroup_file_ctx *ctx = of->priv;  in of_peak()
1979 	return &ctx->peak;  in of_peak()
1984 	if (current->nsproxy->cgroup_ns == &init_cgroup_ns) {  in apply_cgroup_root_flags()
2036 	apply_cgroup_root_flags(ctx->flags);  in cgroup_reconfigure()
2045 	INIT_LIST_HEAD(&cgrp->self.sibling);  in init_cgroup_housekeeping()
2046 	INIT_LIST_HEAD(&cgrp->self.children);  in init_cgroup_housekeeping()
2047 	INIT_LIST_HEAD(&cgrp->cset_links);  in init_cgroup_housekeeping()
2048 	INIT_LIST_HEAD(&cgrp->pidlists);  in init_cgroup_housekeeping()
2049 	mutex_init(&cgrp->pidlist_mutex);  in init_cgroup_housekeeping()
2050 	cgrp->self.cgroup = cgrp;  in init_cgroup_housekeeping()
2051 	cgrp->self.flags |= CSS_ONLINE;  in init_cgroup_housekeeping()
2052 	cgrp->dom_cgrp = cgrp;  in init_cgroup_housekeeping()
2053 	cgrp->max_descendants = INT_MAX;  in init_cgroup_housekeeping()
2054 	cgrp->max_depth = INT_MAX;  in init_cgroup_housekeeping()
2055 	INIT_LIST_HEAD(&cgrp->rstat_css_list);  in init_cgroup_housekeeping()
2056 	prev_cputime_init(&cgrp->prev_cputime);  in init_cgroup_housekeeping()
2059 		INIT_LIST_HEAD(&cgrp->e_csets[ssid]);  in init_cgroup_housekeeping()
2061 	init_waitqueue_head(&cgrp->offline_waitq);  in init_cgroup_housekeeping()
2062 	INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent);  in init_cgroup_housekeeping()
2067 	struct cgroup_root *root = ctx->root;  in init_cgroup_root()
2068 	struct cgroup *cgrp = &root->cgrp;  in init_cgroup_root()
2070 	INIT_LIST_HEAD_RCU(&root->root_list);  in init_cgroup_root()
2071 	atomic_set(&root->nr_cgrps, 1);  in init_cgroup_root()
2072 	cgrp->root = root;  in init_cgroup_root()
2076 	root->flags = ctx->flags & ~CGRP_ROOT_FAVOR_DYNMODS;  in init_cgroup_root()
2077 	if (ctx->release_agent)  in init_cgroup_root()
2078 		strscpy(root->release_agent_path, ctx->release_agent, PATH_MAX);  in init_cgroup_root()
2079 	if (ctx->name)  in init_cgroup_root()
2080 		strscpy(root->name, ctx->name, MAX_CGROUP_ROOT_NAMELEN);  in init_cgroup_root()
2081 	if (ctx->cpuset_clone_children)  in init_cgroup_root()
2082 		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);  in init_cgroup_root()
2088 	struct cgroup *root_cgrp = &root->cgrp;  in cgroup_setup_root()
2095 	ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release,  in cgroup_setup_root()
2102 	 * but that's OK - it can only be increased by someone holding  in cgroup_setup_root()
2118 	root->kf_root = kernfs_create_root(kf_sops,  in cgroup_setup_root()
2123 	if (IS_ERR(root->kf_root)) {  in cgroup_setup_root()
2124 		ret = PTR_ERR(root->kf_root);  in cgroup_setup_root()
2127 	root_cgrp->kn = kernfs_root_to_node(root->kf_root);  in cgroup_setup_root()
2129 	root_cgrp->ancestors[0] = root_cgrp;  in cgroup_setup_root()
2131 	ret = css_populate_dir(&root_cgrp->self);  in cgroup_setup_root()
2155 	list_add_rcu(&root->root_list, &cgroup_roots);  in cgroup_setup_root()
2170 	BUG_ON(!list_empty(&root_cgrp->self.children));  in cgroup_setup_root()
2171 	BUG_ON(atomic_read(&root->nr_cgrps) != 1);  in cgroup_setup_root()
2179 	kernfs_destroy_root(root->kf_root);  in cgroup_setup_root()
2180 	root->kf_root = NULL;  in cgroup_setup_root()
2184 	percpu_ref_exit(&root_cgrp->self.refcnt);  in cgroup_setup_root()
2195 	ctx->kfc.root = ctx->root->kf_root;  in cgroup_do_get_tree()
2196 	if (fc->fs_type == &cgroup2_fs_type)  in cgroup_do_get_tree()
2197 		ctx->kfc.magic = CGROUP2_SUPER_MAGIC;  in cgroup_do_get_tree()
2199 		ctx->kfc.magic = CGROUP_SUPER_MAGIC;  in cgroup_do_get_tree()
2203 	 * In non-init cgroup namespace, instead of root cgroup's dentry,  in cgroup_do_get_tree()
2204 	 * we return the dentry corresponding to the cgroupns->root_cgrp.  in cgroup_do_get_tree()
2206 	if (!ret && ctx->ns != &init_cgroup_ns) {  in cgroup_do_get_tree()
2208 		struct super_block *sb = fc->root->d_sb;  in cgroup_do_get_tree()
2214 		cgrp = cset_cgroup_from_root(ctx->ns->root_cset, ctx->root);  in cgroup_do_get_tree()
2219 		nsdentry = kernfs_node_dentry(cgrp->kn, sb);  in cgroup_do_get_tree()
2220 		dput(fc->root);  in cgroup_do_get_tree()
2226 		fc->root = nsdentry;  in cgroup_do_get_tree()
2229 	if (!ctx->kfc.new_sb_created)  in cgroup_do_get_tree()
2230 		cgroup_put(&ctx->root->cgrp);  in cgroup_do_get_tree()
2242 	kfree(ctx->name);  in cgroup_fs_context_free()
2243 	kfree(ctx->release_agent);  in cgroup_fs_context_free()
2244 	put_cgroup_ns(ctx->ns);  in cgroup_fs_context_free()
2256 	ctx->root = &cgrp_dfl_root;  in cgroup_get_tree()
2260 		apply_cgroup_root_flags(ctx->flags);  in cgroup_get_tree()
2280  * we select the namespace we're going to use.
2288 		return -ENOMEM;  in cgroup_init_fs_context()
2290 	ctx->ns = current->nsproxy->cgroup_ns;  in cgroup_init_fs_context()
2291 	get_cgroup_ns(ctx->ns);  in cgroup_init_fs_context()
2292 	fc->fs_private = &ctx->kfc;  in cgroup_init_fs_context()
2293 	if (fc->fs_type == &cgroup2_fs_type)  in cgroup_init_fs_context()
2294 		fc->ops = &cgroup_fs_context_ops;  in cgroup_init_fs_context()
2296 		fc->ops = &cgroup1_fs_context_ops;  in cgroup_init_fs_context()
2297 	put_user_ns(fc->user_ns);  in cgroup_init_fs_context()
2298 	fc->user_ns = get_user_ns(ctx->ns->user_ns);  in cgroup_init_fs_context()
2299 	fc->global = true;  in cgroup_init_fs_context()
2302 		ctx->flags |= CGRP_ROOT_FAVOR_DYNMODS;  in cgroup_init_fs_context()
2318 	if (list_empty(&root->cgrp.self.children) && root != &cgrp_dfl_root &&  in cgroup_kill_sb()
2319 	    !percpu_ref_is_dying(&root->cgrp.self.refcnt))  in cgroup_kill_sb()
2320 		percpu_ref_kill(&root->cgrp.self.refcnt);  in cgroup_kill_sb()
2321 	cgroup_put(&root->cgrp);  in cgroup_kill_sb()
2364 	fc->ops = &cpuset_fs_context_ops;  in cpuset_init_fs_context()
2367 	ctx->subsys_mask = 1 << cpuset_cgrp_id;  in cpuset_init_fs_context()
2368 	ctx->flags |= CGRP_ROOT_NOPREFIX;  in cpuset_init_fs_context()
2369 	ctx->release_agent = agent;  in cpuset_init_fs_context()
2372 	put_filesystem(fc->fs_type);  in cpuset_init_fs_context()
2373 	fc->fs_type = &cgroup_fs_type;  in cpuset_init_fs_context()
2388 	struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root);  in cgroup_path_ns_locked()
2390 	return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen);  in cgroup_path_ns_locked()
2411  * cgroup_attach_lock - Lock for ->attach()
2415  * exits by write-locking cgroup_threadgroup_rwsem. However, some ->attach()
2417  * Unfortunately, letting ->attach() operations acquire cpus_read_lock() can
2421  * read-locking threadgroup_rwsem, so threadgroup_rwsem nests inside
2422  * cpus_read_lock(). If we call an ->attach() which acquires the cpus lock while
2423  * write-locking threadgroup_rwsem, the locking order is reversed and we end up
2424  * waiting for an on-going CPU hotplug operation which in turn is waiting for
2430  * write-locking cgroup_threadgroup_rwsem. This allows ->attach() to assume that
2441  * cgroup_attach_unlock - Undo cgroup_attach_lock()
2452  * cgroup_migrate_add_task - add a migration target task to a migration context
2456  * Add @task, which is a migration target, to @mgctx->tset.  This function
2458  * should have been added as a migration source and @task->cg_list will be
2469 	if (task->flags & PF_EXITING)  in cgroup_migrate_add_task()
2473 	WARN_ON_ONCE(list_empty(&task->cg_list));  in cgroup_migrate_add_task()
2476 	if (!cset->mg_src_cgrp)  in cgroup_migrate_add_task()
2479 	mgctx->tset.nr_tasks++;  in cgroup_migrate_add_task()
2481 	list_move_tail(&task->cg_list, &cset->mg_tasks);  in cgroup_migrate_add_task()
2482 	if (list_empty(&cset->mg_node))  in cgroup_migrate_add_task()
2483 		list_add_tail(&cset->mg_node,  in cgroup_migrate_add_task()
2484 			      &mgctx->tset.src_csets);  in cgroup_migrate_add_task()
2485 	if (list_empty(&cset->mg_dst_cset->mg_node))  in cgroup_migrate_add_task()
2486 		list_add_tail(&cset->mg_dst_cset->mg_node,  in cgroup_migrate_add_task()
2487 			      &mgctx->tset.dst_csets);  in cgroup_migrate_add_task()
2491  * cgroup_taskset_first - reset taskset and return the first task
2500 	tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);  in cgroup_taskset_first()
2501 	tset->cur_task = NULL;  in cgroup_taskset_first()
2507  * cgroup_taskset_next - iterate to the next task in taskset
2517 	struct css_set *cset = tset->cur_cset;  in cgroup_taskset_next()
2518 	struct task_struct *task = tset->cur_task;  in cgroup_taskset_next()
2520 	while (CGROUP_HAS_SUBSYS_CONFIG && &cset->mg_node != tset->csets) {  in cgroup_taskset_next()
2522 			task = list_first_entry(&cset->mg_tasks,  in cgroup_taskset_next()
2527 		if (&task->cg_list != &cset->mg_tasks) {  in cgroup_taskset_next()
2528 			tset->cur_cset = cset;  in cgroup_taskset_next()
2529 			tset->cur_task = task;  in cgroup_taskset_next()
2535 			 * has its ->mg_dst_cset set.  in cgroup_taskset_next()
2537 			if (cset->mg_dst_cset)  in cgroup_taskset_next()
2538 				*dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];  in cgroup_taskset_next()
2540 				*dst_cssp = cset->subsys[tset->ssid];  in cgroup_taskset_next()
2553  * cgroup_migrate_execute - migrate a taskset
2557  * This function fails iff one of the ->can_attach callbacks fails and
2563 	struct cgroup_taskset *tset = &mgctx->tset;  in cgroup_migrate_execute()
2570 	if (tset->nr_tasks) {  in cgroup_migrate_execute()
2571 		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {  in cgroup_migrate_execute()
2572 			if (ss->can_attach) {  in cgroup_migrate_execute()
2573 				tset->ssid = ssid;  in cgroup_migrate_execute()
2574 				ret = ss->can_attach(tset);  in cgroup_migrate_execute()
2589 	list_for_each_entry(cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2590 		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {  in cgroup_migrate_execute()
2592 			struct css_set *to_cset = cset->mg_dst_cset;  in cgroup_migrate_execute()
2595 			to_cset->nr_tasks++;  in cgroup_migrate_execute()
2597 			from_cset->nr_tasks--;  in cgroup_migrate_execute()
2602 			cgroup_freezer_migrate_task(task, from_cset->dfl_cgrp,  in cgroup_migrate_execute()
2603 						    to_cset->dfl_cgrp);  in cgroup_migrate_execute()
2615 	tset->csets = &tset->dst_csets;  in cgroup_migrate_execute()
2617 	if (tset->nr_tasks) {  in cgroup_migrate_execute()
2618 		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {  in cgroup_migrate_execute()
2619 			if (ss->attach) {  in cgroup_migrate_execute()
2620 				tset->ssid = ssid;  in cgroup_migrate_execute()
2621 				ss->attach(tset);  in cgroup_migrate_execute()
2630 	if (tset->nr_tasks) {  in cgroup_migrate_execute()
2631 		do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {  in cgroup_migrate_execute()
2634 			if (ss->cancel_attach) {  in cgroup_migrate_execute()
2635 				tset->ssid = ssid;  in cgroup_migrate_execute()
2636 				ss->cancel_attach(tset);  in cgroup_migrate_execute()
2642 	list_splice_init(&tset->dst_csets, &tset->src_csets);  in cgroup_migrate_execute()
2643 	list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2644 		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);  in cgroup_migrate_execute()
2645 		list_del_init(&cset->mg_node);  in cgroup_migrate_execute()
2650 	 * Re-initialize the cgroup_taskset structure in case it is reused  in cgroup_migrate_execute()
2654 	tset->nr_tasks = 0;  in cgroup_migrate_execute()
2655 	tset->csets    = &tset->src_csets;  in cgroup_migrate_execute()
2660  * cgroup_migrate_vet_dst - verify whether a cgroup can be migration destination
2675 	if (!cgroup_is_valid_domain(dst_cgrp->dom_cgrp))  in cgroup_migrate_vet_dst()
2676 		return -EOPNOTSUPP;  in cgroup_migrate_vet_dst()
2685 	/* apply no-internal-process constraint */  in cgroup_migrate_vet_dst()
2686 	if (dst_cgrp->subtree_control)  in cgroup_migrate_vet_dst()
2687 		return -EBUSY;  in cgroup_migrate_vet_dst()
2693  * cgroup_migrate_finish - cleanup after attach
2707 	list_for_each_entry_safe(cset, tmp_cset, &mgctx->preloaded_src_csets,  in cgroup_migrate_finish()
2709 		cset->mg_src_cgrp = NULL;  in cgroup_migrate_finish()
2710 		cset->mg_dst_cgrp = NULL;  in cgroup_migrate_finish()
2711 		cset->mg_dst_cset = NULL;  in cgroup_migrate_finish()
2712 		list_del_init(&cset->mg_src_preload_node);  in cgroup_migrate_finish()
2716 	list_for_each_entry_safe(cset, tmp_cset, &mgctx->preloaded_dst_csets,  in cgroup_migrate_finish()
2718 		cset->mg_src_cgrp = NULL;  in cgroup_migrate_finish()
2719 		cset->mg_dst_cgrp = NULL;  in cgroup_migrate_finish()
2720 		cset->mg_dst_cset = NULL;  in cgroup_migrate_finish()
2721 		list_del_init(&cset->mg_dst_preload_node);  in cgroup_migrate_finish()
2729  * cgroup_migrate_add_src - add a migration source css_set
2735  * @src_cset and add it to @mgctx->src_csets, which should later be cleaned
2754 	 * If ->dead, @src_set is associated with one or more dead cgroups  in cgroup_migrate_add_src()
2758 	if (src_cset->dead)  in cgroup_migrate_add_src()
2761 	if (!list_empty(&src_cset->mg_src_preload_node))  in cgroup_migrate_add_src()
2764 	src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);  in cgroup_migrate_add_src()
2766 	WARN_ON(src_cset->mg_src_cgrp);  in cgroup_migrate_add_src()
2767 	WARN_ON(src_cset->mg_dst_cgrp);  in cgroup_migrate_add_src()
2768 	WARN_ON(!list_empty(&src_cset->mg_tasks));  in cgroup_migrate_add_src()
2769 	WARN_ON(!list_empty(&src_cset->mg_node));  in cgroup_migrate_add_src()
2771 	src_cset->mg_src_cgrp = src_cgrp;  in cgroup_migrate_add_src()
2772 	src_cset->mg_dst_cgrp = dst_cgrp;  in cgroup_migrate_add_src()
2774 	list_add_tail(&src_cset->mg_src_preload_node, &mgctx->preloaded_src_csets);  in cgroup_migrate_add_src()
2778  * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2782  * preloaded to @mgctx->preloaded_src_csets.  This function looks up and
2784  * to @mgctx->preloaded_dst_csets.
2798 	list_for_each_entry_safe(src_cset, tmp_cset, &mgctx->preloaded_src_csets,  in cgroup_migrate_prepare_dst()
2804 		dst_cset = find_css_set(src_cset, src_cset->mg_dst_cgrp);  in cgroup_migrate_prepare_dst()
2806 			return -ENOMEM;  in cgroup_migrate_prepare_dst()
2808 		WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);  in cgroup_migrate_prepare_dst()
2816 			src_cset->mg_src_cgrp = NULL;  in cgroup_migrate_prepare_dst()
2817 			src_cset->mg_dst_cgrp = NULL;  in cgroup_migrate_prepare_dst()
2818 			list_del_init(&src_cset->mg_src_preload_node);  in cgroup_migrate_prepare_dst()
2824 		src_cset->mg_dst_cset = dst_cset;  in cgroup_migrate_prepare_dst()
2826 		if (list_empty(&dst_cset->mg_dst_preload_node))  in cgroup_migrate_prepare_dst()
2827 			list_add_tail(&dst_cset->mg_dst_preload_node,  in cgroup_migrate_prepare_dst()
2828 				      &mgctx->preloaded_dst_csets);  in cgroup_migrate_prepare_dst()
2833 			if (src_cset->subsys[ssid] != dst_cset->subsys[ssid])  in cgroup_migrate_prepare_dst()
2834 				mgctx->ss_mask |= 1 << ssid;  in cgroup_migrate_prepare_dst()
2841  * cgroup_migrate - migrate a process or task to a cgroup
2852  * As long as a controller's ->can_attach() doesn't fail, this function is
2853  * guaranteed to succeed.  This means that, excluding ->can_attach()
2881  * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
2927 		return ERR_PTR(-EINVAL);  in cgroup_procs_write_start()
2945 			tsk = ERR_PTR(-ESRCH);  in cgroup_procs_write_start()
2953 		tsk = tsk->group_leader;  in cgroup_procs_write_start()
2957 	 * If userland migrates such a kthread to a non-root cgroup, it can  in cgroup_procs_write_start()
2961 	if (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY)) {  in cgroup_procs_write_start()
2962 		tsk = ERR_PTR(-EINVAL);  in cgroup_procs_write_start()
2988 		if (ss->post_attach)  in cgroup_procs_write_finish()
2989 			ss->post_attach();  in cgroup_procs_write_finish()
3001 		seq_puts(seq, ss->name);  in cgroup_print_ss_mask()
3011 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_controllers_show()
3020 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_subtree_control_show()
3022 	cgroup_print_ss_mask(seq, cgrp->subtree_control);  in cgroup_subtree_control_show()
3027  * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
3060 		list_for_each_entry(link, &dsct->cset_links, cset_link)  in cgroup_update_dfl_csses()
3061 			cgroup_migrate_add_src(link->cset, dsct, &mgctx);  in cgroup_update_dfl_csses()
3066 	 * We need to write-lock threadgroup_rwsem while migrating tasks.  in cgroup_update_dfl_csses()
3069 	 * write-locking can be skipped safely.  in cgroup_update_dfl_csses()
3085 		list_for_each_entry_safe(task, ntask, &src_cset->tasks, cg_list)  in cgroup_update_dfl_csses()
3098  * cgroup_lock_and_drain_offline - lock cgroup_mutex and drain offlined csses
3101  * Because css offlining is asynchronous, userland may try to re-enable a
3121 			if (!css || !percpu_ref_is_dying(&css->refcnt))  in cgroup_lock_and_drain_offline()
3125 			prepare_to_wait(&dsct->offline_waitq, &wait,  in cgroup_lock_and_drain_offline()
3130 			finish_wait(&dsct->offline_waitq, &wait);  in cgroup_lock_and_drain_offline()
3139  * cgroup_save_control - save control masks and dom_cgrp of a subtree
3142  * Save ->subtree_control, ->subtree_ss_mask and ->dom_cgrp to the
3152 		dsct->old_subtree_control = dsct->subtree_control;  in cgroup_save_control()
3153 		dsct->old_subtree_ss_mask = dsct->subtree_ss_mask;  in cgroup_save_control()
3154 		dsct->old_dom_cgrp = dsct->dom_cgrp;  in cgroup_save_control()
3159  * cgroup_propagate_control - refresh control masks of a subtree
3162  * For @cgrp and its subtree, ensure ->subtree_ss_mask matches
3163  * ->subtree_control and propagate controller availability through the
3172 		dsct->subtree_control &= cgroup_control(dsct);  in cgroup_propagate_control()
3173 		dsct->subtree_ss_mask =  in cgroup_propagate_control()
3174 			cgroup_calc_subtree_ss_mask(dsct->subtree_control,  in cgroup_propagate_control()
3180  * cgroup_restore_control - restore control masks and dom_cgrp of a subtree
3183  * Restore ->subtree_control, ->subtree_ss_mask and ->dom_cgrp from the
3193 		dsct->subtree_control = dsct->old_subtree_control;  in cgroup_restore_control()
3194 		dsct->subtree_ss_mask = dsct->old_subtree_ss_mask;  in cgroup_restore_control()
3195 		dsct->dom_cgrp = dsct->old_dom_cgrp;  in cgroup_restore_control()
3201 	struct cgroup_subsys *ss = css->ss;  in css_visible()
3202 	struct cgroup *cgrp = css->cgroup;  in css_visible()
3204 	if (cgroup_control(cgrp) & (1 << ss->id))  in css_visible()
3206 	if (!(cgroup_ss_mask(cgrp) & (1 << ss->id)))  in css_visible()
3208 	return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl;  in css_visible()
3212  * cgroup_apply_control_enable - enable or show csses according to control
3220  * Returns 0 on success, -errno on failure.  On failure, csses which have
3235 			if (!(cgroup_ss_mask(dsct) & (1 << ss->id)))  in cgroup_apply_control_enable()
3244 			WARN_ON_ONCE(percpu_ref_is_dying(&css->refcnt));  in cgroup_apply_control_enable()
3258  * cgroup_apply_control_disable - kill or hide csses according to control
3267  * Controllers which may be depended upon should provide ->css_reset() for
3284 			WARN_ON_ONCE(percpu_ref_is_dying(&css->refcnt));  in cgroup_apply_control_disable()
3286 			if (css->parent &&  in cgroup_apply_control_disable()
3287 			    !(cgroup_ss_mask(dsct) & (1 << ss->id))) {  in cgroup_apply_control_disable()
3291 				if (ss->css_reset)  in cgroup_apply_control_disable()
3292 					ss->css_reset(css);  in cgroup_apply_control_disable()
3299  * cgroup_apply_control - apply control mask updates to the subtree
3306  * 2. Update ->subtree_control masks in the subtree as desired.
3334  * cgroup_finalize_control - finalize control mask update
3359 	if (!cgroup_is_valid_domain(cgrp->dom_cgrp))  in cgroup_vet_subtree_control_enable()
3360 		return -EOPNOTSUPP;  in cgroup_vet_subtree_control_enable()
3369 			return -EOPNOTSUPP;  in cgroup_vet_subtree_control_enable()
3385 		return -EBUSY;  in cgroup_vet_subtree_control_enable()
3402 	 * Parse input - space separated list of subsystem names prefixed  in cgroup_subtree_control_write()
3403 	 * with either + or -.  in cgroup_subtree_control_write()
3411 			    strcmp(tok + 1, ss->name))  in cgroup_subtree_control_write()
3417 			} else if (*tok == '-') {  in cgroup_subtree_control_write()
3421 				return -EINVAL;  in cgroup_subtree_control_write()
3426 			return -EINVAL;  in cgroup_subtree_control_write()
3429 	cgrp = cgroup_kn_lock_live(of->kn, true);  in cgroup_subtree_control_write()
3431 		return -ENODEV;  in cgroup_subtree_control_write()
3435 			if (cgrp->subtree_control & (1 << ssid)) {  in cgroup_subtree_control_write()
3441 				ret = -ENOENT;  in cgroup_subtree_control_write()
3445 			if (!(cgrp->subtree_control & (1 << ssid))) {  in cgroup_subtree_control_write()
3452 				if (child->subtree_control & (1 << ssid)) {  in cgroup_subtree_control_write()
3453 					ret = -EBUSY;  in cgroup_subtree_control_write()
3472 	cgrp->subtree_control |= enable;  in cgroup_subtree_control_write()
3473 	cgrp->subtree_control &= ~disable;  in cgroup_subtree_control_write()
3480 	kernfs_activate(cgrp->kn);  in cgroup_subtree_control_write()
3482 	cgroup_kn_unlock(of->kn);  in cgroup_subtree_control_write()
3487  * cgroup_enable_threaded - make @cgrp threaded
3498 	struct cgroup *dom_cgrp = parent->dom_cgrp;  in cgroup_enable_threaded()
3516 	    cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask)  in cgroup_enable_threaded()
3517 		return -EOPNOTSUPP;  in cgroup_enable_threaded()
3522 		return -EOPNOTSUPP;  in cgroup_enable_threaded()
3532 			dsct->dom_cgrp = dom_cgrp;  in cgroup_enable_threaded()
3536 		parent->nr_threaded_children++;  in cgroup_enable_threaded()
3544 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_type_show()
3566 		return -EINVAL;  in cgroup_type_write()
3568 	/* drain dying csses before we re-apply (threaded) subtree control */  in cgroup_type_write()
3569 	cgrp = cgroup_kn_lock_live(of->kn, true);  in cgroup_type_write()
3571 		return -ENOENT;  in cgroup_type_write()
3576 	cgroup_kn_unlock(of->kn);  in cgroup_type_write()
3582 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_max_descendants_show()
3583 	int descendants = READ_ONCE(cgrp->max_descendants);  in cgroup_max_descendants_show()
3610 		return -ERANGE;  in cgroup_max_descendants_write()
3612 	cgrp = cgroup_kn_lock_live(of->kn, false);  in cgroup_max_descendants_write()
3614 		return -ENOENT;  in cgroup_max_descendants_write()
3616 	cgrp->max_descendants = descendants;  in cgroup_max_descendants_write()
3618 	cgroup_kn_unlock(of->kn);  in cgroup_max_descendants_write()
3625 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_max_depth_show()
3626 	int depth = READ_ONCE(cgrp->max_depth);  in cgroup_max_depth_show()
3653 		return -ERANGE;  in cgroup_max_depth_write()
3655 	cgrp = cgroup_kn_lock_live(of->kn, false);  in cgroup_max_depth_write()
3657 		return -ENOENT;  in cgroup_max_depth_write()
3659 	cgrp->max_depth = depth;  in cgroup_max_depth_write()
3661 	cgroup_kn_unlock(of->kn);  in cgroup_max_depth_write()
3668 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_events_show()
3671 	seq_printf(seq, "frozen %d\n", test_bit(CGRP_FROZEN, &cgrp->flags));  in cgroup_events_show()
3678 	struct cgroup *cgroup = seq_css(seq)->cgroup;  in cgroup_stat_show()
3684 		   cgroup->nr_descendants);  in cgroup_stat_show()
3688 	 * non-inhibited cgroup subsystems that is bound to cgroup v2.  in cgroup_stat_show()
3695 		dying_cnt[ssid] = -1;  in cgroup_stat_show()
3697 		    (cgroup_subsys[ssid]->root !=  &cgrp_dfl_root))  in cgroup_stat_show()
3699 		css = rcu_dereference_raw(cgroup->subsys[ssid]);  in cgroup_stat_show()
3700 		dying_cnt[ssid] = cgroup->nr_dying_subsys[ssid];  in cgroup_stat_show()
3701 		seq_printf(seq, "nr_subsys_%s %d\n", cgroup_subsys[ssid]->name,  in cgroup_stat_show()
3702 			   css ? (css->nr_descendants + 1) : 0);  in cgroup_stat_show()
3706 		   cgroup->nr_dying_descendants);  in cgroup_stat_show()
3710 				   cgroup_subsys[ssid]->name, dying_cnt[ssid]);  in cgroup_stat_show()
3718  * cgroup_tryget_css - try to get a cgroup's css for the specified subsystem
3741 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_extra_stat_show()
3746 	if (!ss->css_extra_stat_show)  in cgroup_extra_stat_show()
3753 	ret = ss->css_extra_stat_show(seq, css);  in cgroup_extra_stat_show()
3765 	if (!ss->css_local_stat_show)  in cgroup_local_stat_show()
3772 	ret = ss->css_local_stat_show(seq, css);  in cgroup_local_stat_show()
3791 	struct cgroup __maybe_unused *cgrp = seq_css(seq)->cgroup;  in cpu_local_stat_show()
3803 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_io_pressure_show()
3810 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_memory_pressure_show()
3817 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_cpu_pressure_show()
3826 	struct cgroup_file_ctx *ctx = of->priv;  in pressure_write()
3831 	cgrp = cgroup_kn_lock_live(of->kn, false);  in pressure_write()
3833 		return -ENODEV;  in pressure_write()
3836 	cgroup_kn_unlock(of->kn);  in pressure_write()
3839 	if (ctx->psi.trigger) {  in pressure_write()
3841 		return -EBUSY;  in pressure_write()
3845 	new = psi_trigger_create(psi, buf, res, of->file, of);  in pressure_write()
3851 	smp_store_release(&ctx->psi.trigger, new);  in pressure_write()
3881 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_irq_pressure_show()
3897 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_pressure_show()
3900 	seq_printf(seq, "%d\n", psi->enabled);  in cgroup_pressure_show()
3919 		return -ERANGE;  in cgroup_pressure_write()
3921 	cgrp = cgroup_kn_lock_live(of->kn, false);  in cgroup_pressure_write()
3923 		return -ENOENT;  in cgroup_pressure_write()
3926 	if (psi->enabled != enable) {  in cgroup_pressure_write()
3931 			cgroup_file_show(&cgrp->psi_files[i], enable);  in cgroup_pressure_write()
3933 		psi->enabled = enable;  in cgroup_pressure_write()
3938 	cgroup_kn_unlock(of->kn);  in cgroup_pressure_write()
3946 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_pressure_poll()
3948 	return psi_trigger_poll(&ctx->psi.trigger, of->file, pt);  in cgroup_pressure_poll()
3953 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_pressure_release()
3955 	psi_trigger_destroy(ctx->psi.trigger);  in cgroup_pressure_release()
3976 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_freeze_show()
3978 	seq_printf(seq, "%d\n", cgrp->freezer.freeze);  in cgroup_freeze_show()
3995 		return -ERANGE;  in cgroup_freeze_write()
3997 	cgrp = cgroup_kn_lock_live(of->kn, false);  in cgroup_freeze_write()
3999 		return -ENOENT;  in cgroup_freeze_write()
4003 	cgroup_kn_unlock(of->kn);  in cgroup_freeze_write()
4016 	cgrp->kill_seq++;  in __cgroup_kill()
4019 	css_task_iter_start(&cgrp->self, CSS_TASK_ITER_PROCS | CSS_TASK_ITER_THREADED, &it);  in __cgroup_kill()
4022 		if (task->flags & PF_KTHREAD)  in __cgroup_kill()
4057 		return -ERANGE;  in cgroup_kill_write()
4059 	cgrp = cgroup_kn_lock_live(of->kn, false);  in cgroup_kill_write()
4061 		return -ENOENT;  in cgroup_kill_write()
4064 	 * Killing is a process directed operation, i.e. the whole thread-group  in cgroup_kill_write()
4066 	 * writable in non-threaded cgroups.  in cgroup_kill_write()
4069 		ret = -EOPNOTSUPP;  in cgroup_kill_write()
4073 	cgroup_kn_unlock(of->kn);  in cgroup_kill_write()
4086 		return -ENOMEM;  in cgroup_file_open()
4088 	ctx->ns = current->nsproxy->cgroup_ns;  in cgroup_file_open()
4089 	get_cgroup_ns(ctx->ns);  in cgroup_file_open()
4090 	of->priv = ctx;  in cgroup_file_open()
4092 	if (!cft->open)  in cgroup_file_open()
4095 	ret = cft->open(of);  in cgroup_file_open()
4097 		put_cgroup_ns(ctx->ns);  in cgroup_file_open()
4106 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_file_release()
4108 	if (cft->release)  in cgroup_file_release()
4109 		cft->release(of);  in cgroup_file_release()
4110 	put_cgroup_ns(ctx->ns);  in cgroup_file_release()
4117 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_file_write()
4118 	struct cgroup *cgrp = of->kn->parent->priv;  in cgroup_file_write()
4128 	 * files in an non-init namespace root from inside the namespace  in cgroup_file_write()
4129 	 * except for the files explicitly marked delegatable -  in cgroup_file_write()
4132 	if ((cgrp->root->flags & CGRP_ROOT_NS_DELEGATE) &&  in cgroup_file_write()
4133 	    !(cft->flags & CFTYPE_NS_DELEGATABLE) &&  in cgroup_file_write()
4134 	    ctx->ns != &init_cgroup_ns && ctx->ns->root_cset->dfl_cgrp == cgrp)  in cgroup_file_write()
4135 		return -EPERM;  in cgroup_file_write()
4137 	if (cft->write)  in cgroup_file_write()
4138 		return cft->write(of, buf, nbytes, off);  in cgroup_file_write()
4147 	css = cgroup_css(cgrp, cft->ss);  in cgroup_file_write()
4150 	if (cft->write_u64) {  in cgroup_file_write()
4154 			ret = cft->write_u64(css, cft, v);  in cgroup_file_write()
4155 	} else if (cft->write_s64) {  in cgroup_file_write()
4159 			ret = cft->write_s64(css, cft, v);  in cgroup_file_write()
4161 		ret = -EINVAL;  in cgroup_file_write()
4171 	if (cft->poll)  in cgroup_file_poll()
4172 		return cft->poll(of, pt);  in cgroup_file_poll()
4179 	return seq_cft(seq)->seq_start(seq, ppos);  in cgroup_seqfile_start()
4184 	return seq_cft(seq)->seq_next(seq, v, ppos);  in cgroup_seqfile_next()
4189 	if (seq_cft(seq)->seq_stop)  in cgroup_seqfile_stop()
4190 		seq_cft(seq)->seq_stop(seq, v);  in cgroup_seqfile_stop()
4198 	if (cft->seq_show)  in cgroup_seqfile_show()
4199 		return cft->seq_show(m, arg);  in cgroup_seqfile_show()
4201 	if (cft->read_u64)  in cgroup_seqfile_show()
4202 		seq_printf(m, "%llu\n", cft->read_u64(css, cft));  in cgroup_seqfile_show()
4203 	else if (cft->read_s64)  in cgroup_seqfile_show()
4204 		seq_printf(m, "%lld\n", cft->read_s64(css, cft));  in cgroup_seqfile_show()
4206 		return -EINVAL;  in cgroup_seqfile_show()
4245 	key = &cft->lockdep_key;  in cgroup_add_file()
4247 	kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),  in cgroup_add_file()
4250 				  0, cft->kf_ops, cft,  in cgroup_add_file()
4255 	if (cft->file_offset) {  in cgroup_add_file()
4256 		struct cgroup_file *cfile = (void *)css + cft->file_offset;  in cgroup_add_file()
4258 		timer_setup(&cfile->notify_timer, cgroup_file_notify_timer, 0);  in cgroup_add_file()
4261 		cfile->kn = kn;  in cgroup_add_file()
4269  * cgroup_addrm_files - add or remove files to a cgroup directory
4271  * @cgrp: the target cgroup (usually css->cgroup)
4288 	for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) {  in cgroup_addrm_files()
4289 		/* does cft->flags tell us to skip this file on @cgrp? */  in cgroup_addrm_files()
4290 		if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))  in cgroup_addrm_files()
4292 		if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp))  in cgroup_addrm_files()
4294 		if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))  in cgroup_addrm_files()
4296 		if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))  in cgroup_addrm_files()
4298 		if ((cft->flags & CFTYPE_DEBUG) && !cgroup_debug)  in cgroup_addrm_files()
4304 					__func__, cft->name, ret);  in cgroup_addrm_files()
4319 	struct cgroup *root = &ss->root->cgrp;  in cgroup_apply_cftypes()
4327 		struct cgroup *cgrp = css->cgroup;  in cgroup_apply_cftypes()
4329 		if (!(css->flags & CSS_VISIBLE))  in cgroup_apply_cftypes()
4338 		kernfs_activate(root->kn);  in cgroup_apply_cftypes()
4346 	for (cft = cfts; cft->name[0] != '\0'; cft++) {  in cgroup_exit_cftypes()
4348 		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)  in cgroup_exit_cftypes()
4349 			kfree(cft->kf_ops);  in cgroup_exit_cftypes()
4350 		cft->kf_ops = NULL;  in cgroup_exit_cftypes()
4351 		cft->ss = NULL;  in cgroup_exit_cftypes()
4354 		cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL |  in cgroup_exit_cftypes()
4364 	for (cft = cfts; cft->name[0] != '\0'; cft++) {  in cgroup_init_cftypes()
4367 		WARN_ON(cft->ss || cft->kf_ops);  in cgroup_init_cftypes()
4369 		if (cft->flags & __CFTYPE_ADDED) {  in cgroup_init_cftypes()
4370 			ret = -EBUSY;  in cgroup_init_cftypes()
4374 		if (cft->seq_start)  in cgroup_init_cftypes()
4383 		if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {  in cgroup_init_cftypes()
4386 				ret = -ENOMEM;  in cgroup_init_cftypes()
4389 			kf_ops->atomic_write_len = cft->max_write_len;  in cgroup_init_cftypes()
4392 		cft->kf_ops = kf_ops;  in cgroup_init_cftypes()
4393 		cft->ss = ss;  in cgroup_init_cftypes()
4394 		cft->flags |= __CFTYPE_ADDED;  in cgroup_init_cftypes()
4406 	list_del(&cfts->node);  in cgroup_rm_cftypes_locked()
4412  * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
4413  * @cfts: zero-length name terminated array of cftypes
4419  * Returns 0 on successful unregistration, -ENOENT if @cfts is not
4428 		return -ENOENT;  in cgroup_rm_cftypes()
4437  * cgroup_add_cftypes - add an array of cftypes to a subsystem
4439  * @cfts: zero-length name terminated array of cftypes
4446  * Returns 0 on successful registration, -errno on failure.  Note that this
4454 	if (!cgroup_ssid_enabled(ss->id))  in cgroup_add_cftypes()
4466 	list_add_tail(&cfts->node, &ss->cfts);  in cgroup_add_cftypes()
4476  * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
4478  * @cfts: zero-length name terminated array of cftypes
4487 	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)  in cgroup_add_dfl_cftypes()
4488 		cft->flags |= __CFTYPE_ONLY_ON_DFL;  in cgroup_add_dfl_cftypes()
4493  * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
4495  * @cfts: zero-length name terminated array of cftypes
4504 	for (cft = cfts; cft && cft->name[0] != '\0'; cft++)  in cgroup_add_legacy_cftypes()
4505 		cft->flags |= __CFTYPE_NOT_ON_DFL;  in cgroup_add_legacy_cftypes()
4510  * cgroup_file_notify - generate a file modified event for a cgroup_file
4513  * @cfile must have been obtained by setting cftype->file_offset.
4520 	if (cfile->kn) {  in cgroup_file_notify()
4521 		unsigned long last = cfile->notified_at;  in cgroup_file_notify()
4525 			timer_reduce(&cfile->notify_timer, next);  in cgroup_file_notify()
4527 			kernfs_notify(cfile->kn);  in cgroup_file_notify()
4528 			cfile->notified_at = jiffies;  in cgroup_file_notify()
4535  * cgroup_file_show - show or hide a hidden cgroup file
4536  * @cfile: target cgroup_file obtained by setting cftype->file_offset
4544 	kn = cfile->kn;  in cgroup_file_show()
4555  * css_next_child - find the next child of a given css
4564  * If a subsystem synchronizes ->css_online() and the start of iteration, a
4565  * css which finished ->css_online() is guaranteed to be visible in the
4567  * A css which hasn't finished ->css_online() or already finished
4568  * ->css_offline() may show up during traversal.  It's each subsystem's
4580 	 * Once a cgroup is removed, its ->sibling.next is no longer  in css_next_child()
4588 	 * have dropped rcu_read_lock() in-between iterations.  in css_next_child()
4599 		next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);  in css_next_child()
4600 	} else if (likely(!(pos->flags & CSS_RELEASED))) {  in css_next_child()
4601 		next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);  in css_next_child()
4603 		list_for_each_entry_rcu(next, &parent->children, sibling,  in css_next_child()
4605 			if (next->serial_nr > pos->serial_nr)  in css_next_child()
4613 	if (&next->sibling != &parent->children)  in css_next_child()
4619  * css_next_descendant_pre - find the next descendant for pre-order walk
4624  * to visit for pre-order traversal of @root's descendants.  @root is
4633  * If a subsystem synchronizes ->css_online() and the start of iteration, a
4634  * css which finished ->css_online() is guaranteed to be visible in the
4636  * A css which hasn't finished ->css_online() or already finished
4637  * ->css_offline() may show up during traversal.  It's each subsystem's
4659 		next = css_next_child(pos, pos->parent);  in css_next_descendant_pre()
4662 		pos = pos->parent;  in css_next_descendant_pre()
4670  * css_rightmost_descendant - return the rightmost descendant of a css
4674  * is returned.  This can be used during pre-order traversal to skip
4692 		/* ->prev isn't RCU safe, walk ->next till the end */  in css_rightmost_descendant()
4715  * css_next_descendant_post - find the next descendant for post-order walk
4720  * to visit for post-order traversal of @root's descendants.  @root is
4729  * If a subsystem synchronizes ->css_online() and the start of iteration, a
4730  * css which finished ->css_online() is guaranteed to be visible in the
4732  * A css which hasn't finished ->css_online() or already finished
4733  * ->css_offline() may show up during traversal.  It's each subsystem's
4753 	next = css_next_child(pos, pos->parent);  in css_next_descendant_post()
4758 	return pos->parent;  in css_next_descendant_post()
4762  * css_has_online_children - does a css have online children
4776 		if (child->flags & CSS_ONLINE) {  in css_has_online_children()
4794 	if (it->tcset_pos) {  in css_task_iter_next_css_set()
4795 		l = it->tcset_pos->next;  in css_task_iter_next_css_set()
4797 		if (l != it->tcset_head) {  in css_task_iter_next_css_set()
4798 			it->tcset_pos = l;  in css_task_iter_next_css_set()
4803 		it->tcset_pos = NULL;  in css_task_iter_next_css_set()
4807 	l = it->cset_pos;  in css_task_iter_next_css_set()
4808 	l = l->next;  in css_task_iter_next_css_set()
4809 	if (l == it->cset_head) {  in css_task_iter_next_css_set()
4810 		it->cset_pos = NULL;  in css_task_iter_next_css_set()
4814 	if (it->ss) {  in css_task_iter_next_css_set()
4815 		cset = container_of(l, struct css_set, e_cset_node[it->ss->id]);  in css_task_iter_next_css_set()
4818 		cset = link->cset;  in css_task_iter_next_css_set()
4821 	it->cset_pos = l;  in css_task_iter_next_css_set()
4824 	if (it->flags & CSS_TASK_ITER_THREADED) {  in css_task_iter_next_css_set()
4825 		if (it->cur_dcset)  in css_task_iter_next_css_set()
4826 			put_css_set_locked(it->cur_dcset);  in css_task_iter_next_css_set()
4827 		it->cur_dcset = cset;  in css_task_iter_next_css_set()
4830 		it->tcset_head = &cset->threaded_csets;  in css_task_iter_next_css_set()
4831 		it->tcset_pos = &cset->threaded_csets;  in css_task_iter_next_css_set()
4838  * css_task_iter_advance_css_set - advance a task iterator to the next css_set
4849 	/* Advance to the next non-empty css_set and find first non-empty tasks list*/  in css_task_iter_advance_css_set()
4851 		if (!list_empty(&cset->tasks)) {  in css_task_iter_advance_css_set()
4852 			it->cur_tasks_head = &cset->tasks;  in css_task_iter_advance_css_set()
4854 		} else if (!list_empty(&cset->mg_tasks)) {  in css_task_iter_advance_css_set()
4855 			it->cur_tasks_head = &cset->mg_tasks;  in css_task_iter_advance_css_set()
4857 		} else if (!list_empty(&cset->dying_tasks)) {  in css_task_iter_advance_css_set()
4858 			it->cur_tasks_head = &cset->dying_tasks;  in css_task_iter_advance_css_set()
4863 		it->task_pos = NULL;  in css_task_iter_advance_css_set()
4866 	it->task_pos = it->cur_tasks_head->next;  in css_task_iter_advance_css_set()
4871 	 * the lock is re-acquired.  Iteration is performed at two levels -  in css_task_iter_advance_css_set()
4883 	if (it->cur_cset) {  in css_task_iter_advance_css_set()
4884 		list_del(&it->iters_node);  in css_task_iter_advance_css_set()
4885 		put_css_set_locked(it->cur_cset);  in css_task_iter_advance_css_set()
4888 	it->cur_cset = cset;  in css_task_iter_advance_css_set()
4889 	list_add(&it->iters_node, &cset->task_iters);  in css_task_iter_advance_css_set()
4897 	if (it->task_pos == &task->cg_list) {  in css_task_iter_skip()
4898 		it->task_pos = it->task_pos->next;  in css_task_iter_skip()
4899 		it->flags |= CSS_TASK_ITER_SKIPPED;  in css_task_iter_skip()
4909 	if (it->task_pos) {  in css_task_iter_advance()
4915 		if (it->flags & CSS_TASK_ITER_SKIPPED)  in css_task_iter_advance()
4916 			it->flags &= ~CSS_TASK_ITER_SKIPPED;  in css_task_iter_advance()
4918 			it->task_pos = it->task_pos->next;  in css_task_iter_advance()
4920 		if (it->task_pos == &it->cur_cset->tasks) {  in css_task_iter_advance()
4921 			it->cur_tasks_head = &it->cur_cset->mg_tasks;  in css_task_iter_advance()
4922 			it->task_pos = it->cur_tasks_head->next;  in css_task_iter_advance()
4924 		if (it->task_pos == &it->cur_cset->mg_tasks) {  in css_task_iter_advance()
4925 			it->cur_tasks_head = &it->cur_cset->dying_tasks;  in css_task_iter_advance()
4926 			it->task_pos = it->cur_tasks_head->next;  in css_task_iter_advance()
4928 		if (it->task_pos == &it->cur_cset->dying_tasks)  in css_task_iter_advance()
4935 	if (!it->task_pos)  in css_task_iter_advance()
4938 	task = list_entry(it->task_pos, struct task_struct, cg_list);  in css_task_iter_advance()
4940 	if (it->flags & CSS_TASK_ITER_PROCS) {  in css_task_iter_advance()
4946 		if (it->cur_tasks_head == &it->cur_cset->dying_tasks &&  in css_task_iter_advance()
4947 		    !atomic_read(&task->signal->live))  in css_task_iter_advance()
4951 		if (it->cur_tasks_head == &it->cur_cset->dying_tasks)  in css_task_iter_advance()
4957  * css_task_iter_start - initiate task iteration
4976 	it->ss = css->ss;  in css_task_iter_start()
4977 	it->flags = flags;  in css_task_iter_start()
4979 	if (CGROUP_HAS_SUBSYS_CONFIG && it->ss)  in css_task_iter_start()
4980 		it->cset_pos = &css->cgroup->e_csets[css->ss->id];  in css_task_iter_start()
4982 		it->cset_pos = &css->cgroup->cset_links;  in css_task_iter_start()
4984 	it->cset_head = it->cset_pos;  in css_task_iter_start()
4992  * css_task_iter_next - return the next task for the iterator
5003 	if (it->cur_task) {  in css_task_iter_next()
5004 		put_task_struct(it->cur_task);  in css_task_iter_next()
5005 		it->cur_task = NULL;  in css_task_iter_next()
5010 	/* @it may be half-advanced by skips, finish advancing */  in css_task_iter_next()
5011 	if (it->flags & CSS_TASK_ITER_SKIPPED)  in css_task_iter_next()
5014 	if (it->task_pos) {  in css_task_iter_next()
5015 		it->cur_task = list_entry(it->task_pos, struct task_struct,  in css_task_iter_next()
5017 		get_task_struct(it->cur_task);  in css_task_iter_next()
5023 	return it->cur_task;  in css_task_iter_next()
5027  * css_task_iter_end - finish task iteration
5036 	if (it->cur_cset) {  in css_task_iter_end()
5038 		list_del(&it->iters_node);  in css_task_iter_end()
5039 		put_css_set_locked(it->cur_cset);  in css_task_iter_end()
5043 	if (it->cur_dcset)  in css_task_iter_end()
5044 		put_css_set(it->cur_dcset);  in css_task_iter_end()
5046 	if (it->cur_task)  in css_task_iter_end()
5047 		put_task_struct(it->cur_task);  in css_task_iter_end()
5052 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_procs_release()
5054 	if (ctx->procs.started)  in cgroup_procs_release()
5055 		css_task_iter_end(&ctx->procs.iter);  in cgroup_procs_release()
5060 	struct kernfs_open_file *of = s->private;  in cgroup_procs_next()
5061 	struct cgroup_file_ctx *ctx = of->priv;  in cgroup_procs_next()
5066 	return css_task_iter_next(&ctx->procs.iter);  in cgroup_procs_next()
5072 	struct kernfs_open_file *of = s->private;  in __cgroup_procs_start()
5073 	struct cgroup *cgrp = seq_css(s)->cgroup;  in __cgroup_procs_start()
5074 	struct cgroup_file_ctx *ctx = of->priv;  in __cgroup_procs_start()
5075 	struct css_task_iter *it = &ctx->procs.iter;  in __cgroup_procs_start()
5081 	if (!ctx->procs.started) {  in __cgroup_procs_start()
5083 			return ERR_PTR(-EINVAL);  in __cgroup_procs_start()
5084 		css_task_iter_start(&cgrp->self, iter_flags, it);  in __cgroup_procs_start()
5085 		ctx->procs.started = true;  in __cgroup_procs_start()
5088 		css_task_iter_start(&cgrp->self, iter_flags, it);  in __cgroup_procs_start()
5090 		return it->cur_task;  in __cgroup_procs_start()
5097 	struct cgroup *cgrp = seq_css(s)->cgroup;  in cgroup_procs_start()
5106 		return ERR_PTR(-EOPNOTSUPP);  in cgroup_procs_start()
5125 	inode = kernfs_get_inode(sb, cgrp->procs_file.kn);  in cgroup_may_write()
5127 		return -ENOMEM;  in cgroup_may_write()
5158 	    (!cgroup_is_descendant(src_cgrp, ns->root_cset->dfl_cgrp) ||  in cgroup_procs_write_permission()
5159 	     !cgroup_is_descendant(dst_cgrp, ns->root_cset->dfl_cgrp)))  in cgroup_procs_write_permission()
5160 		return -ENOENT;  in cgroup_procs_write_permission()
5180 	if (!threadgroup && (src_cgrp->dom_cgrp != dst_cgrp->dom_cgrp))  in cgroup_attach_permissions()
5181 		ret = -EOPNOTSUPP;  in cgroup_attach_permissions()
5189 	struct cgroup_file_ctx *ctx = of->priv;  in __cgroup_procs_write()
5196 	dst_cgrp = cgroup_kn_lock_live(of->kn, false);  in __cgroup_procs_write()
5198 		return -ENODEV;  in __cgroup_procs_write()
5215 	saved_cred = override_creds(of->file->f_cred);  in __cgroup_procs_write()
5217 					of->file->f_path.dentry->d_sb,  in __cgroup_procs_write()
5218 					threadgroup, ctx->ns);  in __cgroup_procs_write()
5228 	cgroup_kn_unlock(of->kn);  in __cgroup_procs_write()
5375  * css destruction is four-stage process.
5393  * and thus involve punting to css->destroy_work adding two additional
5400 	struct cgroup_subsys *ss = css->ss;  in css_free_rwork_fn()
5401 	struct cgroup *cgrp = css->cgroup;  in css_free_rwork_fn()
5403 	percpu_ref_exit(&css->refcnt);  in css_free_rwork_fn()
5407 		struct cgroup_subsys_state *parent = css->parent;  in css_free_rwork_fn()
5408 		int id = css->id;  in css_free_rwork_fn()
5410 		ss->css_free(css);  in css_free_rwork_fn()
5411 		cgroup_idr_remove(&ss->css_idr, id);  in css_free_rwork_fn()
5418 		atomic_dec(&cgrp->root->nr_cgrps);  in css_free_rwork_fn()
5421 		cancel_work_sync(&cgrp->release_agent_work);  in css_free_rwork_fn()
5432 			kernfs_put(cgrp->kn);  in css_free_rwork_fn()
5442 			cgroup_destroy_root(cgrp->root);  in css_free_rwork_fn()
5451 	struct cgroup_subsys *ss = css->ss;  in css_release_work_fn()
5452 	struct cgroup *cgrp = css->cgroup;  in css_release_work_fn()
5456 	css->flags |= CSS_RELEASED;  in css_release_work_fn()
5457 	list_del_rcu(&css->sibling);  in css_release_work_fn()
5463 		if (!list_empty(&css->rstat_css_node)) {  in css_release_work_fn()
5465 			list_del_rcu(&css->rstat_css_node);  in css_release_work_fn()
5468 		cgroup_idr_replace(&ss->css_idr, NULL, css->id);  in css_release_work_fn()
5469 		if (ss->css_released)  in css_release_work_fn()
5470 			ss->css_released(css);  in css_release_work_fn()
5472 		cgrp->nr_dying_subsys[ss->id]--;  in css_release_work_fn()
5476 		 * cgrp->nr_dying_subsys[ss->id] may not be 0 if a subsystem  in css_release_work_fn()
5480 		WARN_ON_ONCE(css->nr_descendants);  in css_release_work_fn()
5483 			parent_cgrp->nr_dying_subsys[ss->id]--;  in css_release_work_fn()
5497 			tcgrp->nr_dying_descendants--;  in css_release_work_fn()
5502 		 * cgroup from dentry without going through kernfs -  in css_release_work_fn()
5505 		 * cgrp->kn->priv backpointer.  in css_release_work_fn()
5507 		if (cgrp->kn)  in css_release_work_fn()
5508 			RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv,  in css_release_work_fn()
5514 	INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);  in css_release_work_fn()
5515 	queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);  in css_release_work_fn()
5523 	INIT_WORK(&css->destroy_work, css_release_work_fn);  in css_release()
5524 	queue_work(cgroup_destroy_wq, &css->destroy_work);  in css_release()
5535 	css->cgroup = cgrp;  in init_and_link_css()
5536 	css->ss = ss;  in init_and_link_css()
5537 	css->id = -1;  in init_and_link_css()
5538 	INIT_LIST_HEAD(&css->sibling);  in init_and_link_css()
5539 	INIT_LIST_HEAD(&css->children);  in init_and_link_css()
5540 	INIT_LIST_HEAD(&css->rstat_css_node);  in init_and_link_css()
5541 	css->serial_nr = css_serial_nr_next++;  in init_and_link_css()
5542 	atomic_set(&css->online_cnt, 0);  in init_and_link_css()
5545 		css->parent = cgroup_css(cgroup_parent(cgrp), ss);  in init_and_link_css()
5546 		css_get(css->parent);  in init_and_link_css()
5549 	if (ss->css_rstat_flush)  in init_and_link_css()
5550 		list_add_rcu(&css->rstat_css_node, &cgrp->rstat_css_list);  in init_and_link_css()
5555 /* invoke ->css_online() on a new CSS and mark it online if successful */
5558 	struct cgroup_subsys *ss = css->ss;  in online_css()
5563 	if (ss->css_online)  in online_css()
5564 		ret = ss->css_online(css);  in online_css()
5566 		css->flags |= CSS_ONLINE;  in online_css()
5567 		rcu_assign_pointer(css->cgroup->subsys[ss->id], css);  in online_css()
5569 		atomic_inc(&css->online_cnt);  in online_css()
5570 		if (css->parent) {  in online_css()
5571 			atomic_inc(&css->parent->online_cnt);  in online_css()
5572 			while ((css = css->parent))  in online_css()
5573 				css->nr_descendants++;  in online_css()
5579 /* if the CSS is online, invoke ->css_offline() on it and mark it offline */
5582 	struct cgroup_subsys *ss = css->ss;  in offline_css()
5586 	if (!(css->flags & CSS_ONLINE))  in offline_css()
5589 	if (ss->css_offline)  in offline_css()
5590 		ss->css_offline(css);  in offline_css()
5592 	css->flags &= ~CSS_ONLINE;  in offline_css()
5593 	RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);  in offline_css()
5595 	wake_up_all(&css->cgroup->offline_waitq);  in offline_css()
5597 	css->cgroup->nr_dying_subsys[ss->id]++;  in offline_css()
5602 	while ((css = css->parent)) {  in offline_css()
5603 		css->nr_descendants--;  in offline_css()
5604 		css->cgroup->nr_dying_subsys[ss->id]++;  in offline_css()
5609  * css_create - create a cgroup_subsys_state
5613  * Create a new css associated with @cgrp - @ss pair.  On success, the new
5615  * interface files.  Returns 0 on success, -errno on failure.
5627 	css = ss->css_alloc(parent_css);  in css_create()
5629 		css = ERR_PTR(-ENOMEM);  in css_create()
5635 	err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL);  in css_create()
5639 	err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_KERNEL);  in css_create()
5642 	css->id = err;  in css_create()
5645 	list_add_tail_rcu(&css->sibling, &parent_css->children);  in css_create()
5646 	cgroup_idr_replace(&ss->css_idr, css, css->id);  in css_create()
5655 	list_del_rcu(&css->sibling);  in css_create()
5657 	list_del_rcu(&css->rstat_css_node);  in css_create()
5658 	INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);  in css_create()
5659 	queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);  in css_create()
5670 	struct cgroup_root *root = parent->root;  in cgroup_create()
5673 	int level = parent->level + 1;  in cgroup_create()
5679 		return ERR_PTR(-ENOMEM);  in cgroup_create()
5681 	ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL);  in cgroup_create()
5690 	kn = kernfs_create_dir_ns(parent->kn, name, mode,  in cgroup_create()
5697 	cgrp->kn = kn;  in cgroup_create()
5701 	cgrp->self.parent = &parent->self;  in cgroup_create()
5702 	cgrp->root = root;  in cgroup_create()
5703 	cgrp->level = level;  in cgroup_create()
5709 	if (cgrp->root == &cgrp_dfl_root) {  in cgroup_create()
5719 	cgrp->freezer.e_freeze = parent->freezer.e_freeze;  in cgroup_create()
5720 	if (cgrp->freezer.e_freeze) {  in cgroup_create()
5727 		set_bit(CGRP_FREEZE, &cgrp->flags);  in cgroup_create()
5728 		set_bit(CGRP_FROZEN, &cgrp->flags);  in cgroup_create()
5733 		cgrp->ancestors[tcgrp->level] = tcgrp;  in cgroup_create()
5736 			tcgrp->nr_descendants++;  in cgroup_create()
5743 			if (cgrp->freezer.e_freeze)  in cgroup_create()
5744 				tcgrp->freezer.nr_frozen_descendants++;  in cgroup_create()
5750 		set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);  in cgroup_create()
5752 	if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))  in cgroup_create()
5753 		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);  in cgroup_create()
5755 	cgrp->self.serial_nr = css_serial_nr_next++;  in cgroup_create()
5758 	list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);  in cgroup_create()
5759 	atomic_inc(&root->nr_cgrps);  in cgroup_create()
5767 		cgrp->subtree_control = cgroup_control(cgrp);  in cgroup_create()
5776 	kernfs_remove(cgrp->kn);  in cgroup_create()
5780 	percpu_ref_exit(&cgrp->self.refcnt);  in cgroup_create()
5795 		if (cgroup->nr_descendants >= cgroup->max_descendants)  in cgroup_check_hierarchy_limits()
5798 		if (level >= cgroup->max_depth)  in cgroup_check_hierarchy_limits()
5816 		return -EINVAL;  in cgroup_mkdir()
5820 		return -ENODEV;  in cgroup_mkdir()
5823 		ret = -EAGAIN;  in cgroup_mkdir()
5835 	 * that @cgrp->kn is always accessible.  in cgroup_mkdir()
5837 	kernfs_get(cgrp->kn);  in cgroup_mkdir()
5839 	ret = css_populate_dir(&cgrp->self);  in cgroup_mkdir()
5850 	kernfs_activate(cgrp->kn);  in cgroup_mkdir()
5878 		css = css->parent;  in css_killed_work_fn()
5879 	} while (css && atomic_dec_and_test(&css->online_cnt));  in css_killed_work_fn()
5890 	if (atomic_dec_and_test(&css->online_cnt)) {  in css_killed_ref_fn()
5891 		INIT_WORK(&css->destroy_work, css_killed_work_fn);  in css_killed_ref_fn()
5892 		queue_work(cgroup_destroy_wq, &css->destroy_work);  in css_killed_ref_fn()
5897  * kill_css - destroy a css
5901  * files and putting its base reference.  ->css_offline() will be invoked
5909 	if (css->flags & CSS_DYING)  in kill_css()
5912 	css->flags |= CSS_DYING;  in kill_css()
5922 	 * until after ->css_offline().  in kill_css()
5927 	 * cgroup core guarantees that, by the time ->css_offline() is  in kill_css()
5936 	percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);  in kill_css()
5940  * cgroup_destroy_locked - the first stage of cgroup destruction
5946  * ->css_offline() is invoked.  To satisfy all the requirements,
5954  * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
5956  *     cgroup is RCU-freed.
5978 		return -EBUSY;  in cgroup_destroy_locked()
5982 	 * ->self.children as dead children linger on it while being  in cgroup_destroy_locked()
5985 	if (css_has_online_children(&cgrp->self))  in cgroup_destroy_locked()
5986 		return -EBUSY;  in cgroup_destroy_locked()
5994 	cgrp->self.flags &= ~CSS_ONLINE;  in cgroup_destroy_locked()
5997 	list_for_each_entry(link, &cgrp->cset_links, cset_link)  in cgroup_destroy_locked()
5998 		link->cset->dead = true;  in cgroup_destroy_locked()
6006 	css_clear_dir(&cgrp->self);  in cgroup_destroy_locked()
6007 	kernfs_remove(cgrp->kn);  in cgroup_destroy_locked()
6010 		parent->nr_threaded_children--;  in cgroup_destroy_locked()
6014 		tcgrp->nr_descendants--;  in cgroup_destroy_locked()
6015 		tcgrp->nr_dying_descendants++;  in cgroup_destroy_locked()
6020 		if (test_bit(CGRP_FROZEN, &cgrp->flags))  in cgroup_destroy_locked()
6021 			tcgrp->freezer.nr_frozen_descendants--;  in cgroup_destroy_locked()
6027 	if (cgrp->root == &cgrp_dfl_root)  in cgroup_destroy_locked()
6031 	percpu_ref_kill(&cgrp->self.refcnt);  in cgroup_destroy_locked()
6064 	pr_debug("Initializing cgroup subsys %s\n", ss->name);  in cgroup_init_subsys()
6068 	idr_init(&ss->css_idr);  in cgroup_init_subsys()
6069 	INIT_LIST_HEAD(&ss->cfts);  in cgroup_init_subsys()
6072 	ss->root = &cgrp_dfl_root;  in cgroup_init_subsys()
6073 	css = ss->css_alloc(NULL);  in cgroup_init_subsys()
6082 	css->flags |= CSS_NO_REF;  in cgroup_init_subsys()
6086 		css->id = 1;  in cgroup_init_subsys()
6088 		css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);  in cgroup_init_subsys()
6089 		BUG_ON(css->id < 0);  in cgroup_init_subsys()
6093 	 * pointer to this state - since the subsystem is  in cgroup_init_subsys()
6096 	init_css_set.subsys[ss->id] = css;  in cgroup_init_subsys()
6098 	have_fork_callback |= (bool)ss->fork << ss->id;  in cgroup_init_subsys()
6099 	have_exit_callback |= (bool)ss->exit << ss->id;  in cgroup_init_subsys()
6100 	have_release_callback |= (bool)ss->release << ss->id;  in cgroup_init_subsys()
6101 	have_canfork_callback |= (bool)ss->can_fork << ss->id;  in cgroup_init_subsys()
6114  * cgroup_init_early - cgroup initialization at system boot
6132 		WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,  in cgroup_init_early()
6134 		     i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,  in cgroup_init_early()
6135 		     ss->id, ss->name);  in cgroup_init_early()
6139 		ss->id = i;  in cgroup_init_early()
6140 		ss->name = cgroup_subsys_name[i];  in cgroup_init_early()
6141 		if (!ss->legacy_name)  in cgroup_init_early()
6142 			ss->legacy_name = cgroup_subsys_name[i];  in cgroup_init_early()
6144 		if (ss->early_init)  in cgroup_init_early()
6151  * cgroup_init - cgroup initialization
6184 		if (ss->early_init) {  in cgroup_init()
6186 				init_css_set.subsys[ss->id];  in cgroup_init()
6188 			css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,  in cgroup_init()
6190 			BUG_ON(css->id < 0);  in cgroup_init()
6208 				ss->legacy_name);  in cgroup_init()
6210 		cgrp_dfl_root.subsys_mask |= 1 << ss->id;  in cgroup_init()
6213 		WARN_ON(ss->implicit_on_dfl && !ss->threaded);  in cgroup_init()
6215 		if (ss->implicit_on_dfl)  in cgroup_init()
6216 			cgrp_dfl_implicit_ss_mask |= 1 << ss->id;  in cgroup_init()
6217 		else if (!ss->dfl_cftypes)  in cgroup_init()
6218 			cgrp_dfl_inhibit_ss_mask |= 1 << ss->id;  in cgroup_init()
6220 		if (ss->threaded)  in cgroup_init()
6221 			cgrp_dfl_threaded_ss_mask |= 1 << ss->id;  in cgroup_init()
6223 		if (ss->dfl_cftypes == ss->legacy_cftypes) {  in cgroup_init()
6224 			WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes));  in cgroup_init()
6226 			WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes));  in cgroup_init()
6227 			WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes));  in cgroup_init()
6230 		if (ss->bind)  in cgroup_init()
6231 			ss->bind(init_css_set.subsys[ssid]);  in cgroup_init()
6238 	/* init_css_set.subsys[] has been updated, re-hash */  in cgroup_init()
6294 		return ERR_PTR(-ENOENT);  in cgroup_get_from_id()
6298 		return ERR_PTR(-ENOENT);  in cgroup_get_from_id()
6303 	cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv);  in cgroup_get_from_id()
6311 		return ERR_PTR(-ENOENT);  in cgroup_get_from_id()
6316 		return ERR_PTR(-ENOENT);  in cgroup_get_from_id()
6325  *  - Print task's cgroup paths into seq_file, one line for each hierarchy
6326  *  - Used for /proc/<pid>/cgroup.
6335 	retval = -ENOMEM;  in proc_cgroup_show()
6356 		seq_printf(m, "%d:", root->hierarchy_id);  in proc_cgroup_show()
6359 				if (root->subsys_mask & (1 << ssid))  in proc_cgroup_show()
6361 						   ss->legacy_name);  in proc_cgroup_show()
6362 		if (strlen(root->name))  in proc_cgroup_show()
6364 				   root->name);  in proc_cgroup_show()
6375 		if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) {  in proc_cgroup_show()
6377 						current->nsproxy->cgroup_ns);  in proc_cgroup_show()
6378 			if (retval == -E2BIG)  in proc_cgroup_show()
6379 				retval = -ENAMETOOLONG;  in proc_cgroup_show()
6404  * cgroup_fork - initialize cgroup related fields during copy_process()
6412 	RCU_INIT_POINTER(child->cgroups, &init_css_set);  in cgroup_fork()
6413 	INIT_LIST_HEAD(&child->cg_list);  in cgroup_fork()
6417  * cgroup_v1v2_get_from_file - get a cgroup pointer from a file pointer
6428 	css = css_tryget_online_from_dir(f->f_path.dentry, NULL);  in cgroup_v1v2_get_from_file()
6432 	return css->cgroup;  in cgroup_v1v2_get_from_file()
6436  * cgroup_get_from_file - same as cgroup_v1v2_get_from_file, but only supports
6449 		return ERR_PTR(-EBADF);  in cgroup_get_from_file()
6456  * cgroup_css_set_fork - find or create a css_set for a child process
6479 	if (kargs->flags & CLONE_INTO_CGROUP)  in cgroup_css_set_fork()
6487 	if (kargs->cgrp)  in cgroup_css_set_fork()
6488 		kargs->kill_seq = kargs->cgrp->kill_seq;  in cgroup_css_set_fork()
6490 		kargs->kill_seq = cset->dfl_cgrp->kill_seq;  in cgroup_css_set_fork()
6493 	if (!(kargs->flags & CLONE_INTO_CGROUP)) {  in cgroup_css_set_fork()
6494 		kargs->cset = cset;  in cgroup_css_set_fork()
6498 	CLASS(fd_raw, f)(kargs->cgroup);  in cgroup_css_set_fork()
6500 		ret = -EBADF;  in cgroup_css_set_fork()
6503 	sb = fd_file(f)->f_path.dentry->d_sb;  in cgroup_css_set_fork()
6513 		ret = -ENODEV;  in cgroup_css_set_fork()
6534 	 *   write(fd, <child-pid>, ...);  in cgroup_css_set_fork()
6540 	ret = cgroup_attach_permissions(cset->dfl_cgrp, dst_cgrp, sb,  in cgroup_css_set_fork()
6541 					!(kargs->flags & CLONE_THREAD),  in cgroup_css_set_fork()
6542 					current->nsproxy->cgroup_ns);  in cgroup_css_set_fork()
6546 	kargs->cset = find_css_set(cset, dst_cgrp);  in cgroup_css_set_fork()
6547 	if (!kargs->cset) {  in cgroup_css_set_fork()
6548 		ret = -ENOMEM;  in cgroup_css_set_fork()
6553 	kargs->cgrp = dst_cgrp;  in cgroup_css_set_fork()
6562 	if (kargs->cset)  in cgroup_css_set_fork()
6563 		put_css_set(kargs->cset);  in cgroup_css_set_fork()
6568  * cgroup_css_set_put_fork - drop references we took during fork
6577 	struct cgroup *cgrp = kargs->cgrp;  in cgroup_css_set_put_fork()
6578 	struct css_set *cset = kargs->cset;  in cgroup_css_set_put_fork()
6584 		kargs->cset = NULL;  in cgroup_css_set_put_fork()
6587 	if (kargs->flags & CLONE_INTO_CGROUP) {  in cgroup_css_set_put_fork()
6591 			kargs->cgrp = NULL;  in cgroup_css_set_put_fork()
6597  * cgroup_can_fork - called on a new task before the process is exposed
6617 		ret = ss->can_fork(child, kargs->cset);  in cgroup_can_fork()
6628 		if (ss->cancel_fork)  in cgroup_can_fork()
6629 			ss->cancel_fork(child, kargs->cset);  in cgroup_can_fork()
6638  * cgroup_cancel_fork - called if a fork failed after cgroup_can_fork()
6653 		if (ss->cancel_fork)  in cgroup_cancel_fork()
6654 			ss->cancel_fork(child, kargs->cset);  in cgroup_cancel_fork()
6660  * cgroup_post_fork - finalize cgroup setup for the child process
6678 	cset = kargs->cset;  in cgroup_post_fork()
6679 	kargs->cset = NULL;  in cgroup_post_fork()
6684 	if (likely(child->pid)) {  in cgroup_post_fork()
6685 		if (kargs->cgrp) {  in cgroup_post_fork()
6686 			cgrp_flags = kargs->cgrp->flags;  in cgroup_post_fork()
6687 			cgrp_kill_seq = kargs->cgrp->kill_seq;  in cgroup_post_fork()
6689 			cgrp_flags = cset->dfl_cgrp->flags;  in cgroup_post_fork()
6690 			cgrp_kill_seq = cset->dfl_cgrp->kill_seq;  in cgroup_post_fork()
6693 		WARN_ON_ONCE(!list_empty(&child->cg_list));  in cgroup_post_fork()
6694 		cset->nr_tasks++;  in cgroup_post_fork()
6701 	if (!(child->flags & PF_KTHREAD)) {  in cgroup_post_fork()
6708 			spin_lock(&child->sighand->siglock);  in cgroup_post_fork()
6709 			WARN_ON_ONCE(child->frozen);  in cgroup_post_fork()
6710 			child->jobctl |= JOBCTL_TRAP_FREEZE;  in cgroup_post_fork()
6711 			spin_unlock(&child->sighand->siglock);  in cgroup_post_fork()
6726 		kill = kargs->kill_seq != cgrp_kill_seq;  in cgroup_post_fork()
6732 	 * Call ss->fork().  This must happen after @child is linked on  in cgroup_post_fork()
6733 	 * css_set; otherwise, @child might change state between ->fork()  in cgroup_post_fork()
6737 		ss->fork(child);  in cgroup_post_fork()
6741 	if (kargs->flags & CLONE_NEWCGROUP) {  in cgroup_post_fork()
6742 		struct css_set *rcset = child->nsproxy->cgroup_ns->root_cset;  in cgroup_post_fork()
6745 		child->nsproxy->cgroup_ns->root_cset = cset;  in cgroup_post_fork()
6757  * cgroup_exit - detach cgroup from exiting task
6771 	WARN_ON_ONCE(list_empty(&tsk->cg_list));  in cgroup_exit()
6774 	cset->nr_tasks--;  in cgroup_exit()
6775 	/* matches the signal->live check in css_task_iter_advance() */  in cgroup_exit()
6776 	if (thread_group_leader(tsk) && atomic_read(&tsk->signal->live))  in cgroup_exit()
6777 		list_add_tail(&tsk->cg_list, &cset->dying_tasks);  in cgroup_exit()
6783 	if (unlikely(!(tsk->flags & PF_KTHREAD) &&  in cgroup_exit()
6784 		     test_bit(CGRP_FREEZE, &task_dfl_cgroup(tsk)->flags)))  in cgroup_exit()
6791 		ss->exit(tsk);  in cgroup_exit()
6801 		ss->release(task);  in cgroup_release()
6804 	if (!list_empty(&task->cg_list)) {  in cgroup_release()
6807 		list_del_init(&task->cg_list);  in cgroup_release()
6829 			if (strcmp(token, ss->name) &&  in cgroup_disable()
6830 			    strcmp(token, ss->legacy_name))  in cgroup_disable()
6835 				ss->name);  in cgroup_disable()
6868  * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
6880 	struct file_system_type *s_type = dentry->d_sb->s_type;  in css_tryget_online_from_dir()
6887 		return ERR_PTR(-EBADF);  in css_tryget_online_from_dir()
6893 	 * have been or be removed at any point.  @kn->priv is RCU  in css_tryget_online_from_dir()
6896 	cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv);  in css_tryget_online_from_dir()
6901 		css = ERR_PTR(-ENOENT);  in css_tryget_online_from_dir()
6908  * css_from_id - lookup css by id
6918 	return idr_find(&ss->css_idr, id);  in css_from_id()
6922  * cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path
6927  * success, ERR_PTR(-ENOENT) if @path doesn't exist or if the cgroup has already
6928  * been released and ERR_PTR(-ENOTDIR) if @path points to a non-directory.
6933 	struct cgroup *cgrp = ERR_PTR(-ENOENT);  in cgroup_get_from_path()
6937 	kn = kernfs_walk_and_get(root_cgrp->kn, path);  in cgroup_get_from_path()
6942 		cgrp = ERR_PTR(-ENOTDIR);  in cgroup_get_from_path()
6948 	cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv);  in cgroup_get_from_path()
6950 		cgrp = ERR_PTR(-ENOENT);  in cgroup_get_from_path()
6962  * cgroup_v1v2_get_from_fd - get a cgroup pointer from a fd
6974 		return ERR_PTR(-EBADF);  in cgroup_v1v2_get_from_fd()
6980  * cgroup_get_from_fd - same as cgroup_v1v2_get_from_fd, but only supports
6993 		return ERR_PTR(-EBADF);  in cgroup_get_from_fd()
7002 	while (power--)  in power_of_ten()
7008  * cgroup_parse_float - parse a floating number
7016  * Returns 0 on success, -errno otherwise.
7027 		return -EINVAL;  in cgroup_parse_float()
7029 		return -EINVAL;  in cgroup_parse_float()
7031 	flen = fend > fstart ? fend - fstart : 0;  in cgroup_parse_float()
7033 		frac *= power_of_ten(dec_shift - flen);  in cgroup_parse_float()
7035 		frac = DIV_ROUND_CLOSEST_ULL(frac, power_of_ten(flen - dec_shift));  in cgroup_parse_float()
7042  * sock->sk_cgrp_data handling.  For more info, see sock_cgroup_data
7043  * definition in cgroup-defs.h.
7063 		if (likely(cgroup_tryget(cset->dfl_cgrp))) {  in cgroup_sk_alloc()
7064 			cgroup = cset->dfl_cgrp;  in cgroup_sk_alloc()
7070 	skcd->cgroup = cgroup;  in cgroup_sk_alloc()
7105 	for (cft = files; cft && cft->name[0] != '\0'; cft++) {  in show_delegatable_files()
7106 		if (!(cft->flags & CFTYPE_NS_DELEGATABLE))  in show_delegatable_files()
7110 			ret += snprintf(buf + ret, size - ret, "%s.", prefix);  in show_delegatable_files()
7112 		ret += snprintf(buf + ret, size - ret, "%s\n", cft->name);  in show_delegatable_files()
7129 				     PAGE_SIZE - ret, NULL);  in delegate_show()
7132 					      PAGE_SIZE - ret, NULL);  in delegate_show()
7135 		ret += show_delegatable_files(ss->dfl_cftypes, buf + ret,  in delegate_show()
7136 					      PAGE_SIZE - ret,  in delegate_show()