Lines Matching refs:cgrp
197 .cgrp.self.rstat_cpu = &root_rstat_cpu,
198 .cgrp.rstat_base_cpu = &root_rstat_base_cpu,
279 static int cgroup_apply_control(struct cgroup *cgrp);
280 static void cgroup_finalize_control(struct cgroup *cgrp, int ret);
283 static int cgroup_destroy_locked(struct cgroup *cgrp);
284 static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
289 struct cgroup *cgrp, struct cftype cfts[],
358 bool cgroup_on_dfl(const struct cgroup *cgrp) in cgroup_on_dfl() argument
360 return cgrp->root == &cgrp_dfl_root; in cgroup_on_dfl()
394 static bool cgroup_has_tasks(struct cgroup *cgrp) in cgroup_has_tasks() argument
396 return cgrp->nr_populated_csets; in cgroup_has_tasks()
399 static bool cgroup_is_threaded(struct cgroup *cgrp) in cgroup_is_threaded() argument
401 return cgrp->dom_cgrp != cgrp; in cgroup_is_threaded()
405 static bool cgroup_is_mixable(struct cgroup *cgrp) in cgroup_is_mixable() argument
412 return !cgroup_parent(cgrp); in cgroup_is_mixable()
416 static bool cgroup_can_be_thread_root(struct cgroup *cgrp) in cgroup_can_be_thread_root() argument
419 if (cgroup_is_mixable(cgrp)) in cgroup_can_be_thread_root()
423 if (cgroup_is_threaded(cgrp)) in cgroup_can_be_thread_root()
427 if (cgrp->nr_populated_domain_children) in cgroup_can_be_thread_root()
431 if (cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask) in cgroup_can_be_thread_root()
438 static bool cgroup_is_thread_root(struct cgroup *cgrp) in cgroup_is_thread_root() argument
441 if (cgroup_is_threaded(cgrp)) in cgroup_is_thread_root()
445 if (cgrp->nr_threaded_children) in cgroup_is_thread_root()
452 if (cgroup_has_tasks(cgrp) && in cgroup_is_thread_root()
453 (cgrp->subtree_control & cgrp_dfl_threaded_ss_mask)) in cgroup_is_thread_root()
460 static bool cgroup_is_valid_domain(struct cgroup *cgrp) in cgroup_is_valid_domain() argument
463 if (cgroup_is_threaded(cgrp)) in cgroup_is_valid_domain()
467 while ((cgrp = cgroup_parent(cgrp))) { in cgroup_is_valid_domain()
468 if (!cgroup_is_mixable(cgrp) && cgroup_is_thread_root(cgrp)) in cgroup_is_valid_domain()
470 if (cgroup_is_threaded(cgrp)) in cgroup_is_valid_domain()
478 static u16 cgroup_control(struct cgroup *cgrp) in cgroup_control() argument
480 struct cgroup *parent = cgroup_parent(cgrp); in cgroup_control()
481 u16 root_ss_mask = cgrp->root->subsys_mask; in cgroup_control()
487 if (cgroup_is_threaded(cgrp)) in cgroup_control()
492 if (cgroup_on_dfl(cgrp)) in cgroup_control()
499 static u16 cgroup_ss_mask(struct cgroup *cgrp) in cgroup_ss_mask() argument
501 struct cgroup *parent = cgroup_parent(cgrp); in cgroup_ss_mask()
507 if (cgroup_is_threaded(cgrp)) in cgroup_ss_mask()
512 return cgrp->root->subsys_mask; in cgroup_ss_mask()
526 static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp, in cgroup_css() argument
530 return rcu_dereference_check(cgrp->subsys[ss->id], in cgroup_css()
533 return &cgrp->self; in cgroup_css()
546 static struct cgroup_subsys_state *cgroup_e_css_by_mask(struct cgroup *cgrp, in cgroup_e_css_by_mask() argument
552 return &cgrp->self; in cgroup_e_css_by_mask()
558 while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) { in cgroup_e_css_by_mask()
559 cgrp = cgroup_parent(cgrp); in cgroup_e_css_by_mask()
560 if (!cgrp) in cgroup_e_css_by_mask()
564 return cgroup_css(cgrp, ss); in cgroup_e_css_by_mask()
580 struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, in cgroup_e_css() argument
589 css = cgroup_css(cgrp, ss); in cgroup_e_css()
593 cgrp = cgroup_parent(cgrp); in cgroup_e_css()
594 } while (cgrp); in cgroup_e_css()
610 struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp, in cgroup_get_e_css() argument
621 css = cgroup_css(cgrp, ss); in cgroup_get_e_css()
625 cgrp = cgroup_parent(cgrp); in cgroup_get_e_css()
626 } while (cgrp); in cgroup_get_e_css()
636 static void cgroup_get_live(struct cgroup *cgrp) in cgroup_get_live() argument
638 WARN_ON_ONCE(cgroup_is_dead(cgrp)); in cgroup_get_live()
639 cgroup_get(cgrp); in cgroup_get_live()
647 int __cgroup_task_count(const struct cgroup *cgrp) in __cgroup_task_count() argument
654 list_for_each_entry(link, &cgrp->cset_links, cset_link) in __cgroup_task_count()
664 int cgroup_task_count(const struct cgroup *cgrp) in cgroup_task_count() argument
669 count = __cgroup_task_count(cgrp); in cgroup_task_count()
690 struct cgroup *cgrp = kn_priv(of->kn); in of_css() local
702 return rcu_dereference_raw(cgrp->subsys[cft->ss->id]); in of_css()
704 return &cgrp->self; in of_css()
716 #define for_each_css(css, ssid, cgrp) \ argument
719 (cgrp)->subsys[(ssid)], \
748 #define cgroup_for_each_live_child(child, cgrp) \ argument
749 list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
756 #define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) \ argument
757 css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL)) \
765 #define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) \ argument
766 css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL)) \
799 .dfl_cgrp = &cgrp_dfl_root.cgrp,
842 static void cgroup_update_populated(struct cgroup *cgrp, bool populated) in cgroup_update_populated() argument
850 bool was_populated = cgroup_is_populated(cgrp); in cgroup_update_populated()
853 cgrp->nr_populated_csets += adj; in cgroup_update_populated()
856 cgrp->nr_populated_threaded_children += adj; in cgroup_update_populated()
858 cgrp->nr_populated_domain_children += adj; in cgroup_update_populated()
861 if (was_populated == cgroup_is_populated(cgrp)) in cgroup_update_populated()
864 cgroup1_check_for_release(cgrp); in cgroup_update_populated()
865 TRACE_CGROUP_PATH(notify_populated, cgrp, in cgroup_update_populated()
866 cgroup_is_populated(cgrp)); in cgroup_update_populated()
867 cgroup_file_notify(&cgrp->events_file); in cgroup_update_populated()
869 child = cgrp; in cgroup_update_populated()
870 cgrp = cgroup_parent(cgrp); in cgroup_update_populated()
871 } while (cgrp); in cgroup_update_populated()
889 cgroup_update_populated(link->cgrp, populated); in css_set_update_populated()
1002 if (cgroup_parent(link->cgrp)) in put_css_set_locked()
1003 cgroup_put(link->cgrp); in put_css_set_locked()
1075 cgrp1 = link1->cgrp; in compare_css_sets()
1076 cgrp2 = link2->cgrp; in compare_css_sets()
1105 struct cgroup *cgrp, in find_existing_css_set() argument
1108 struct cgroup_root *root = cgrp->root; in find_existing_css_set()
1125 template[i] = cgroup_e_css_by_mask(cgrp, ss); in find_existing_css_set()
1137 if (!compare_css_sets(cset, old_cset, cgrp, template)) in find_existing_css_set()
1191 struct cgroup *cgrp) in link_css_set() argument
1197 if (cgroup_on_dfl(cgrp)) in link_css_set()
1198 cset->dfl_cgrp = cgrp; in link_css_set()
1202 link->cgrp = cgrp; in link_css_set()
1208 list_move_tail(&link->cset_link, &cgrp->cset_links); in link_css_set()
1211 if (cgroup_parent(cgrp)) in link_css_set()
1212 cgroup_get_live(cgrp); in link_css_set()
1224 struct cgroup *cgrp) in find_css_set() argument
1239 cset = find_existing_css_set(old_cset, cgrp, template); in find_css_set()
1277 struct cgroup *c = link->cgrp; in find_css_set()
1279 if (c->root == cgrp->root) in find_css_set()
1280 c = cgrp; in find_css_set()
1392 struct cgroup *cgrp = &root->cgrp; in cgroup_destroy_root() local
1398 cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); in cgroup_destroy_root()
1401 BUG_ON(!list_empty(&cgrp->self.children)); in cgroup_destroy_root()
1404 CGROUP_LIFETIME_OFFLINE, cgrp); in cgroup_destroy_root()
1416 list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) { in cgroup_destroy_root()
1448 res_cgroup = &root->cgrp; in __cset_cgroup_from_root()
1456 struct cgroup *c = link->cgrp; in __cset_cgroup_from_root()
1529 return &cgrp_dfl_root.cgrp; in current_cgns_cgroup_dfl()
1586 static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft, in cgroup_file_name() argument
1592 !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) { in cgroup_file_name()
1596 dbg, cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name, in cgroup_file_name()
1683 struct cgroup *cgrp; in cgroup_kn_unlock() local
1686 cgrp = kn->priv; in cgroup_kn_unlock()
1688 cgrp = kn_priv(kn); in cgroup_kn_unlock()
1693 cgroup_put(cgrp); in cgroup_kn_unlock()
1715 struct cgroup *cgrp; in cgroup_kn_lock_live() local
1718 cgrp = kn->priv; in cgroup_kn_lock_live()
1720 cgrp = kn_priv(kn); in cgroup_kn_lock_live()
1728 if (!cgroup_tryget(cgrp)) in cgroup_kn_lock_live()
1733 cgroup_lock_and_drain_offline(cgrp); in cgroup_kn_lock_live()
1737 if (!cgroup_is_dead(cgrp)) in cgroup_kn_lock_live()
1738 return cgrp; in cgroup_kn_lock_live()
1744 static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft) in cgroup_rm_file() argument
1751 struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss); in cgroup_rm_file()
1761 kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name)); in cgroup_rm_file()
1770 struct cgroup *cgrp = css->cgroup; in css_clear_dir() local
1779 if (cgroup_on_dfl(cgrp)) { in css_clear_dir()
1780 cgroup_addrm_files(css, cgrp, in css_clear_dir()
1783 cgroup_addrm_files(css, cgrp, in css_clear_dir()
1786 cgroup_addrm_files(css, cgrp, in css_clear_dir()
1791 cgroup_addrm_files(css, cgrp, cfts, false); in css_clear_dir()
1803 struct cgroup *cgrp = css->cgroup; in css_populate_dir() local
1811 if (cgroup_on_dfl(cgrp)) { in css_populate_dir()
1812 ret = cgroup_addrm_files(css, cgrp, in css_populate_dir()
1818 ret = cgroup_addrm_files(css, cgrp, in css_populate_dir()
1821 cgroup_addrm_files(css, cgrp, in css_populate_dir()
1827 ret = cgroup_addrm_files(css, cgrp, in css_populate_dir()
1834 ret = cgroup_addrm_files(css, cgrp, cfts, true); in css_populate_dir()
1849 cgroup_addrm_files(css, cgrp, cfts, false); in css_populate_dir()
1856 struct cgroup *dcgrp = &dst_root->cgrp; in rebind_subsystems()
1869 if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) && in rebind_subsystems()
1887 struct cgroup *scgrp = &cgrp_dfl_root.cgrp; in rebind_subsystems()
1900 struct cgroup *scgrp = &src_root->cgrp; in rebind_subsystems()
2106 static void init_cgroup_housekeeping(struct cgroup *cgrp) in init_cgroup_housekeeping() argument
2111 INIT_LIST_HEAD(&cgrp->self.sibling); in init_cgroup_housekeeping()
2112 INIT_LIST_HEAD(&cgrp->self.children); in init_cgroup_housekeeping()
2113 INIT_LIST_HEAD(&cgrp->cset_links); in init_cgroup_housekeeping()
2114 INIT_LIST_HEAD(&cgrp->pidlists); in init_cgroup_housekeeping()
2115 mutex_init(&cgrp->pidlist_mutex); in init_cgroup_housekeeping()
2116 cgrp->self.cgroup = cgrp; in init_cgroup_housekeeping()
2117 cgrp->self.flags |= CSS_ONLINE; in init_cgroup_housekeeping()
2118 cgrp->dom_cgrp = cgrp; in init_cgroup_housekeeping()
2119 cgrp->max_descendants = INT_MAX; in init_cgroup_housekeeping()
2120 cgrp->max_depth = INT_MAX; in init_cgroup_housekeeping()
2121 prev_cputime_init(&cgrp->prev_cputime); in init_cgroup_housekeeping()
2124 INIT_LIST_HEAD(&cgrp->e_csets[ssid]); in init_cgroup_housekeeping()
2127 for (int i = 0; i < ARRAY_SIZE(cgrp->bpf.revisions); i++) in init_cgroup_housekeeping()
2128 cgrp->bpf.revisions[i] = 1; in init_cgroup_housekeeping()
2131 init_waitqueue_head(&cgrp->offline_waitq); in init_cgroup_housekeeping()
2132 INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent); in init_cgroup_housekeeping()
2138 struct cgroup *cgrp = &root->cgrp; in init_cgroup_root() local
2142 cgrp->root = root; in init_cgroup_root()
2143 init_cgroup_housekeeping(cgrp); in init_cgroup_root()
2152 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); in init_cgroup_root()
2158 struct cgroup *root_cgrp = &root->cgrp; in cgroup_setup_root()
2279 struct cgroup *cgrp; in cgroup_do_get_tree() local
2284 cgrp = cset_cgroup_from_root(ctx->ns->root_cset, ctx->root); in cgroup_do_get_tree()
2289 nsdentry = kernfs_node_dentry(cgrp->kn, sb); in cgroup_do_get_tree()
2300 cgroup_put(&ctx->root->cgrp); in cgroup_do_get_tree()
2325 cgroup_get_live(&cgrp_dfl_root.cgrp); in cgroup_get_tree()
2388 if (list_empty(&root->cgrp.self.children) && root != &cgrp_dfl_root && in cgroup_kill_sb()
2389 !percpu_ref_is_dying(&root->cgrp.self.refcnt)) in cgroup_kill_sb()
2390 percpu_ref_kill(&root->cgrp.self.refcnt); in cgroup_kill_sb()
2391 cgroup_put(&root->cgrp); in cgroup_kill_sb()
2484 int cgroup_path_ns_locked(struct cgroup *cgrp, char *buf, size_t buflen, in cgroup_path_ns_locked() argument
2487 struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root); in cgroup_path_ns_locked()
2489 return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen); in cgroup_path_ns_locked()
2492 int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen, in cgroup_path_ns() argument
2500 ret = cgroup_path_ns_locked(cgrp, buf, buflen, ns); in cgroup_path_ns()
3156 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_controllers_show() local
3158 cgroup_print_ss_mask(seq, cgroup_control(cgrp)); in cgroup_controllers_show()
3165 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_subtree_control_show() local
3167 cgroup_print_ss_mask(seq, cgrp->subtree_control); in cgroup_subtree_control_show()
3180 static int cgroup_update_dfl_csses(struct cgroup *cgrp) in cgroup_update_dfl_csses() argument
3194 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_update_dfl_csses()
3203 if (dsct == cgrp) in cgroup_update_dfl_csses()
3257 void cgroup_lock_and_drain_offline(struct cgroup *cgrp) in cgroup_lock_and_drain_offline() argument
3268 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { in cgroup_lock_and_drain_offline()
3298 static void cgroup_save_control(struct cgroup *cgrp) in cgroup_save_control() argument
3303 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_save_control()
3318 static void cgroup_propagate_control(struct cgroup *cgrp) in cgroup_propagate_control() argument
3323 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_propagate_control()
3339 static void cgroup_restore_control(struct cgroup *cgrp) in cgroup_restore_control() argument
3344 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { in cgroup_restore_control()
3354 struct cgroup *cgrp = css->cgroup; in css_visible() local
3356 if (cgroup_control(cgrp) & (1 << ss->id)) in css_visible()
3358 if (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) in css_visible()
3360 return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl; in css_visible()
3376 static int cgroup_apply_control_enable(struct cgroup *cgrp) in cgroup_apply_control_enable() argument
3383 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { in cgroup_apply_control_enable()
3422 static void cgroup_apply_control_disable(struct cgroup *cgrp) in cgroup_apply_control_disable() argument
3429 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { in cgroup_apply_control_disable()
3467 static int cgroup_apply_control(struct cgroup *cgrp) in cgroup_apply_control() argument
3471 cgroup_propagate_control(cgrp); in cgroup_apply_control()
3473 ret = cgroup_apply_control_enable(cgrp); in cgroup_apply_control()
3482 return cgroup_update_dfl_csses(cgrp); in cgroup_apply_control()
3492 static void cgroup_finalize_control(struct cgroup *cgrp, int ret) in cgroup_finalize_control() argument
3495 cgroup_restore_control(cgrp); in cgroup_finalize_control()
3496 cgroup_propagate_control(cgrp); in cgroup_finalize_control()
3499 cgroup_apply_control_disable(cgrp); in cgroup_finalize_control()
3502 static int cgroup_vet_subtree_control_enable(struct cgroup *cgrp, u16 enable) in cgroup_vet_subtree_control_enable() argument
3511 if (!cgroup_is_valid_domain(cgrp->dom_cgrp)) in cgroup_vet_subtree_control_enable()
3515 if (cgroup_is_mixable(cgrp)) in cgroup_vet_subtree_control_enable()
3520 if (cgroup_is_thread_root(cgrp) || cgroup_is_threaded(cgrp)) in cgroup_vet_subtree_control_enable()
3528 if (cgroup_can_be_thread_root(cgrp) || cgroup_is_threaded(cgrp)) in cgroup_vet_subtree_control_enable()
3536 if (cgroup_has_tasks(cgrp)) in cgroup_vet_subtree_control_enable()
3548 struct cgroup *cgrp, *child; in cgroup_subtree_control_write() local
3581 cgrp = cgroup_kn_lock_live(of->kn, true); in cgroup_subtree_control_write()
3582 if (!cgrp) in cgroup_subtree_control_write()
3587 if (cgrp->subtree_control & (1 << ssid)) { in cgroup_subtree_control_write()
3592 if (!(cgroup_control(cgrp) & (1 << ssid))) { in cgroup_subtree_control_write()
3597 if (!(cgrp->subtree_control & (1 << ssid))) { in cgroup_subtree_control_write()
3603 cgroup_for_each_live_child(child, cgrp) { in cgroup_subtree_control_write()
3617 ret = cgroup_vet_subtree_control_enable(cgrp, enable); in cgroup_subtree_control_write()
3622 cgroup_save_control(cgrp); in cgroup_subtree_control_write()
3624 cgrp->subtree_control |= enable; in cgroup_subtree_control_write()
3625 cgrp->subtree_control &= ~disable; in cgroup_subtree_control_write()
3627 ret = cgroup_apply_control(cgrp); in cgroup_subtree_control_write()
3628 cgroup_finalize_control(cgrp, ret); in cgroup_subtree_control_write()
3632 kernfs_activate(cgrp->kn); in cgroup_subtree_control_write()
3647 static int cgroup_enable_threaded(struct cgroup *cgrp) in cgroup_enable_threaded() argument
3649 struct cgroup *parent = cgroup_parent(cgrp); in cgroup_enable_threaded()
3658 if (cgroup_is_threaded(cgrp)) in cgroup_enable_threaded()
3667 if (cgroup_is_populated(cgrp) || in cgroup_enable_threaded()
3668 cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask) in cgroup_enable_threaded()
3680 cgroup_save_control(cgrp); in cgroup_enable_threaded()
3682 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) in cgroup_enable_threaded()
3683 if (dsct == cgrp || cgroup_is_threaded(dsct)) in cgroup_enable_threaded()
3686 ret = cgroup_apply_control(cgrp); in cgroup_enable_threaded()
3690 cgroup_finalize_control(cgrp, ret); in cgroup_enable_threaded()
3696 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_type_show() local
3698 if (cgroup_is_threaded(cgrp)) in cgroup_type_show()
3700 else if (!cgroup_is_valid_domain(cgrp)) in cgroup_type_show()
3702 else if (cgroup_is_thread_root(cgrp)) in cgroup_type_show()
3713 struct cgroup *cgrp; in cgroup_type_write() local
3721 cgrp = cgroup_kn_lock_live(of->kn, true); in cgroup_type_write()
3722 if (!cgrp) in cgroup_type_write()
3726 ret = cgroup_enable_threaded(cgrp); in cgroup_type_write()
3734 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_max_descendants_show() local
3735 int descendants = READ_ONCE(cgrp->max_descendants); in cgroup_max_descendants_show()
3748 struct cgroup *cgrp; in cgroup_max_descendants_write() local
3764 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_max_descendants_write()
3765 if (!cgrp) in cgroup_max_descendants_write()
3768 cgrp->max_descendants = descendants; in cgroup_max_descendants_write()
3777 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_max_depth_show() local
3778 int depth = READ_ONCE(cgrp->max_depth); in cgroup_max_depth_show()
3791 struct cgroup *cgrp; in cgroup_max_depth_write() local
3807 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_max_depth_write()
3808 if (!cgrp) in cgroup_max_depth_write()
3811 cgrp->max_depth = depth; in cgroup_max_depth_write()
3820 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_events_show() local
3822 seq_printf(seq, "populated %d\n", cgroup_is_populated(cgrp)); in cgroup_events_show()
3823 seq_printf(seq, "frozen %d\n", test_bit(CGRP_FROZEN, &cgrp->flags)); in cgroup_events_show()
3870 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_core_local_stat_show() local
3875 sequence = read_seqcount_begin(&cgrp->freezer.freeze_seq); in cgroup_core_local_stat_show()
3876 freeze_time = cgrp->freezer.frozen_nsec; in cgroup_core_local_stat_show()
3878 if (test_bit(CGRP_FREEZE, &cgrp->flags)) in cgroup_core_local_stat_show()
3880 cgrp->freezer.freeze_start_nsec); in cgroup_core_local_stat_show()
3881 } while (read_seqcount_retry(&cgrp->freezer.freeze_seq, sequence)); in cgroup_core_local_stat_show()
3898 static struct cgroup_subsys_state *cgroup_tryget_css(struct cgroup *cgrp, in cgroup_tryget_css() argument
3904 css = cgroup_css(cgrp, ss); in cgroup_tryget_css()
3914 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_extra_stat_show() local
3922 css = cgroup_tryget_css(cgrp, ss); in cgroup_extra_stat_show()
3932 struct cgroup *cgrp, int ssid) in cgroup_local_stat_show() argument
3941 css = cgroup_tryget_css(cgrp, ss); in cgroup_local_stat_show()
3964 struct cgroup __maybe_unused *cgrp = seq_css(seq)->cgroup; in cpu_local_stat_show() local
3968 ret = cgroup_local_stat_show(seq, cgrp, cpu_cgrp_id); in cpu_local_stat_show()
3976 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_io_pressure_show() local
3977 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_io_pressure_show()
3983 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_memory_pressure_show() local
3984 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_memory_pressure_show()
3990 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_cpu_pressure_show() local
3991 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_cpu_pressure_show()
4001 struct cgroup *cgrp; in pressure_write() local
4004 cgrp = cgroup_kn_lock_live(of->kn, false); in pressure_write()
4005 if (!cgrp) in pressure_write()
4008 cgroup_get(cgrp); in pressure_write()
4013 cgroup_put(cgrp); in pressure_write()
4017 psi = cgroup_psi(cgrp); in pressure_write()
4020 cgroup_put(cgrp); in pressure_write()
4025 cgroup_put(cgrp); in pressure_write()
4054 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_irq_pressure_show() local
4055 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_irq_pressure_show()
4070 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_pressure_show() local
4071 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_pressure_show()
4084 struct cgroup *cgrp; in cgroup_pressure_write() local
4094 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_pressure_write()
4095 if (!cgrp) in cgroup_pressure_write()
4098 psi = cgroup_psi(cgrp); in cgroup_pressure_write()
4104 cgroup_file_show(&cgrp->psi_files[i], enable); in cgroup_pressure_write()
4149 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_freeze_show() local
4151 seq_printf(seq, "%d\n", cgrp->freezer.freeze); in cgroup_freeze_show()
4159 struct cgroup *cgrp; in cgroup_freeze_write() local
4170 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_freeze_write()
4171 if (!cgrp) in cgroup_freeze_write()
4174 cgroup_freeze(cgrp, freeze); in cgroup_freeze_write()
4181 static void __cgroup_kill(struct cgroup *cgrp) in __cgroup_kill() argument
4189 cgrp->kill_seq++; in __cgroup_kill()
4192 css_task_iter_start(&cgrp->self, CSS_TASK_ITER_PROCS | CSS_TASK_ITER_THREADED, &it); in __cgroup_kill()
4207 static void cgroup_kill(struct cgroup *cgrp) in cgroup_kill() argument
4214 cgroup_for_each_live_descendant_pre(dsct, css, cgrp) in cgroup_kill()
4223 struct cgroup *cgrp; in cgroup_kill_write() local
4232 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_kill_write()
4233 if (!cgrp) in cgroup_kill_write()
4241 if (cgroup_is_threaded(cgrp)) in cgroup_kill_write()
4244 cgroup_kill(cgrp); in cgroup_kill_write()
4292 struct cgroup *cgrp = kn_priv(of->kn); in cgroup_file_write() local
4306 if ((cgrp->root->flags & CGRP_ROOT_NS_DELEGATE) && in cgroup_file_write()
4308 ctx->ns != &init_cgroup_ns && ctx->ns->root_cset->dfl_cgrp == cgrp) in cgroup_file_write()
4321 css = cgroup_css(cgrp, cft->ss); in cgroup_file_write()
4411 static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp, in cgroup_add_file() argument
4421 kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name), in cgroup_add_file()
4453 struct cgroup *cgrp, struct cftype cfts[], in cgroup_addrm_files() argument
4464 if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp)) in cgroup_addrm_files()
4466 if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp)) in cgroup_addrm_files()
4468 if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp)) in cgroup_addrm_files()
4470 if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp)) in cgroup_addrm_files()
4475 ret = cgroup_add_file(css, cgrp, cft); in cgroup_addrm_files()
4484 cgroup_rm_file(cgrp, cft); in cgroup_addrm_files()
4493 struct cgroup *root = &ss->root->cgrp; in cgroup_apply_cftypes()
4501 struct cgroup *cgrp = css->cgroup; in cgroup_apply_cftypes() local
4506 ret = cgroup_addrm_files(css, cgrp, cfts, is_add); in cgroup_apply_cftypes()
5248 struct cgroup *cgrp = seq_css(s)->cgroup; in __cgroup_procs_start() local
5259 css_task_iter_start(&cgrp->self, iter_flags, it); in __cgroup_procs_start()
5263 css_task_iter_start(&cgrp->self, iter_flags, it); in __cgroup_procs_start()
5272 struct cgroup *cgrp = seq_css(s)->cgroup; in cgroup_procs_start() local
5280 if (cgroup_is_threaded(cgrp)) in cgroup_procs_start()
5293 static int cgroup_may_write(const struct cgroup *cgrp, struct super_block *sb) in cgroup_may_write() argument
5300 inode = kernfs_get_inode(sb, cgrp->procs_file.kn); in cgroup_may_write()
5579 struct cgroup *cgrp = css->cgroup; in css_free_rwork_fn() local
5591 cgroup_put(cgrp); in css_free_rwork_fn()
5597 atomic_dec(&cgrp->root->nr_cgrps); in css_free_rwork_fn()
5598 if (!cgroup_on_dfl(cgrp)) in css_free_rwork_fn()
5599 cgroup1_pidlist_destroy_all(cgrp); in css_free_rwork_fn()
5600 cancel_work_sync(&cgrp->release_agent_work); in css_free_rwork_fn()
5601 bpf_cgrp_storage_free(cgrp); in css_free_rwork_fn()
5603 if (cgroup_parent(cgrp)) { in css_free_rwork_fn()
5610 cgroup_put(cgroup_parent(cgrp)); in css_free_rwork_fn()
5611 kernfs_put(cgrp->kn); in css_free_rwork_fn()
5612 psi_cgroup_free(cgrp); in css_free_rwork_fn()
5613 kfree(cgrp); in css_free_rwork_fn()
5620 cgroup_destroy_root(cgrp->root); in css_free_rwork_fn()
5630 struct cgroup *cgrp = css->cgroup; in css_release_work_fn() local
5646 cgrp->nr_dying_subsys[ss->id]--; in css_release_work_fn()
5655 parent_cgrp = cgroup_parent(cgrp); in css_release_work_fn()
5664 TRACE_CGROUP_PATH(release, cgrp); in css_release_work_fn()
5666 css_rstat_flush(&cgrp->self); in css_release_work_fn()
5669 for (tcgrp = cgroup_parent(cgrp); tcgrp; in css_release_work_fn()
5681 if (cgrp->kn) in css_release_work_fn()
5682 RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, in css_release_work_fn()
5702 struct cgroup_subsys *ss, struct cgroup *cgrp) in init_and_link_css() argument
5706 cgroup_get_live(cgrp); in init_and_link_css()
5709 css->cgroup = cgrp; in init_and_link_css()
5717 if (cgroup_parent(cgrp)) { in init_and_link_css()
5718 css->parent = cgroup_css(cgroup_parent(cgrp), ss); in init_and_link_css()
5722 BUG_ON(cgroup_css(cgrp, ss)); in init_and_link_css()
5787 static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, in css_create() argument
5790 struct cgroup *parent = cgroup_parent(cgrp); in css_create()
5803 init_and_link_css(css, ss, cgrp); in css_create()
5844 struct cgroup *cgrp, *tcgrp; in cgroup_create() local
5850 cgrp = kzalloc(struct_size(cgrp, ancestors, (level + 1)), GFP_KERNEL); in cgroup_create()
5851 if (!cgrp) in cgroup_create()
5854 ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL); in cgroup_create()
5861 cgrp, NULL); in cgroup_create()
5866 cgrp->kn = kn; in cgroup_create()
5868 init_cgroup_housekeeping(cgrp); in cgroup_create()
5870 cgrp->self.parent = &parent->self; in cgroup_create()
5871 cgrp->root = root; in cgroup_create()
5872 cgrp->level = level; in cgroup_create()
5878 ret = css_rstat_init(&cgrp->self); in cgroup_create()
5882 ret = psi_cgroup_alloc(cgrp); in cgroup_create()
5886 for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) in cgroup_create()
5887 cgrp->ancestors[tcgrp->level] = tcgrp; in cgroup_create()
5893 cgrp->freezer.e_freeze = parent->freezer.e_freeze; in cgroup_create()
5894 seqcount_spinlock_init(&cgrp->freezer.freeze_seq, &css_set_lock); in cgroup_create()
5895 if (cgrp->freezer.e_freeze) { in cgroup_create()
5902 set_bit(CGRP_FREEZE, &cgrp->flags); in cgroup_create()
5903 cgrp->freezer.freeze_start_nsec = ktime_get_ns(); in cgroup_create()
5904 set_bit(CGRP_FROZEN, &cgrp->flags); in cgroup_create()
5908 set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); in cgroup_create()
5911 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags); in cgroup_create()
5913 cgrp->self.serial_nr = css_serial_nr_next++; in cgroup_create()
5917 CGROUP_LIFETIME_OFFLINE, cgrp); in cgroup_create()
5925 tcgrp = cgrp->ancestors[i]; in cgroup_create()
5933 if (cgrp->freezer.e_freeze) in cgroup_create()
5938 list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children); in cgroup_create()
5946 if (!cgroup_on_dfl(cgrp)) in cgroup_create()
5947 cgrp->subtree_control = cgroup_control(cgrp); in cgroup_create()
5949 cgroup_propagate_control(cgrp); in cgroup_create()
5951 return cgrp; in cgroup_create()
5954 psi_cgroup_free(cgrp); in cgroup_create()
5956 css_rstat_exit(&cgrp->self); in cgroup_create()
5958 kernfs_remove(cgrp->kn); in cgroup_create()
5960 percpu_ref_exit(&cgrp->self.refcnt); in cgroup_create()
5962 kfree(cgrp); in cgroup_create()
5991 struct cgroup *parent, *cgrp; in cgroup_mkdir() local
6007 cgrp = cgroup_create(parent, name, mode); in cgroup_mkdir()
6008 if (IS_ERR(cgrp)) { in cgroup_mkdir()
6009 ret = PTR_ERR(cgrp); in cgroup_mkdir()
6017 kernfs_get(cgrp->kn); in cgroup_mkdir()
6019 ret = css_populate_dir(&cgrp->self); in cgroup_mkdir()
6023 ret = cgroup_apply_control_enable(cgrp); in cgroup_mkdir()
6027 TRACE_CGROUP_PATH(mkdir, cgrp); in cgroup_mkdir()
6030 kernfs_activate(cgrp->kn); in cgroup_mkdir()
6036 cgroup_destroy_locked(cgrp); in cgroup_mkdir()
6149 static int cgroup_destroy_locked(struct cgroup *cgrp) in cgroup_destroy_locked() argument
6152 struct cgroup *tcgrp, *parent = cgroup_parent(cgrp); in cgroup_destroy_locked()
6163 if (cgroup_is_populated(cgrp)) in cgroup_destroy_locked()
6171 if (css_has_online_children(&cgrp->self)) in cgroup_destroy_locked()
6180 cgrp->self.flags &= ~CSS_ONLINE; in cgroup_destroy_locked()
6183 list_for_each_entry(link, &cgrp->cset_links, cset_link) in cgroup_destroy_locked()
6188 for_each_css(css, ssid, cgrp) in cgroup_destroy_locked()
6192 css_clear_dir(&cgrp->self); in cgroup_destroy_locked()
6193 kernfs_remove(cgrp->kn); in cgroup_destroy_locked()
6195 if (cgroup_is_threaded(cgrp)) in cgroup_destroy_locked()
6206 if (test_bit(CGRP_FROZEN, &cgrp->flags)) in cgroup_destroy_locked()
6214 CGROUP_LIFETIME_OFFLINE, cgrp); in cgroup_destroy_locked()
6218 percpu_ref_kill(&cgrp->self.refcnt); in cgroup_destroy_locked()
6225 struct cgroup *cgrp; in cgroup_rmdir() local
6228 cgrp = cgroup_kn_lock_live(kn, false); in cgroup_rmdir()
6229 if (!cgrp) in cgroup_rmdir()
6232 ret = cgroup_destroy_locked(cgrp); in cgroup_rmdir()
6234 TRACE_CGROUP_PATH(rmdir, cgrp); in cgroup_rmdir()
6263 init_and_link_css(css, ss, &cgrp_dfl_root.cgrp); in cgroup_init_subsys()
6317 cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF; in cgroup_init_early()
6391 &cgrp_dfl_root.cgrp.e_csets[ssid]); in cgroup_init()
6492 struct cgroup *cgrp; in __cgroup_get_from_id() local
6505 cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv); in __cgroup_get_from_id()
6506 if (cgrp && !cgroup_tryget(cgrp)) in __cgroup_get_from_id()
6507 cgrp = NULL; in __cgroup_get_from_id()
6512 if (!cgrp) in __cgroup_get_from_id()
6514 return cgrp; in __cgroup_get_from_id()
6525 struct cgroup *cgrp, *root_cgrp; in cgroup_get_from_id() local
6527 cgrp = __cgroup_get_from_id(id); in cgroup_get_from_id()
6528 if (IS_ERR(cgrp)) in cgroup_get_from_id()
6529 return cgrp; in cgroup_get_from_id()
6532 if (!cgroup_is_descendant(cgrp, root_cgrp)) { in cgroup_get_from_id()
6533 cgroup_put(cgrp); in cgroup_get_from_id()
6537 return cgrp; in cgroup_get_from_id()
6563 struct cgroup *cgrp; in proc_cgroup_show() local
6569 cgrp = task_cgroup_from_root(tsk, root); in proc_cgroup_show()
6571 if (!cgrp) in proc_cgroup_show()
6593 if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) { in proc_cgroup_show()
6594 retval = cgroup_path_ns_locked(cgrp, buf, PATH_MAX, in proc_cgroup_show()
6606 if (cgroup_on_dfl(cgrp) && cgroup_is_dead(cgrp)) in proc_cgroup_show()
6660 struct cgroup *cgrp = cgroup_v1v2_get_from_file(f); in cgroup_get_from_file() local
6662 if (IS_ERR(cgrp)) in cgroup_get_from_file()
6663 return ERR_CAST(cgrp); in cgroup_get_from_file()
6665 if (!cgroup_on_dfl(cgrp)) { in cgroup_get_from_file()
6666 cgroup_put(cgrp); in cgroup_get_from_file()
6670 return cgrp; in cgroup_get_from_file()
6705 if (kargs->cgrp) in cgroup_css_set_fork()
6706 kargs->kill_seq = kargs->cgrp->kill_seq; in cgroup_css_set_fork()
6771 kargs->cgrp = dst_cgrp; in cgroup_css_set_fork()
6795 struct cgroup *cgrp = kargs->cgrp; in cgroup_css_set_put_fork() local
6807 if (cgrp) { in cgroup_css_set_put_fork()
6808 cgroup_put(cgrp); in cgroup_css_set_put_fork()
6809 kargs->cgrp = NULL; in cgroup_css_set_put_fork()
6903 if (kargs->cgrp) { in cgroup_post_fork()
6904 cgrp_flags = kargs->cgrp->flags; in cgroup_post_fork()
6905 cgrp_kill_seq = kargs->cgrp->kill_seq; in cgroup_post_fork()
7157 struct cgroup *cgrp; in css_tryget_online_from_dir() local
7171 cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv); in css_tryget_online_from_dir()
7172 if (cgrp) in css_tryget_online_from_dir()
7173 css = cgroup_css(cgrp, ss); in css_tryget_online_from_dir()
7208 struct cgroup *cgrp = ERR_PTR(-ENOENT); in cgroup_get_from_path() local
7217 cgrp = ERR_PTR(-ENOTDIR); in cgroup_get_from_path()
7223 cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv); in cgroup_get_from_path()
7224 if (!cgrp || !cgroup_tryget(cgrp)) in cgroup_get_from_path()
7225 cgrp = ERR_PTR(-ENOENT); in cgroup_get_from_path()
7232 return cgrp; in cgroup_get_from_path()
7261 struct cgroup *cgrp = cgroup_v1v2_get_from_fd(fd); in cgroup_get_from_fd() local
7263 if (IS_ERR(cgrp)) in cgroup_get_from_fd()
7264 return ERR_CAST(cgrp); in cgroup_get_from_fd()
7266 if (!cgroup_on_dfl(cgrp)) { in cgroup_get_from_fd()
7267 cgroup_put(cgrp); in cgroup_get_from_fd()
7270 return cgrp; in cgroup_get_from_fd()
7329 cgroup = &cgrp_dfl_root.cgrp; in cgroup_sk_alloc()
7352 struct cgroup *cgrp = sock_cgroup_ptr(skcd); in cgroup_sk_clone() local
7359 cgroup_get(cgrp); in cgroup_sk_clone()
7360 cgroup_bpf_get(cgrp); in cgroup_sk_clone()
7365 struct cgroup *cgrp = sock_cgroup_ptr(skcd); in cgroup_sk_free() local
7367 cgroup_bpf_put(cgrp); in cgroup_sk_free()
7368 cgroup_put(cgrp); in cgroup_sk_free()