19  * Determines whether a given css can participate in rstat.
20  * css's that are cgroup::self use rstat for base stats.
21  * Other css's associated with a subsystem use rstat only when
24 static inline bool css_uses_rstat(struct cgroup_subsys_state *css)  in css_uses_rstat()  argument
26 	return css_is_self(css) || css->ss->css_rstat_flush != NULL;  in css_uses_rstat()
30 		struct cgroup_subsys_state *css, int cpu)  in css_rstat_cpu()  argument
32 	return per_cpu_ptr(css->rstat_cpu, cpu);  in css_rstat_cpu()
58  * @css: target cgroup subsystem state
61  * Atomically inserts the css in the ss's llist for the given cpu. This is
71 void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu)  in __css_rstat_updated()  argument
78 	if (!css_uses_rstat(css))  in __css_rstat_updated()
90 	rstatc = css_rstat_cpu(css, cpu);  in __css_rstat_updated()
121 	lhead = ss_lhead_cpu(css->ss, cpu);  in __css_rstat_updated()
129 __bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)  in css_rstat_updated()  argument
134 	__css_rstat_updated(css, cpu);  in css_rstat_updated()
137 static void __css_process_update_tree(struct cgroup_subsys_state *css, int cpu)  in __css_process_update_tree()  argument
139 	/* put @css and all ancestors on the corresponding updated lists */  in __css_process_update_tree()
141 		struct css_rstat_cpu *rstatc = css_rstat_cpu(css, cpu);  in __css_process_update_tree()
142 		struct cgroup_subsys_state *parent = css->parent;  in __css_process_update_tree()
154 			rstatc->updated_next = css;  in __css_process_update_tree()
160 		prstatc->updated_children = css;  in __css_process_update_tree()
162 		css = parent;  in __css_process_update_tree()
195  * css_rstat_push_children - push children css's into the given list
199  * Return: A new singly linked list of css's to be flushed
204  * tail backward like "pushing" css's into a stack. The root is pushed by
211 	struct cgroup_subsys_state *cnext = child;	/* Next head of child css level */  in css_rstat_push_children()
212 	struct cgroup_subsys_state *ghead = NULL;	/* Head of grandchild css level */  in css_rstat_push_children()
273  * css_rstat_updated_list - build a list of updated css's to be flushed
274  * @root: root of the css subtree to traverse
276  * Return: A singly linked list of css's to be flushed
279  * each returned css is unlinked from the updated tree.
286  * child css's if not empty. Whereas updated_next is like a sibling link
287  * within the children list and terminated by the parent css. An exception
288  * here is the css root whose updated_next can be self terminated.
367 static inline void __css_rstat_lock(struct cgroup_subsys_state *css,  in __css_rstat_lock()  argument
369 	__acquires(ss_rstat_lock(css->ss))  in __css_rstat_lock()
371 	struct cgroup *cgrp = css->cgroup;  in __css_rstat_lock()
375 	lock = ss_rstat_lock(css->ss);  in __css_rstat_lock()
384 static inline void __css_rstat_unlock(struct cgroup_subsys_state *css,  in __css_rstat_unlock()  argument
386 	__releases(ss_rstat_lock(css->ss))  in __css_rstat_unlock()
388 	struct cgroup *cgrp = css->cgroup;  in __css_rstat_unlock()
391 	lock = ss_rstat_lock(css->ss);  in __css_rstat_unlock()
397  * css_rstat_flush - flush stats in @css's rstat subtree
398  * @css: target cgroup subsystem state
400  * Collect all per-cpu stats in @css's subtree into the global counters
404  * This also gets all rstat nodes in the subtree including @css off the
409 __bpf_kfunc void css_rstat_flush(struct cgroup_subsys_state *css)  in css_rstat_flush()  argument
412 	bool is_self = css_is_self(css);  in css_rstat_flush()
418 	if (!css_uses_rstat(css))  in css_rstat_flush()
426 		__css_rstat_lock(css, cpu);  in css_rstat_flush()
427 		pos = css_rstat_updated_list(css, cpu);  in css_rstat_flush()
436 		__css_rstat_unlock(css, cpu);  in css_rstat_flush()
442 int css_rstat_init(struct cgroup_subsys_state *css)  in css_rstat_init()  argument
444 	struct cgroup *cgrp = css->cgroup;  in css_rstat_init()
446 	bool is_self = css_is_self(css);  in css_rstat_init()
455 	} else if (css->ss->css_rstat_flush == NULL)  in css_rstat_init()
458 	/* the root cgrp's self css has rstat_cpu preallocated */  in css_rstat_init()
459 	if (!css->rstat_cpu) {  in css_rstat_init()
460 		css->rstat_cpu = alloc_percpu(struct css_rstat_cpu);  in css_rstat_init()
461 		if (!css->rstat_cpu) {  in css_rstat_init()
471 		struct css_rstat_cpu *rstatc = css_rstat_cpu(css, cpu);  in css_rstat_init()
473 		rstatc->owner = rstatc->updated_children = css;  in css_rstat_init()
487 void css_rstat_exit(struct cgroup_subsys_state *css)  in css_rstat_exit()  argument
491 	if (!css_uses_rstat(css))  in css_rstat_exit()
494 	if (!css->rstat_cpu)  in css_rstat_exit()
497 	css_rstat_flush(css);  in css_rstat_exit()
501 		struct css_rstat_cpu *rstatc = css_rstat_cpu(css, cpu);  in css_rstat_exit()
503 		if (WARN_ON_ONCE(rstatc->updated_children != css) ||  in css_rstat_exit()
508 	if (css_is_self(css)) {  in css_rstat_exit()
509 		struct cgroup *cgrp = css->cgroup;  in css_rstat_exit()
515 	free_percpu(css->rstat_cpu);  in css_rstat_exit()
516 	css->rstat_cpu = NULL;  in css_rstat_exit()