15 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
35 		struct cgroup *cgrp, int cpu)  in cgroup_rstat_base_cpu()  argument
37 	return per_cpu_ptr(cgrp->rstat_base_cpu, cpu);  in cgroup_rstat_base_cpu()
342 __weak noinline void bpf_rstat_flush(struct cgroup *cgrp,  in bpf_rstat_flush()  argument
362 	struct cgroup *cgrp = css->cgroup;  in __css_rstat_lock()  local
369 		trace_cgroup_rstat_lock_contended(cgrp, cpu_in_loop, contended);  in __css_rstat_lock()
372 	trace_cgroup_rstat_locked(cgrp, cpu_in_loop, contended);  in __css_rstat_lock()
379 	struct cgroup *cgrp = css->cgroup;  in __css_rstat_unlock()  local
383 	trace_cgroup_rstat_unlock(cgrp, cpu_in_loop, false);  in __css_rstat_unlock()
435 	struct cgroup *cgrp = css->cgroup;  in css_rstat_init()  local
441 		if (!cgrp->rstat_base_cpu) {  in css_rstat_init()
442 			cgrp->rstat_base_cpu = alloc_percpu(struct cgroup_rstat_base_cpu);  in css_rstat_init()
443 			if (!cgrp->rstat_base_cpu)  in css_rstat_init()
454 				free_percpu(cgrp->rstat_base_cpu);  in css_rstat_init()
470 			rstatbc = cgroup_rstat_base_cpu(cgrp, cpu);  in css_rstat_init()
500 		struct cgroup *cgrp = css->cgroup;  in css_rstat_exit()  local
502 		free_percpu(cgrp->rstat_base_cpu);  in css_rstat_exit()
503 		cgrp->rstat_base_cpu = NULL;  in css_rstat_exit()
563 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)  in cgroup_base_stat_flush()  argument
565 	struct cgroup_rstat_base_cpu *rstatbc = cgroup_rstat_base_cpu(cgrp, cpu);  in cgroup_base_stat_flush()
566 	struct cgroup *parent = cgroup_parent(cgrp);  in cgroup_base_stat_flush()
583 	cgroup_base_stat_add(&cgrp->bstat, &delta);  in cgroup_base_stat_flush()
589 		delta = cgrp->bstat;  in cgroup_base_stat_flush()
590 		cgroup_base_stat_sub(&delta, &cgrp->last_bstat);  in cgroup_base_stat_flush()
592 		cgroup_base_stat_add(&cgrp->last_bstat, &delta);  in cgroup_base_stat_flush()
603 cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp, unsigned long *flags)  in cgroup_base_stat_cputime_account_begin()  argument
607 	rstatbc = get_cpu_ptr(cgrp->rstat_base_cpu);  in cgroup_base_stat_cputime_account_begin()
612 static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,  in cgroup_base_stat_cputime_account_end()  argument
617 	css_rstat_updated(&cgrp->self, smp_processor_id());  in cgroup_base_stat_cputime_account_end()
621 void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)  in __cgroup_account_cputime()  argument
626 	rstatbc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);  in __cgroup_account_cputime()
628 	cgroup_base_stat_cputime_account_end(cgrp, rstatbc, flags);  in __cgroup_account_cputime()
631 void __cgroup_account_cputime_field(struct cgroup *cgrp,  in __cgroup_account_cputime_field()  argument
637 	rstatbc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);  in __cgroup_account_cputime_field()
660 	cgroup_base_stat_cputime_account_end(cgrp, rstatbc, flags);  in __cgroup_account_cputime_field()
715 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_base_stat_cputime_show()  local
718 	if (cgroup_parent(cgrp)) {  in cgroup_base_stat_cputime_show()
719 		css_rstat_flush(&cgrp->self);  in cgroup_base_stat_cputime_show()
720 		__css_rstat_lock(&cgrp->self, -1);  in cgroup_base_stat_cputime_show()
721 		bstat = cgrp->bstat;  in cgroup_base_stat_cputime_show()
722 		cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime,  in cgroup_base_stat_cputime_show()
724 		__css_rstat_unlock(&cgrp->self, -1);  in cgroup_base_stat_cputime_show()