1 /* SPDX-License-Identifier: GPL-2.0 */
16 		rq->rq_sched_info.run_delay += delta;  in rq_sched_info_arrive()
17 		rq->rq_sched_info.pcount++;  in rq_sched_info_arrive()
28 		rq->rq_cpu_time += delta;  in rq_sched_info_depart()
35 		rq->rq_sched_info.run_delay += delta;  in rq_sched_info_dequeue()
47 void __update_stats_wait_start(struct rq *rq, struct task_struct *p,
50 void __update_stats_wait_end(struct rq *rq, struct task_struct *p,
52 void __update_stats_enqueue_sleeper(struct rq *rq, struct task_struct *p,
85 # define __update_stats_wait_start(rq, p, stats)       do { } while (0)  argument
86 # define __update_stats_wait_end(rq, p, stats)         do { } while (0)  argument
87 # define __update_stats_enqueue_sleeper(rq, p, stats)  do { } while (0)  argument
104 		return &container_of(se, struct sched_entity_stats, se)->stats;  in __schedstats_from_se()
106 	return &task_of(se)->stats;  in __schedstats_from_se()
127  * go through migration requeues. In this case, *sleeping* states need
130 static inline void psi_enqueue(struct task_struct *p, int flags)  in psi_enqueue()  argument
142 	if (task_on_cpu(task_rq(p), p))  in psi_enqueue()
145 	if (p->se.sched_delayed) {  in psi_enqueue()
148 		if (p->in_memstall)  in psi_enqueue()
150 		if (p->in_iowait)  in psi_enqueue()
155 		if (p->in_memstall)  in psi_enqueue()
159 		if (p->in_iowait)  in psi_enqueue()
162 		if (p->in_memstall)  in psi_enqueue()
166 	psi_task_change(p, clear, set);  in psi_enqueue()
169 static inline void psi_dequeue(struct task_struct *p, int flags)  in psi_dequeue()  argument
189 	if ((flags & DEQUEUE_SLEEP) && (p->psi_flags & TSK_ONCPU))  in psi_dequeue()
196 	psi_task_change(p, p->psi_flags, 0);  in psi_dequeue()
199 static inline void psi_ttwu_dequeue(struct task_struct *p)  in psi_ttwu_dequeue()  argument
205 	 * deregister its sleep-persistent psi states from the old  in psi_ttwu_dequeue()
208 	if (unlikely(p->psi_flags)) {  in psi_ttwu_dequeue()
212 		rq = __task_rq_lock(p, &rf);  in psi_ttwu_dequeue()
213 		psi_task_change(p, p->psi_flags, 0);  in psi_ttwu_dequeue()
214 		__task_rq_unlock(rq, p, &rf);  in psi_ttwu_dequeue()
229 static inline void psi_enqueue(struct task_struct *p, bool migrate) {}  in psi_enqueue()  argument
230 static inline void psi_dequeue(struct task_struct *p, bool migrate) {}  in psi_dequeue()  argument
231 static inline void psi_ttwu_dequeue(struct task_struct *p) {}  in psi_ttwu_dequeue()  argument
243  * from dequeue_task() to account for possible rq->clock skew across CPUs. The
250 	if (!t->sched_info.last_queued)  in sched_info_dequeue()
253 	delta = rq_clock(rq) - t->sched_info.last_queued;  in sched_info_dequeue()
254 	t->sched_info.last_queued = 0;  in sched_info_dequeue()
255 	t->sched_info.run_delay += delta;  in sched_info_dequeue()
256 	if (delta > t->sched_info.max_run_delay)  in sched_info_dequeue()
257 		t->sched_info.max_run_delay = delta;  in sched_info_dequeue()
258 	if (delta && (!t->sched_info.min_run_delay || delta < t->sched_info.min_run_delay))  in sched_info_dequeue()
259 		t->sched_info.min_run_delay = delta;  in sched_info_dequeue()
266  * can keep stats on how long its time-slice is.
272 	if (!t->sched_info.last_queued)  in sched_info_arrive()
276 	delta = now - t->sched_info.last_queued;  in sched_info_arrive()
277 	t->sched_info.last_queued = 0;  in sched_info_arrive()
278 	t->sched_info.run_delay += delta;  in sched_info_arrive()
279 	t->sched_info.last_arrival = now;  in sched_info_arrive()
280 	t->sched_info.pcount++;  in sched_info_arrive()
281 	if (delta > t->sched_info.max_run_delay)  in sched_info_arrive()
282 		t->sched_info.max_run_delay = delta;  in sched_info_arrive()
283 	if (delta && (!t->sched_info.min_run_delay || delta < t->sched_info.min_run_delay))  in sched_info_arrive()
284 		t->sched_info.min_run_delay = delta;  in sched_info_arrive()
296 	if (!t->sched_info.last_queued)  in sched_info_enqueue()
297 		t->sched_info.last_queued = rq_clock(rq);  in sched_info_enqueue()
301  * Called when a process ceases being the active-running process involuntarily
310 	unsigned long long delta = rq_clock(rq) - t->sched_info.last_arrival;  in sched_info_depart()
331 	if (prev != rq->idle)  in sched_info_switch()
334 	if (next != rq->idle)  in sched_info_switch()